fix(k1): harden live handoff and camera recovery
This commit is contained in:
@@ -50,6 +50,7 @@ import type {
|
|||||||
} from "./core/observation/sessionArchive";
|
} from "./core/observation/sessionArchive";
|
||||||
import { useRecordedSessionAdmission } from "./core/observation/useRecordedSessionAdmission";
|
import { useRecordedSessionAdmission } from "./core/observation/useRecordedSessionAdmission";
|
||||||
import { useWorkspaceLayoutProfile } from "./core/observation/useWorkspaceLayoutProfile";
|
import { useWorkspaceLayoutProfile } from "./core/observation/useWorkspaceLayoutProfile";
|
||||||
|
import { viewerSettingsTargetIdentity } from "./core/observation/viewerSettingsTarget";
|
||||||
import { resolvePolygonRunRoute } from "./core/polygon/runArchive";
|
import { resolvePolygonRunRoute } from "./core/polygon/runArchive";
|
||||||
import {
|
import {
|
||||||
OBSERVATION_WORKSPACE_ID,
|
OBSERVATION_WORKSPACE_ID,
|
||||||
@@ -302,14 +303,7 @@ export default function App() {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
}, [recordedReplay, replayPresented, replaySources, runtime.state]);
|
}, [recordedReplay, replayPresented, replaySources, runtime.state]);
|
||||||
const viewerSettingsTargetIdentity = [
|
const viewerSettingsTarget = viewerSettingsTargetIdentity(runtime.state);
|
||||||
runtime.state?.activeDevice?.pluginId,
|
|
||||||
runtime.state?.activeDevice?.modelId,
|
|
||||||
runtime.state?.activeDevice?.instanceId,
|
|
||||||
runtime.state?.deviceSession?.sessionId,
|
|
||||||
runtime.state?.deviceSession?.deviceId,
|
|
||||||
runtime.state?.acquisition?.acquisitionId,
|
|
||||||
].filter(Boolean).join(":") || "local-runtime";
|
|
||||||
const observationLayout = useObservationLayout(
|
const observationLayout = useObservationLayout(
|
||||||
activeObservationSources,
|
activeObservationSources,
|
||||||
replayPresented ? undefined : runtime.setObservationSourceActive,
|
replayPresented ? undefined : runtime.setObservationSourceActive,
|
||||||
@@ -389,7 +383,7 @@ export default function App() {
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const profile = workspaceLayoutProfile.profile;
|
const profile = workspaceLayoutProfile.profile;
|
||||||
const applicationKey = profile
|
const applicationKey = profile
|
||||||
? `${profile.revision}:${viewerSettingsTargetIdentity}`
|
? `${profile.revision}:${viewerSettingsTarget}`
|
||||||
: null;
|
: null;
|
||||||
if (
|
if (
|
||||||
!profile ||
|
!profile ||
|
||||||
@@ -410,7 +404,7 @@ export default function App() {
|
|||||||
}, [
|
}, [
|
||||||
runtime.backendStatus,
|
runtime.backendStatus,
|
||||||
runtime.updateViewerSettings,
|
runtime.updateViewerSettings,
|
||||||
viewerSettingsTargetIdentity,
|
viewerSettingsTarget,
|
||||||
workspaceLayoutProfile.profile,
|
workspaceLayoutProfile.profile,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,14 @@ import {
|
|||||||
reduceCameraPlaybackRecovery,
|
reduceCameraPlaybackRecovery,
|
||||||
type CameraPlaybackRecoveryEvent,
|
type CameraPlaybackRecoveryEvent,
|
||||||
} from "../core/observation/liveCameraRecovery";
|
} from "../core/observation/liveCameraRecovery";
|
||||||
import { subscribeToLiveViewerBuildFence } from "../core/observation/liveViewerDiagnostics";
|
import {
|
||||||
|
createLiveViewerDiagnosticLifecycle,
|
||||||
|
createLiveViewerInstanceId,
|
||||||
|
createLiveViewerLineage,
|
||||||
|
subscribeToLiveViewerBuildFence,
|
||||||
|
type LiveViewerDiagnosticLifecycle,
|
||||||
|
type LiveViewerFailureStage,
|
||||||
|
} from "../core/observation/liveViewerDiagnostics";
|
||||||
|
|
||||||
type PlayerStatus = "connecting" | "buffering" | "playing" | "error";
|
type PlayerStatus = "connecting" | "buffering" | "playing" | "error";
|
||||||
|
|
||||||
@@ -168,6 +175,15 @@ export function cameraTransportCloseRecoveryMessage(code: number): string {
|
|||||||
return "Browser-preview прерван; восстанавливаем текущую камеру.";
|
return "Browser-preview прерван; восстанавливаем текущую камеру.";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function cameraPlaybackRecoveryFailureStage(
|
||||||
|
event: CameraPlaybackRecoveryEvent,
|
||||||
|
): LiveViewerFailureStage {
|
||||||
|
if (event.type === "heartbeat") return "camera-heartbeat-reopen";
|
||||||
|
if (event.type === "network-online") return "camera-network-online-reopen";
|
||||||
|
if (event.type === "page-restore") return "camera-page-restore-reopen";
|
||||||
|
return "camera-visibility-reopen";
|
||||||
|
}
|
||||||
|
|
||||||
function websocketUrl(path: string): string {
|
function websocketUrl(path: string): string {
|
||||||
const url = new URL(path, window.location.href);
|
const url = new URL(path, window.location.href);
|
||||||
if (url.protocol === "http:") url.protocol = "ws:";
|
if (url.protocol === "http:") url.protocol = "ws:";
|
||||||
@@ -195,6 +211,9 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
const transportEpochRef = useRef(0);
|
const transportEpochRef = useRef(0);
|
||||||
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
|
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
|
||||||
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
|
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
|
||||||
|
const activeDiagnosticRef = useRef<LiveViewerDiagnosticLifecycle | null>(null);
|
||||||
|
const diagnosticInstanceIdRef = useRef<string | null>(null);
|
||||||
|
const diagnosticGenerationRef = useRef(0);
|
||||||
const uiBuildStaleRef = useRef(false);
|
const uiBuildStaleRef = useRef(false);
|
||||||
const [attempt, setAttempt] = useState(0);
|
const [attempt, setAttempt] = useState(0);
|
||||||
const [uiBuildStale, setUiBuildStale] = useState(false);
|
const [uiBuildStale, setUiBuildStale] = useState(false);
|
||||||
@@ -232,6 +251,16 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
let disposed = false;
|
let disposed = false;
|
||||||
const transportEpoch = transportEpochRef.current + 1;
|
const transportEpoch = transportEpochRef.current + 1;
|
||||||
transportEpochRef.current = transportEpoch;
|
transportEpochRef.current = transportEpoch;
|
||||||
|
diagnosticInstanceIdRef.current ??= createLiveViewerInstanceId();
|
||||||
|
diagnosticGenerationRef.current += 1;
|
||||||
|
const diagnosticLifecycle = createLiveViewerDiagnosticLifecycle({
|
||||||
|
lineage: createLiveViewerLineage(
|
||||||
|
diagnosticInstanceIdRef.current,
|
||||||
|
diagnosticGenerationRef.current,
|
||||||
|
),
|
||||||
|
});
|
||||||
|
activeDiagnosticRef.current = diagnosticLifecycle;
|
||||||
|
diagnosticLifecycle.verifyBuild();
|
||||||
const transportIsCurrent = () => cameraTransportCallbackIsCurrent(
|
const transportIsCurrent = () => cameraTransportCallbackIsCurrent(
|
||||||
transportEpochRef.current,
|
transportEpochRef.current,
|
||||||
transportEpoch,
|
transportEpoch,
|
||||||
@@ -245,6 +274,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
let retryTimer: number | undefined;
|
let retryTimer: number | undefined;
|
||||||
let receivedMedia = false;
|
let receivedMedia = false;
|
||||||
let failed = false;
|
let failed = false;
|
||||||
|
let playingReported = false;
|
||||||
let startupWatchdog: CameraStartupWatchdog | null = null;
|
let startupWatchdog: CameraStartupWatchdog | null = null;
|
||||||
transportHealthyRef.current = false;
|
transportHealthyRef.current = false;
|
||||||
const recovering = Boolean(
|
const recovering = Boolean(
|
||||||
@@ -271,7 +301,16 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const retryTransport = (copy: string) => {
|
const retryTransport = (
|
||||||
|
copy: string,
|
||||||
|
failureStage: LiveViewerFailureStage,
|
||||||
|
websocketCloseCode?: number,
|
||||||
|
appendFailure?: {
|
||||||
|
cameraAppendErrorName: string;
|
||||||
|
cameraMediaSourceState: "closed" | "open" | "ended";
|
||||||
|
cameraVideoErrorCode: number | null;
|
||||||
|
},
|
||||||
|
) => {
|
||||||
if (!transportIsCurrent() || failed) return;
|
if (!transportIsCurrent() || failed) return;
|
||||||
if (!cameraTransportRecoveryIsCurrent(
|
if (!cameraTransportRecoveryIsCurrent(
|
||||||
activeAuthorityRef.current,
|
activeAuthorityRef.current,
|
||||||
@@ -286,10 +325,21 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
startupWatchdog?.clear();
|
startupWatchdog?.clear();
|
||||||
failed = true;
|
failed = true;
|
||||||
transportHealthyRef.current = false;
|
transportHealthyRef.current = false;
|
||||||
queue.length = 0;
|
|
||||||
queuedBytes = 0;
|
|
||||||
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
|
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
|
||||||
leaseRetryRef.current = retry.budget;
|
leaseRetryRef.current = retry.budget;
|
||||||
|
diagnosticLifecycle.post({
|
||||||
|
eventCode: "live_camera_transport_restart_requested",
|
||||||
|
failureStage,
|
||||||
|
streamId: delivery.id,
|
||||||
|
cameraQueueBytes: queuedBytes,
|
||||||
|
cameraQueueSegments: queue.length,
|
||||||
|
cameraRetryCount: retry.budget.count,
|
||||||
|
websocketCloseCode,
|
||||||
|
transportEpoch,
|
||||||
|
...appendFailure,
|
||||||
|
});
|
||||||
|
queue.length = 0;
|
||||||
|
queuedBytes = 0;
|
||||||
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
|
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
|
||||||
setStatus("connecting");
|
setStatus("connecting");
|
||||||
setMessage(copy);
|
setMessage(copy);
|
||||||
@@ -318,7 +368,12 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
schedule: (callback, timeoutMs) => window.setTimeout(callback, timeoutMs),
|
schedule: (callback, timeoutMs) => window.setTimeout(callback, timeoutMs),
|
||||||
cancel: (handle) => window.clearTimeout(handle),
|
cancel: (handle) => window.clearTimeout(handle),
|
||||||
onTimeout: (stage) => {
|
onTimeout: (stage) => {
|
||||||
retryTransport(cameraStartupWatchdogRecoveryMessage(stage));
|
retryTransport(
|
||||||
|
cameraStartupWatchdogRecoveryMessage(stage),
|
||||||
|
stage === "first-media"
|
||||||
|
? "camera-first-media-timeout"
|
||||||
|
: "camera-first-playable-timeout",
|
||||||
|
);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -331,6 +386,16 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
recoveryPendingAuthorityRef.current = null;
|
recoveryPendingAuthorityRef.current = null;
|
||||||
setStatus("playing");
|
setStatus("playing");
|
||||||
setMessage("");
|
setMessage("");
|
||||||
|
if (!playingReported) {
|
||||||
|
playingReported = true;
|
||||||
|
diagnosticLifecycle.post({
|
||||||
|
eventCode: "live_camera_transport_playing",
|
||||||
|
streamId: delivery.id,
|
||||||
|
cameraQueueBytes: queuedBytes,
|
||||||
|
cameraQueueSegments: queue.length,
|
||||||
|
transportEpoch,
|
||||||
|
});
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
video.addEventListener("playing", onPlaying);
|
video.addEventListener("playing", onPlaying);
|
||||||
@@ -349,9 +414,24 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
try {
|
try {
|
||||||
sourceBuffer.appendBuffer(chunk);
|
sourceBuffer.appendBuffer(chunk);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
retryTransport(error instanceof DOMException && error.name === "QuotaExceededError"
|
const quotaExceeded = error instanceof DOMException
|
||||||
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
|
&& error.name === "QuotaExceededError";
|
||||||
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.");
|
retryTransport(
|
||||||
|
quotaExceeded
|
||||||
|
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
|
||||||
|
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.",
|
||||||
|
quotaExceeded ? "camera-append-quota" : "camera-append-error",
|
||||||
|
undefined,
|
||||||
|
{
|
||||||
|
cameraAppendErrorName: error instanceof DOMException
|
||||||
|
? error.name
|
||||||
|
: error instanceof Error
|
||||||
|
? error.name
|
||||||
|
: "UnknownError",
|
||||||
|
cameraMediaSourceState: mediaSource.readyState,
|
||||||
|
cameraVideoErrorCode: video.error?.code ?? null,
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -361,7 +441,10 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
// Never drop arbitrary fMP4 fragments: the following samples may depend
|
// Never drop arbitrary fMP4 fragments: the following samples may depend
|
||||||
// on them. A bounded reset is safer and keeps live latency deterministic.
|
// on them. A bounded reset is safer and keeps live latency deterministic.
|
||||||
if (!cameraPendingQueueCanAccept(queuedBytes, queue.length, chunk.byteLength)) {
|
if (!cameraPendingQueueCanAccept(queuedBytes, queue.length, chunk.byteLength)) {
|
||||||
retryTransport("Видеодекодер отстал от эфира; восстанавливаем live-буфер.");
|
retryTransport(
|
||||||
|
"Видеодекодер отстал от эфира; восстанавливаем live-буфер.",
|
||||||
|
"camera-queue-capacity",
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
queue.push(chunk);
|
queue.push(chunk);
|
||||||
@@ -427,7 +510,10 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
appendNext();
|
appendNext();
|
||||||
};
|
};
|
||||||
onBufferError = () => {
|
onBufferError = () => {
|
||||||
retryTransport("MSE сбросил видеосегмент; восстанавливаем decoder.");
|
retryTransport(
|
||||||
|
"MSE сбросил видеосегмент; восстанавливаем decoder.",
|
||||||
|
"camera-source-buffer-error",
|
||||||
|
);
|
||||||
};
|
};
|
||||||
sourceBuffer.addEventListener("updateend", onBufferUpdateEnd);
|
sourceBuffer.addEventListener("updateend", onBufferUpdateEnd);
|
||||||
sourceBuffer.addEventListener("error", onBufferError);
|
sourceBuffer.addEventListener("error", onBufferError);
|
||||||
@@ -453,16 +539,26 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
enqueue(event.data);
|
enqueue(event.data);
|
||||||
} else if (event.data instanceof Blob) {
|
} else if (event.data instanceof Blob) {
|
||||||
void event.data.arrayBuffer().then(enqueue).catch(() => {
|
void event.data.arrayBuffer().then(enqueue).catch(() => {
|
||||||
retryTransport("Получен повреждённый видеосегмент; восстанавливаем канал.");
|
retryTransport(
|
||||||
|
"Получен повреждённый видеосегмент; восстанавливаем канал.",
|
||||||
|
"camera-fragment-decode",
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
socket.addEventListener("error", () => {
|
socket.addEventListener("error", () => {
|
||||||
retryTransport("Связь с локальным video adapter потеряна; переподключаемся.");
|
retryTransport(
|
||||||
|
"Связь с локальным video adapter потеряна; переподключаемся.",
|
||||||
|
"camera-websocket-error",
|
||||||
|
);
|
||||||
});
|
});
|
||||||
socket.addEventListener("close", (event) => {
|
socket.addEventListener("close", (event) => {
|
||||||
if (!transportIsCurrent() || failed) return;
|
if (!transportIsCurrent() || failed) return;
|
||||||
retryTransport(cameraTransportCloseRecoveryMessage(event.code));
|
retryTransport(
|
||||||
|
cameraTransportCloseRecoveryMessage(event.code),
|
||||||
|
"camera-websocket-close",
|
||||||
|
event.code,
|
||||||
|
);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -504,6 +600,10 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
if (activeTransportDisposeRef.current === disposeTransport) {
|
if (activeTransportDisposeRef.current === disposeTransport) {
|
||||||
activeTransportDisposeRef.current = null;
|
activeTransportDisposeRef.current = null;
|
||||||
}
|
}
|
||||||
|
if (activeDiagnosticRef.current === diagnosticLifecycle) {
|
||||||
|
activeDiagnosticRef.current = null;
|
||||||
|
}
|
||||||
|
diagnosticLifecycle.dispose();
|
||||||
disposeTransport();
|
disposeTransport();
|
||||||
};
|
};
|
||||||
}, [attempt, recoveryAuthorityIdentity, transportIdentity, uiBuildStale]);
|
}, [attempt, recoveryAuthorityIdentity, transportIdentity, uiBuildStale]);
|
||||||
@@ -530,6 +630,12 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
});
|
});
|
||||||
recovery = decision.state;
|
recovery = decision.state;
|
||||||
if (!decision.reopen) return;
|
if (!decision.reopen) return;
|
||||||
|
activeDiagnosticRef.current?.post({
|
||||||
|
eventCode: "live_camera_transport_restart_requested",
|
||||||
|
failureStage: cameraPlaybackRecoveryFailureStage(event),
|
||||||
|
streamId: delivery.id,
|
||||||
|
transportEpoch: transportEpochRef.current,
|
||||||
|
});
|
||||||
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
|
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
|
||||||
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
|
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
|
||||||
setStatus("connecting");
|
setStatus("connecting");
|
||||||
|
|||||||
@@ -1063,9 +1063,10 @@ export function RerunViewport({
|
|||||||
let liveRecoveryRetryTimer: number | undefined;
|
let liveRecoveryRetryTimer: number | undefined;
|
||||||
let recordingOpened = false;
|
let recordingOpened = false;
|
||||||
let recordingOpenTimedOut = false;
|
let recordingOpenTimedOut = false;
|
||||||
let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
|
// A new browser receiver has not observed any backend publication yet.
|
||||||
liveActivitySequenceRef.current,
|
// Starting from the already-consumed runtime sequence makes the first
|
||||||
);
|
// four-second check look stalled and destroys a healthy partial replay.
|
||||||
|
let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(null);
|
||||||
let viewerStartResolved = false;
|
let viewerStartResolved = false;
|
||||||
let latestLiveRangeMaxNs: number | null = null;
|
let latestLiveRangeMaxNs: number | null = null;
|
||||||
let liveWatchdog = initialLiveReceiverWatchdogState();
|
let liveWatchdog = initialLiveReceiverWatchdogState();
|
||||||
@@ -1718,10 +1719,7 @@ export function RerunViewport({
|
|||||||
if (!isRecordedSource) {
|
if (!isRecordedSource) {
|
||||||
// Measure native store admission from the resolved viewer start,
|
// Measure native store admission from the resolved viewer start,
|
||||||
// not from dynamic module import or React effect setup time.
|
// not from dynamic module import or React effect setup time.
|
||||||
liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
|
liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(null, Date.now());
|
||||||
liveActivitySequenceRef.current,
|
|
||||||
Date.now(),
|
|
||||||
);
|
|
||||||
discoverActiveLiveRecording();
|
discoverActiveLiveRecording();
|
||||||
if (!recordingOpened) {
|
if (!recordingOpened) {
|
||||||
diagnosticLifecycle.armAdmissionInterval(
|
diagnosticLifecycle.armAdmissionInterval(
|
||||||
|
|||||||
@@ -4,13 +4,28 @@ export type LiveViewerDiagnosticEventCode =
|
|||||||
| "live_receiver_recovered"
|
| "live_receiver_recovered"
|
||||||
| "live_receiver_recovery_exhausted"
|
| "live_receiver_recovery_exhausted"
|
||||||
| "live_receiver_active_store_admitted"
|
| "live_receiver_active_store_admitted"
|
||||||
| "live_receiver_error";
|
| "live_receiver_error"
|
||||||
|
| "live_camera_transport_restart_requested"
|
||||||
|
| "live_camera_transport_playing";
|
||||||
|
|
||||||
export type LiveViewerFailureStage =
|
export type LiveViewerFailureStage =
|
||||||
| "recording-open-timeout"
|
| "recording-open-timeout"
|
||||||
| "viewer-start"
|
| "viewer-start"
|
||||||
| "module-load"
|
| "module-load"
|
||||||
| "receiver-stalled";
|
| "receiver-stalled"
|
||||||
|
| "camera-first-media-timeout"
|
||||||
|
| "camera-first-playable-timeout"
|
||||||
|
| "camera-queue-capacity"
|
||||||
|
| "camera-append-quota"
|
||||||
|
| "camera-append-error"
|
||||||
|
| "camera-source-buffer-error"
|
||||||
|
| "camera-fragment-decode"
|
||||||
|
| "camera-websocket-error"
|
||||||
|
| "camera-websocket-close"
|
||||||
|
| "camera-heartbeat-reopen"
|
||||||
|
| "camera-visibility-reopen"
|
||||||
|
| "camera-network-online-reopen"
|
||||||
|
| "camera-page-restore-reopen";
|
||||||
|
|
||||||
export interface LiveViewerDiagnostic {
|
export interface LiveViewerDiagnostic {
|
||||||
eventCode: LiveViewerDiagnosticEventCode;
|
eventCode: LiveViewerDiagnosticEventCode;
|
||||||
@@ -20,6 +35,14 @@ export interface LiveViewerDiagnostic {
|
|||||||
viewerRangeMaxNs?: number | null;
|
viewerRangeMaxNs?: number | null;
|
||||||
stalledForMs?: number | null;
|
stalledForMs?: number | null;
|
||||||
recoveryAttempt?: number | null;
|
recoveryAttempt?: number | null;
|
||||||
|
cameraQueueBytes?: number | null;
|
||||||
|
cameraQueueSegments?: number | null;
|
||||||
|
cameraRetryCount?: number | null;
|
||||||
|
websocketCloseCode?: number | null;
|
||||||
|
transportEpoch?: number | null;
|
||||||
|
cameraAppendErrorName?: string | null;
|
||||||
|
cameraMediaSourceState?: "closed" | "open" | "ended" | null;
|
||||||
|
cameraVideoErrorCode?: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LiveViewerLineage {
|
export interface LiveViewerLineage {
|
||||||
@@ -69,7 +92,6 @@ const DEVELOPMENT_UI_BUILD_ID = "development";
|
|||||||
const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||||
const HASHED_UI_BUILD_ID = /^\/assets\/[A-Za-z0-9._/-]+-[A-Za-z0-9_-]{8,}\.js$/;
|
const HASHED_UI_BUILD_ID = /^\/assets\/[A-Za-z0-9._/-]+-[A-Za-z0-9_-]{8,}\.js$/;
|
||||||
const UI_BUILD_CHECK_INTERVAL_MILLISECONDS = 15_000;
|
const UI_BUILD_CHECK_INTERVAL_MILLISECONDS = 15_000;
|
||||||
const UI_BUILD_RELOAD_DELAY_MILLISECONDS = 50;
|
|
||||||
|
|
||||||
let documentInstanceId: string | null = null;
|
let documentInstanceId: string | null = null;
|
||||||
let sharedUiBuildCoordinator: UiBuildStaleCoordinator | null = null;
|
let sharedUiBuildCoordinator: UiBuildStaleCoordinator | null = null;
|
||||||
@@ -172,32 +194,45 @@ export function liveViewerDiagnosticBody(
|
|||||||
...(safeInteger(event.recoveryAttempt) === undefined
|
...(safeInteger(event.recoveryAttempt) === undefined
|
||||||
? {}
|
? {}
|
||||||
: { recovery_attempt: safeInteger(event.recoveryAttempt) }),
|
: { recovery_attempt: safeInteger(event.recoveryAttempt) }),
|
||||||
|
...(safeInteger(event.cameraQueueBytes) === undefined
|
||||||
|
? {}
|
||||||
|
: { camera_queue_bytes: safeInteger(event.cameraQueueBytes) }),
|
||||||
|
...(safeInteger(event.cameraQueueSegments) === undefined
|
||||||
|
? {}
|
||||||
|
: { camera_queue_segments: safeInteger(event.cameraQueueSegments) }),
|
||||||
|
...(safeInteger(event.cameraRetryCount) === undefined
|
||||||
|
? {}
|
||||||
|
: { camera_retry_count: safeInteger(event.cameraRetryCount) }),
|
||||||
|
...(safeInteger(event.websocketCloseCode) === undefined
|
||||||
|
? {}
|
||||||
|
: { websocket_close_code: safeInteger(event.websocketCloseCode) }),
|
||||||
|
...(safeInteger(event.transportEpoch) === undefined
|
||||||
|
? {}
|
||||||
|
: { transport_epoch: safeInteger(event.transportEpoch) }),
|
||||||
|
...(event.cameraAppendErrorName
|
||||||
|
? { camera_append_error_name: event.cameraAppendErrorName }
|
||||||
|
: {}),
|
||||||
|
...(event.cameraMediaSourceState
|
||||||
|
? { camera_media_source_state: event.cameraMediaSourceState }
|
||||||
|
: {}),
|
||||||
|
...(safeInteger(event.cameraVideoErrorCode) === undefined
|
||||||
|
? {}
|
||||||
|
: { camera_video_error_code: safeInteger(event.cameraVideoErrorCode) }),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createUiBuildStaleCoordinator({
|
export function createUiBuildStaleCoordinator(): UiBuildStaleCoordinator {
|
||||||
scheduleReload,
|
|
||||||
reload,
|
|
||||||
}: {
|
|
||||||
scheduleReload: (callback: () => void, delayMilliseconds: number) => void;
|
|
||||||
reload: () => void;
|
|
||||||
}): UiBuildStaleCoordinator {
|
|
||||||
const listeners = new Set<(event: StaleUiBuild) => void>();
|
|
||||||
let staleEvent: StaleUiBuild | null = null;
|
let staleEvent: StaleUiBuild | null = null;
|
||||||
let reloadScheduled = false;
|
|
||||||
return {
|
return {
|
||||||
subscribe(listener) {
|
subscribe(_listener) {
|
||||||
listeners.add(listener);
|
// Build drift is diagnostic-only. The loaded document and its local
|
||||||
if (staleEvent) listener(staleEvent);
|
// Rerun/camera transports remain authoritative until normal unmount or
|
||||||
return () => listeners.delete(listener);
|
// their existing bounded recovery replaces them.
|
||||||
|
return () => undefined;
|
||||||
},
|
},
|
||||||
report(event) {
|
report(event) {
|
||||||
if (event.loadedUiBuildId === event.expectedUiBuildId || staleEvent) return;
|
if (event.loadedUiBuildId === event.expectedUiBuildId || staleEvent) return;
|
||||||
staleEvent = event;
|
staleEvent = event;
|
||||||
for (const listener of [...listeners]) listener(event);
|
|
||||||
if (reloadScheduled) return;
|
|
||||||
reloadScheduled = true;
|
|
||||||
scheduleReload(reload, UI_BUILD_RELOAD_DELAY_MILLISECONDS);
|
|
||||||
},
|
},
|
||||||
stale() {
|
stale() {
|
||||||
return staleEvent !== null;
|
return staleEvent !== null;
|
||||||
@@ -206,12 +241,7 @@ export function createUiBuildStaleCoordinator({
|
|||||||
}
|
}
|
||||||
|
|
||||||
function browserUiBuildCoordinator(): UiBuildStaleCoordinator {
|
function browserUiBuildCoordinator(): UiBuildStaleCoordinator {
|
||||||
sharedUiBuildCoordinator ??= createUiBuildStaleCoordinator({
|
sharedUiBuildCoordinator ??= createUiBuildStaleCoordinator();
|
||||||
scheduleReload: (callback, delayMilliseconds) => {
|
|
||||||
window.setTimeout(callback, delayMilliseconds);
|
|
||||||
},
|
|
||||||
reload: () => window.location.reload(),
|
|
||||||
});
|
|
||||||
return sharedUiBuildCoordinator;
|
return sharedUiBuildCoordinator;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,10 +284,17 @@ export function verifyLiveViewerClientBuild(
|
|||||||
lineage: Pick<LiveViewerLineage, "uiBuildId">,
|
lineage: Pick<LiveViewerLineage, "uiBuildId">,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
): void {
|
): void {
|
||||||
if (signal?.aborted || lineage.uiBuildId === DEVELOPMENT_UI_BUILD_ID) return;
|
if (
|
||||||
|
signal?.aborted ||
|
||||||
|
lineage.uiBuildId === DEVELOPMENT_UI_BUILD_ID ||
|
||||||
|
browserUiBuildCoordinator().stale()
|
||||||
|
) return;
|
||||||
void fetch("/api/v1/viewer/client-contract", {
|
void fetch("/api/v1/viewer/client-contract", {
|
||||||
method: "GET",
|
method: "GET",
|
||||||
headers: { Accept: "application/json" },
|
headers: {
|
||||||
|
Accept: "application/json",
|
||||||
|
"X-MissionCore-UI-Build": lineage.uiBuildId,
|
||||||
|
},
|
||||||
cache: "no-store",
|
cache: "no-store",
|
||||||
signal,
|
signal,
|
||||||
}).then(async (response) => {
|
}).then(async (response) => {
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import type { MissionRuntimeState } from "../runtime/contracts";
|
||||||
|
|
||||||
|
export function viewerSettingsTargetIdentity(
|
||||||
|
state: MissionRuntimeState | null,
|
||||||
|
): string {
|
||||||
|
const deviceSessionId = state?.deviceSession?.sessionId?.trim();
|
||||||
|
const sessionDeviceId = state?.deviceSession?.deviceId?.trim();
|
||||||
|
if (deviceSessionId || sessionDeviceId) {
|
||||||
|
return ["device-session", deviceSessionId, sessionDeviceId]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(":");
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
state?.activeDevice?.pluginId,
|
||||||
|
state?.activeDevice?.modelId,
|
||||||
|
state?.activeDevice?.instanceId,
|
||||||
|
].filter(Boolean).join(":") || "local-runtime";
|
||||||
|
}
|
||||||
@@ -20,6 +20,7 @@ let operatorIntentGeneration;
|
|||||||
let configuration;
|
let configuration;
|
||||||
let compatibility;
|
let compatibility;
|
||||||
let networkProvisionFailureMessage;
|
let networkProvisionFailureMessage;
|
||||||
|
let awaitNetworkProvisionSettlementAfterLostResponse;
|
||||||
let discoveryScanFailureMessage;
|
let discoveryScanFailureMessage;
|
||||||
let connectionVerificationFailureMessage;
|
let connectionVerificationFailureMessage;
|
||||||
let operationById;
|
let operationById;
|
||||||
@@ -99,6 +100,7 @@ before(async () => {
|
|||||||
));
|
));
|
||||||
({
|
({
|
||||||
networkProvisionFailureMessage,
|
networkProvisionFailureMessage,
|
||||||
|
awaitNetworkProvisionSettlementAfterLostResponse,
|
||||||
discoveryScanFailureMessage,
|
discoveryScanFailureMessage,
|
||||||
connectionVerificationFailureMessage,
|
connectionVerificationFailureMessage,
|
||||||
operationById,
|
operationById,
|
||||||
@@ -1043,6 +1045,56 @@ test("K1 API uses bounded no-retry timeout classes per operation family", async
|
|||||||
assert.doesNotMatch(apiSource, /automaticRetry\s*=\s*true/);
|
assert.doesNotMatch(apiSource, /automaticRetry\s*=\s*true/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("lost Quick response follows the admitted Apply by journal reads only", async () => {
|
||||||
|
const idempotencyKey = "network-provision:quick-lost-response";
|
||||||
|
let nowMs = 1_000;
|
||||||
|
let reads = 0;
|
||||||
|
const accepted = [];
|
||||||
|
const operation = {
|
||||||
|
operation_id: "op-quick-lost-response",
|
||||||
|
action: "network.provision",
|
||||||
|
status: "running",
|
||||||
|
idempotency_key: idempotencyKey,
|
||||||
|
deadline_at: new Date(5_000).toISOString(),
|
||||||
|
};
|
||||||
|
const initialState = {
|
||||||
|
snapshot_runtime_id: "runtime-quick-lost-response",
|
||||||
|
operations: [operation],
|
||||||
|
};
|
||||||
|
const terminalState = {
|
||||||
|
...initialState,
|
||||||
|
operations: [{
|
||||||
|
...operation,
|
||||||
|
status: "succeeded",
|
||||||
|
result: { phase: "network_applied" },
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
const settled = await awaitNetworkProvisionSettlementAfterLostResponse(
|
||||||
|
initialState,
|
||||||
|
idempotencyKey,
|
||||||
|
async () => {
|
||||||
|
reads += 1;
|
||||||
|
if (reads === 1) {
|
||||||
|
throw new ApiError("transient localhost handoff", 0, true);
|
||||||
|
}
|
||||||
|
return terminalState;
|
||||||
|
},
|
||||||
|
(state) => accepted.push(state),
|
||||||
|
() => {},
|
||||||
|
{
|
||||||
|
now: () => nowMs,
|
||||||
|
wait: async (delayMs) => {
|
||||||
|
nowMs += delayMs;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(reads, 2);
|
||||||
|
assert.deepEqual(accepted, [terminalState]);
|
||||||
|
assert.equal(settled.operations[0].status, "succeeded");
|
||||||
|
});
|
||||||
|
|
||||||
test("one explicit K1 operator action supplies the backend physical acceptance", () => {
|
test("one explicit K1 operator action supplies the backend physical acceptance", () => {
|
||||||
assert.deepEqual(physicalCommandConfirmation.operatorActionPhysicalAcceptance(), {
|
assert.deepEqual(physicalCommandConfirmation.operatorActionPhysicalAcceptance(), {
|
||||||
operator_present: true,
|
operator_present: true,
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ let transportRefEquivalenceKey;
|
|||||||
let shouldRevealProvisioningNetworkStep;
|
let shouldRevealProvisioningNetworkStep;
|
||||||
let retiredPhysicalReopenAuthority;
|
let retiredPhysicalReopenAuthority;
|
||||||
let SnapshotRuntimeActionArbiter;
|
let SnapshotRuntimeActionArbiter;
|
||||||
|
let runtimeActionResponseAlreadyAccepted;
|
||||||
let connectionActionAuthoritySnapshot;
|
let connectionActionAuthoritySnapshot;
|
||||||
let exactAppliedNetworkIntentCompleted;
|
let exactAppliedNetworkIntentCompleted;
|
||||||
let selectMonotonicXgridsState;
|
let selectMonotonicXgridsState;
|
||||||
@@ -106,6 +107,7 @@ before(async () => {
|
|||||||
SnapshotRuntimeActionArbiter,
|
SnapshotRuntimeActionArbiter,
|
||||||
connectionActionAuthoritySnapshot,
|
connectionActionAuthoritySnapshot,
|
||||||
exactAppliedNetworkIntentCompleted,
|
exactAppliedNetworkIntentCompleted,
|
||||||
|
runtimeActionResponseAlreadyAccepted,
|
||||||
} = await server.ssrLoadModule(
|
} = await server.ssrLoadModule(
|
||||||
"@xgrids-k1/frontend/useXgridsK1Runtime.ts",
|
"@xgrids-k1/frontend/useXgridsK1Runtime.ts",
|
||||||
));
|
));
|
||||||
@@ -2026,6 +2028,27 @@ test("K1 runtime replacement retires A and isolates a new B action from late set
|
|||||||
assert.equal(pendingAction, null);
|
assert.equal(pendingAction, null);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("successful action response may be superseded only by the same runtime revision", () => {
|
||||||
|
const response = {
|
||||||
|
snapshot_runtime_id: "runtime-a",
|
||||||
|
snapshot_revision: 41,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert.equal(runtimeActionResponseAlreadyAccepted(response, {
|
||||||
|
snapshot_runtime_id: "runtime-a",
|
||||||
|
snapshot_revision: 42,
|
||||||
|
}), true);
|
||||||
|
assert.equal(runtimeActionResponseAlreadyAccepted(response, {
|
||||||
|
snapshot_runtime_id: "runtime-a",
|
||||||
|
snapshot_revision: 40,
|
||||||
|
}), false);
|
||||||
|
assert.equal(runtimeActionResponseAlreadyAccepted(response, {
|
||||||
|
snapshot_runtime_id: "runtime-b",
|
||||||
|
snapshot_revision: 99,
|
||||||
|
}), false);
|
||||||
|
assert.equal(runtimeActionResponseAlreadyAccepted(response, null), false);
|
||||||
|
});
|
||||||
|
|
||||||
test("explicit scenario reset B supersedes pending callback A in one runtime", () => {
|
test("explicit scenario reset B supersedes pending callback A in one runtime", () => {
|
||||||
const arbiter = new SnapshotRuntimeActionArbiter();
|
const arbiter = new SnapshotRuntimeActionArbiter();
|
||||||
const actionA = arbiter.begin(12);
|
const actionA = arbiter.begin(12);
|
||||||
|
|||||||
@@ -273,6 +273,21 @@ test("opening receiver preserves partial store replay while backend publication
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("new receiver grants its first confirmed backend publication a full open window", () => {
|
||||||
|
const openState = initialLiveReceiverOpenWatchdogState(null, 0);
|
||||||
|
const recoveryState = initialLiveReceiverRecoveryState();
|
||||||
|
const firstObservedPublication = advanceLiveReceiverOpenWatchdog(
|
||||||
|
openState,
|
||||||
|
recoveryState,
|
||||||
|
1,
|
||||||
|
4_000,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(firstObservedPublication.signal, "wait-for-store");
|
||||||
|
assert.equal(firstObservedPublication.state.lastBackendActivitySequence, 1);
|
||||||
|
assert.deepEqual(firstObservedPublication.recoveryState, recoveryState);
|
||||||
|
});
|
||||||
|
|
||||||
test("unchanged opening sequence delegates to bounded receiver restart", () => {
|
test("unchanged opening sequence delegates to bounded receiver restart", () => {
|
||||||
const openState = initialLiveReceiverOpenWatchdogState(486);
|
const openState = initialLiveReceiverOpenWatchdogState(486);
|
||||||
const recoveryState = initialLiveReceiverRecoveryState();
|
const recoveryState = initialLiveReceiverRecoveryState();
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ let createUiBuildStaleCoordinator;
|
|||||||
let liveViewerDiagnosticBody;
|
let liveViewerDiagnosticBody;
|
||||||
let server;
|
let server;
|
||||||
let uiBuildIdFromModuleScripts;
|
let uiBuildIdFromModuleScripts;
|
||||||
|
let verifyLiveViewerClientBuild;
|
||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
server = await createServer({
|
server = await createServer({
|
||||||
@@ -22,6 +23,7 @@ before(async () => {
|
|||||||
createUiBuildStaleCoordinator,
|
createUiBuildStaleCoordinator,
|
||||||
liveViewerDiagnosticBody,
|
liveViewerDiagnosticBody,
|
||||||
uiBuildIdFromModuleScripts,
|
uiBuildIdFromModuleScripts,
|
||||||
|
verifyLiveViewerClientBuild,
|
||||||
} = await server.ssrLoadModule("/src/core/observation/liveViewerDiagnostics.ts"));
|
} = await server.ssrLoadModule("/src/core/observation/liveViewerDiagnostics.ts"));
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -129,9 +131,9 @@ test("two mounted viewers keep timer and diagnostic lineage isolated", () => {
|
|||||||
assert.equal(posts[0].eventLineage.lifecycleGeneration, 7);
|
assert.equal(posts[0].eventLineage.lifecycleGeneration, 7);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("stale-build and unmount fence callbacks before one reload", () => {
|
test("stale build drift neither fences transports nor reloads the document", () => {
|
||||||
const clock = createFakeScheduler();
|
const clock = createFakeScheduler();
|
||||||
const order = [];
|
const sideEffects = [];
|
||||||
const posts = [];
|
const posts = [];
|
||||||
const lifecycle = createLiveViewerDiagnosticLifecycle({
|
const lifecycle = createLiveViewerDiagnosticLifecycle({
|
||||||
lineage: lineage("00000000-0000-4000-8000-000000000031"),
|
lineage: lineage("00000000-0000-4000-8000-000000000031"),
|
||||||
@@ -142,15 +144,9 @@ test("stale-build and unmount fence callbacks before one reload", () => {
|
|||||||
lifecycle.armAdmissionTimeout(() => {
|
lifecycle.armAdmissionTimeout(() => {
|
||||||
lifecycle.post({ eventCode: "live_receiver_error" });
|
lifecycle.post({ eventCode: "live_receiver_error" });
|
||||||
}, 12_000);
|
}, 12_000);
|
||||||
const coordinator = createUiBuildStaleCoordinator({
|
const coordinator = createUiBuildStaleCoordinator();
|
||||||
scheduleReload: (callback, delay) => {
|
|
||||||
order.push(`scheduled:${delay}`);
|
|
||||||
clock.scheduler.setTimeout(callback, delay);
|
|
||||||
},
|
|
||||||
reload: () => order.push("reload"),
|
|
||||||
});
|
|
||||||
coordinator.subscribe(() => {
|
coordinator.subscribe(() => {
|
||||||
order.push("local-transports-closed");
|
sideEffects.push("local-transports-closed");
|
||||||
lifecycle.dispose();
|
lifecycle.dispose();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -165,9 +161,47 @@ test("stale-build and unmount fence callbacks before one reload", () => {
|
|||||||
clock.advance(60_000);
|
clock.advance(60_000);
|
||||||
lifecycle.post({ eventCode: "live_receiver_error" });
|
lifecycle.post({ eventCode: "live_receiver_error" });
|
||||||
|
|
||||||
assert.deepEqual(order, ["local-transports-closed", "scheduled:50", "reload"]);
|
assert.deepEqual(sideEffects, []);
|
||||||
assert.deepEqual(posts, []);
|
assert.deepEqual(posts, [
|
||||||
assert.equal(lifecycle.active(), false);
|
{ eventCode: "live_receiver_error" },
|
||||||
|
{ eventCode: "live_receiver_error" },
|
||||||
|
]);
|
||||||
|
assert.equal(lifecycle.active(), true);
|
||||||
|
assert.equal(coordinator.stale(), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("build drift is reported once with the exact loaded build", async () => {
|
||||||
|
const originalFetch = globalThis.fetch;
|
||||||
|
const calls = [];
|
||||||
|
globalThis.fetch = async (url, options) => {
|
||||||
|
calls.push({ url, options });
|
||||||
|
return new Response(JSON.stringify({
|
||||||
|
schema_version: "missioncore.live-viewer-client-contract/v1",
|
||||||
|
status: "ready",
|
||||||
|
ui_build_id: "/assets/index-ijklmnop.js",
|
||||||
|
diagnostic_schema_version: "missioncore.live-viewer-diagnostic/v2",
|
||||||
|
}), {
|
||||||
|
status: 200,
|
||||||
|
headers: { "X-MissionCore-UI-Build": "/assets/index-ijklmnop.js" },
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const eventLineage = lineage("00000000-0000-4000-8000-000000000032");
|
||||||
|
verifyLiveViewerClientBuild(eventLineage);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
verifyLiveViewerClientBuild(eventLineage);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
} finally {
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.equal(calls.length, 1);
|
||||||
|
assert.equal(calls[0].url, "/api/v1/viewer/client-contract");
|
||||||
|
assert.equal(
|
||||||
|
calls[0].options.headers["X-MissionCore-UI-Build"],
|
||||||
|
"/assets/index-abcdefgh.js",
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("last unsubscribe fences an already queued build verification callback", () => {
|
test("last unsubscribe fences an already queued build verification callback", () => {
|
||||||
@@ -213,4 +247,41 @@ test("diagnostic body and build id retain exact document/viewer/build lineage",
|
|||||||
),
|
),
|
||||||
"/assets/index-dT7dN-y4.js",
|
"/assets/index-dT7dN-y4.js",
|
||||||
);
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
liveViewerDiagnosticBody(
|
||||||
|
{
|
||||||
|
eventCode: "live_camera_transport_restart_requested",
|
||||||
|
failureStage: "camera-queue-capacity",
|
||||||
|
streamId: "camera-preview-2",
|
||||||
|
cameraQueueBytes: 12_000_000,
|
||||||
|
cameraQueueSegments: 96,
|
||||||
|
cameraRetryCount: 3,
|
||||||
|
websocketCloseCode: 4_008,
|
||||||
|
transportEpoch: 7,
|
||||||
|
cameraAppendErrorName: "InvalidStateError",
|
||||||
|
cameraMediaSourceState: "open",
|
||||||
|
cameraVideoErrorCode: 3,
|
||||||
|
},
|
||||||
|
eventLineage,
|
||||||
|
),
|
||||||
|
{
|
||||||
|
schema_version: "missioncore.live-viewer-diagnostic/v2",
|
||||||
|
event_code: "live_camera_transport_restart_requested",
|
||||||
|
ui_build_id: "/assets/index-abcdefgh.js",
|
||||||
|
document_instance_id: "00000000-0000-4000-8000-000000000001",
|
||||||
|
viewer_instance_id: "00000000-0000-4000-8000-000000000041",
|
||||||
|
lifecycle_generation: 9,
|
||||||
|
failure_stage: "camera-queue-capacity",
|
||||||
|
stream_id: "camera-preview-2",
|
||||||
|
camera_queue_bytes: 12_000_000,
|
||||||
|
camera_queue_segments: 96,
|
||||||
|
camera_retry_count: 3,
|
||||||
|
websocket_close_code: 4_008,
|
||||||
|
transport_epoch: 7,
|
||||||
|
camera_append_error_name: "InvalidStateError",
|
||||||
|
camera_media_source_state: "open",
|
||||||
|
camera_video_error_code: 3,
|
||||||
|
},
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ let cameraTransportRecoveryIsCurrent;
|
|||||||
let cameraTransportCanOpen;
|
let cameraTransportCanOpen;
|
||||||
let cameraPendingQueueCanAccept;
|
let cameraPendingQueueCanAccept;
|
||||||
let cameraTransportCloseRecoveryMessage;
|
let cameraTransportCloseRecoveryMessage;
|
||||||
|
let cameraPlaybackRecoveryFailureStage;
|
||||||
let createCameraStartupWatchdog;
|
let createCameraStartupWatchdog;
|
||||||
let cameraStartupWatchdogRecoveryMessage;
|
let cameraStartupWatchdogRecoveryMessage;
|
||||||
let CAMERA_FIRST_MEDIA_TIMEOUT_MS;
|
let CAMERA_FIRST_MEDIA_TIMEOUT_MS;
|
||||||
@@ -69,6 +70,7 @@ before(async () => {
|
|||||||
cameraTransportCanOpen,
|
cameraTransportCanOpen,
|
||||||
cameraPendingQueueCanAccept,
|
cameraPendingQueueCanAccept,
|
||||||
cameraTransportCloseRecoveryMessage,
|
cameraTransportCloseRecoveryMessage,
|
||||||
|
cameraPlaybackRecoveryFailureStage,
|
||||||
createCameraStartupWatchdog,
|
createCameraStartupWatchdog,
|
||||||
cameraStartupWatchdogRecoveryMessage,
|
cameraStartupWatchdogRecoveryMessage,
|
||||||
CAMERA_FIRST_MEDIA_TIMEOUT_MS,
|
CAMERA_FIRST_MEDIA_TIMEOUT_MS,
|
||||||
@@ -1297,6 +1299,25 @@ test("server slow-reader close is an automatic browser-only camera recovery", ()
|
|||||||
assert.match(cameraTransportCloseRecoveryMessage(1_011), /восстанавливаем/);
|
assert.match(cameraTransportCloseRecoveryMessage(1_011), /восстанавливаем/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("camera transport recovery events keep exact diagnostic causes", () => {
|
||||||
|
assert.equal(
|
||||||
|
cameraPlaybackRecoveryFailureStage({ type: "heartbeat" }),
|
||||||
|
"camera-heartbeat-reopen",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
cameraPlaybackRecoveryFailureStage({ type: "document-visible" }),
|
||||||
|
"camera-visibility-reopen",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
cameraPlaybackRecoveryFailureStage({ type: "network-online" }),
|
||||||
|
"camera-network-online-reopen",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
cameraPlaybackRecoveryFailureStage({ type: "page-restore", persisted: true }),
|
||||||
|
"camera-page-restore-reopen",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test("a laptop sleep gap reopens only the exact authoritative live camera transport", () => {
|
test("a laptop sleep gap reopens only the exact authoritative live camera transport", () => {
|
||||||
const source = xgridsK1ObservationSources(
|
const source = xgridsK1ObservationSources(
|
||||||
cameraStreamingState("sensor.camera.right"),
|
cameraStreamingState("sensor.camera.right"),
|
||||||
|
|||||||
@@ -237,6 +237,14 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn
|
|||||||
source,
|
source,
|
||||||
/if \(readyToRender && !readyPublished\)[\s\S]*eventCode: "live_receiver_active_store_admitted"[\s\S]*diagnosticLifecycle\.markAdmitted\(\)/,
|
/if \(readyToRender && !readyPublished\)[\s\S]*eventCode: "live_receiver_active_store_admitted"[\s\S]*diagnosticLifecycle\.markAdmitted\(\)/,
|
||||||
);
|
);
|
||||||
|
assert.match(
|
||||||
|
source,
|
||||||
|
/liveOpenWatchdog = initialLiveReceiverOpenWatchdogState\(null, Date\.now\(\)\)/,
|
||||||
|
);
|
||||||
|
assert.doesNotMatch(
|
||||||
|
source,
|
||||||
|
/initialLiveReceiverOpenWatchdogState\(\s*liveActivitySequenceRef\.current/,
|
||||||
|
);
|
||||||
assert.doesNotMatch(
|
assert.doesNotMatch(
|
||||||
source,
|
source,
|
||||||
/\], \[followLive, liveRecoveryAuthorityIdentity, liveStreamId, sourceUrl\]\);/,
|
/\], \[followLive, liveRecoveryAuthorityIdentity, liveStreamId, sourceUrl\]\);/,
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { after, before, test } from "node:test";
|
||||||
|
|
||||||
|
import { createServer } from "vite";
|
||||||
|
|
||||||
|
let server;
|
||||||
|
let viewerSettingsTargetIdentity;
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
server = await createServer({
|
||||||
|
appType: "custom",
|
||||||
|
logLevel: "silent",
|
||||||
|
server: { middlewareMode: true },
|
||||||
|
});
|
||||||
|
({ viewerSettingsTargetIdentity } = await server.ssrLoadModule(
|
||||||
|
"/src/core/observation/viewerSettingsTarget.ts",
|
||||||
|
));
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await server?.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
function runtimeState({ acquisitionId, sessionId = "session-a", deviceId = "device-a" }) {
|
||||||
|
return {
|
||||||
|
phase: "connected",
|
||||||
|
sourceMode: "idle",
|
||||||
|
activeDevice: {
|
||||||
|
pluginId: "xgrids-k1",
|
||||||
|
modelId: "lixel-k1",
|
||||||
|
instanceId: "instance-a",
|
||||||
|
displayName: "K1",
|
||||||
|
},
|
||||||
|
deviceSession: { sessionId, deviceId },
|
||||||
|
acquisition: acquisitionId
|
||||||
|
? {
|
||||||
|
acquisitionId,
|
||||||
|
deviceId,
|
||||||
|
deviceSessionId: sessionId,
|
||||||
|
compatibilityProfileId: "profile-a",
|
||||||
|
controlMode: "live",
|
||||||
|
state: "streaming",
|
||||||
|
stateRevision: 1,
|
||||||
|
operatorInstructions: [],
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test("new acquisition on the same control session keeps the viewer settings target stable", () => {
|
||||||
|
const beforeStart = viewerSettingsTargetIdentity(runtimeState({}));
|
||||||
|
const firstRun = viewerSettingsTargetIdentity(runtimeState({ acquisitionId: "acquisition-a" }));
|
||||||
|
const secondRun = viewerSettingsTargetIdentity(runtimeState({ acquisitionId: "acquisition-b" }));
|
||||||
|
|
||||||
|
assert.equal(firstRun, beforeStart);
|
||||||
|
assert.equal(secondRun, beforeStart);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("temporary loss of control authority does not reapply settings within one device session", () => {
|
||||||
|
const authoritative = runtimeState({ acquisitionId: "acquisition-a" });
|
||||||
|
const authorityUnavailable = {
|
||||||
|
...authoritative,
|
||||||
|
activeDevice: null,
|
||||||
|
phase: "starting",
|
||||||
|
};
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
viewerSettingsTargetIdentity(authorityUnavailable),
|
||||||
|
viewerSettingsTargetIdentity(authoritative),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a new control session or device produces a new viewer settings target", () => {
|
||||||
|
const original = viewerSettingsTargetIdentity(runtimeState({ acquisitionId: "acquisition-a" }));
|
||||||
|
|
||||||
|
assert.notEqual(
|
||||||
|
viewerSettingsTargetIdentity(runtimeState({ sessionId: "session-b" })),
|
||||||
|
original,
|
||||||
|
);
|
||||||
|
assert.notEqual(
|
||||||
|
viewerSettingsTargetIdentity(runtimeState({ deviceId: "device-b" })),
|
||||||
|
original,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an unconfigured runtime uses the local target", () => {
|
||||||
|
assert.equal(viewerSettingsTargetIdentity(null), "local-runtime");
|
||||||
|
});
|
||||||
@@ -135,6 +135,23 @@ export class SnapshotRuntimeActionArbiter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function runtimeActionResponseAlreadyAccepted(
|
||||||
|
responseState: XgridsK1State,
|
||||||
|
acceptedState: XgridsK1State | null,
|
||||||
|
): boolean {
|
||||||
|
const responseRuntimeId = responseState.snapshot_runtime_id?.trim() ?? "";
|
||||||
|
const acceptedRuntimeId = acceptedState?.snapshot_runtime_id?.trim() ?? "";
|
||||||
|
const responseRevision = responseState.snapshot_revision;
|
||||||
|
const acceptedRevision = acceptedState?.snapshot_revision;
|
||||||
|
return Boolean(
|
||||||
|
responseRuntimeId
|
||||||
|
&& acceptedRuntimeId === responseRuntimeId
|
||||||
|
&& Number.isSafeInteger(responseRevision)
|
||||||
|
&& Number.isSafeInteger(acceptedRevision)
|
||||||
|
&& (acceptedRevision as number) >= (responseRevision as number)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export type AcquisitionPreparationDraft = Omit<
|
export type AcquisitionPreparationDraft = Omit<
|
||||||
PrepareAcquisitionRequest,
|
PrepareAcquisitionRequest,
|
||||||
| "operation_id"
|
| "operation_id"
|
||||||
@@ -248,6 +265,87 @@ export function connectionActionAuthoritySnapshot(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const CONTROL_STATE_READ_INTERVAL_MS = 250;
|
const CONTROL_STATE_READ_INTERVAL_MS = 250;
|
||||||
|
const NETWORK_PROVISION_SETTLEMENT_FALLBACK_MS = 30_000;
|
||||||
|
const NETWORK_PROVISION_SETTLEMENT_MAX_MS = 300_000;
|
||||||
|
const NETWORK_PROVISION_SETTLEMENT_GRACE_MS = 1_000;
|
||||||
|
|
||||||
|
function networkProvisionSettlementDeadline(
|
||||||
|
operation: XgridsOperation,
|
||||||
|
startedAtMs: number,
|
||||||
|
): number {
|
||||||
|
const operationDeadlineMs = operation.deadline_at
|
||||||
|
? Date.parse(operation.deadline_at)
|
||||||
|
: Number.NaN;
|
||||||
|
const requestedDeadlineMs = Number.isFinite(operationDeadlineMs)
|
||||||
|
&& operationDeadlineMs > startedAtMs
|
||||||
|
? operationDeadlineMs + NETWORK_PROVISION_SETTLEMENT_GRACE_MS
|
||||||
|
: startedAtMs + NETWORK_PROVISION_SETTLEMENT_FALLBACK_MS;
|
||||||
|
return Math.min(
|
||||||
|
requestedDeadlineMs,
|
||||||
|
startedAtMs + NETWORK_PROVISION_SETTLEMENT_MAX_MS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Follow one already-admitted Apply after a browser response is interrupted by
|
||||||
|
* the host Wi-Fi handoff. This loop only reads the local journal; it never
|
||||||
|
* retries the HTTP mutation, BLE write, CoreWLAN association or control open.
|
||||||
|
*/
|
||||||
|
export async function awaitNetworkProvisionSettlementAfterLostResponse(
|
||||||
|
initialState: XgridsK1State,
|
||||||
|
idempotencyKey: string,
|
||||||
|
readState: () => Promise<XgridsK1State>,
|
||||||
|
acceptState: (state: XgridsK1State) => void,
|
||||||
|
assertOperatorIntentCurrent: () => void,
|
||||||
|
options: {
|
||||||
|
now?: () => number;
|
||||||
|
wait?: (delayMs: number) => Promise<void>;
|
||||||
|
} = {},
|
||||||
|
): Promise<XgridsK1State> {
|
||||||
|
const now = options.now ?? Date.now;
|
||||||
|
const wait = options.wait ?? ((delayMs: number) => new Promise<void>((resolve) => {
|
||||||
|
globalThis.setTimeout(resolve, delayMs);
|
||||||
|
}));
|
||||||
|
let state = initialState;
|
||||||
|
let operation = operationByIdempotencyKey(
|
||||||
|
state,
|
||||||
|
"network.provision",
|
||||||
|
idempotencyKey,
|
||||||
|
);
|
||||||
|
if (!operation || !["accepted", "running"].includes(operation.status)) {
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
const deadlineMs = networkProvisionSettlementDeadline(operation, now());
|
||||||
|
|
||||||
|
while (["accepted", "running"].includes(operation.status)) {
|
||||||
|
assertOperatorIntentCurrent();
|
||||||
|
const remainingMs = deadlineMs - now();
|
||||||
|
if (remainingMs <= 0) return state;
|
||||||
|
await wait(Math.min(CONTROL_STATE_READ_INTERVAL_MS, remainingMs));
|
||||||
|
assertOperatorIntentCurrent();
|
||||||
|
try {
|
||||||
|
const nextState = await readState();
|
||||||
|
assertOperatorIntentCurrent();
|
||||||
|
acceptState(nextState);
|
||||||
|
state = nextState;
|
||||||
|
} catch (readError) {
|
||||||
|
// A host Wi-Fi transition can briefly abort even a localhost fetch.
|
||||||
|
// Preserve the admitted operation and perform only the next bounded
|
||||||
|
// journal read; the original Apply is never reissued.
|
||||||
|
if (!(readError instanceof ApiError) || !readError.transportUnavailable) {
|
||||||
|
throw readError;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
operation = operationByIdempotencyKey(
|
||||||
|
state,
|
||||||
|
"network.provision",
|
||||||
|
idempotencyKey,
|
||||||
|
);
|
||||||
|
if (!operation) return state;
|
||||||
|
}
|
||||||
|
return state;
|
||||||
|
}
|
||||||
const CONTROL_PHASE_WAIT_TIMEOUT_MS = 120_000;
|
const CONTROL_PHASE_WAIT_TIMEOUT_MS = 120_000;
|
||||||
|
|
||||||
function controlPhase(state: XgridsK1State): XgridsApplicationControlPhase {
|
function controlPhase(state: XgridsK1State): XgridsApplicationControlPhase {
|
||||||
@@ -977,7 +1075,10 @@ export function useXgridsK1Runtime(enabled: boolean) {
|
|||||||
!operatorIntents.current.isRuntimeCurrent(runtimeToken)
|
!operatorIntents.current.isRuntimeCurrent(runtimeToken)
|
||||||
|| !runtimeActionArbiter.current.isCurrent(actionToken)
|
|| !runtimeActionArbiter.current.isCurrent(actionToken)
|
||||||
) return false;
|
) return false;
|
||||||
if (!acceptState(nextState)) return false;
|
if (
|
||||||
|
!acceptState(nextState)
|
||||||
|
&& !runtimeActionResponseAlreadyAccepted(nextState, latestState.current)
|
||||||
|
) return false;
|
||||||
if (!runtimeActionArbiter.current.isCurrent(actionToken)) return false;
|
if (!runtimeActionArbiter.current.isCurrent(actionToken)) return false;
|
||||||
return true;
|
return true;
|
||||||
} catch (operationError) {
|
} catch (operationError) {
|
||||||
@@ -1355,6 +1456,13 @@ export function useXgridsK1Runtime(enabled: boolean) {
|
|||||||
throw connectError;
|
throw connectError;
|
||||||
}
|
}
|
||||||
acceptState(failedState);
|
acceptState(failedState);
|
||||||
|
failedState = await awaitNetworkProvisionSettlementAfterLostResponse(
|
||||||
|
failedState,
|
||||||
|
request.idempotency_key,
|
||||||
|
() => xgridsK1Api.getState(),
|
||||||
|
acceptState,
|
||||||
|
assertOperatorIntentCurrent,
|
||||||
|
);
|
||||||
const failedOperation = operationByIdempotencyKey(
|
const failedOperation = operationByIdempotencyKey(
|
||||||
failedState,
|
failedState,
|
||||||
"network.provision",
|
"network.provision",
|
||||||
|
|||||||
@@ -1873,6 +1873,7 @@ class ActiveAcquisitionRecoveryCheckpointStore:
|
|||||||
validation_checkpoint,
|
validation_checkpoint,
|
||||||
status_proof=cessation_status_proof,
|
status_proof=cessation_status_proof,
|
||||||
physical_proof=cessation_physical_proof,
|
physical_proof=cessation_physical_proof,
|
||||||
|
allow_reconciled_transport_change=True,
|
||||||
)
|
)
|
||||||
_require_active_reconciled_standby_shape(
|
_require_active_reconciled_standby_shape(
|
||||||
current,
|
current,
|
||||||
@@ -2251,9 +2252,9 @@ class ActiveAcquisitionRecoveryCheckpointStore:
|
|||||||
"""Cease a proven START that became standby before restart PCL.
|
"""Cease a proven START that became standby before restart PCL.
|
||||||
|
|
||||||
This is deliberately distinct from ``cease_prepared_reconciled``:
|
This is deliberately distinct from ``cease_prepared_reconciled``:
|
||||||
READY/SCAN_OVER proves that the old successful START is no longer
|
READY/SCAN_OVER proves that the previously observed active START is no
|
||||||
active, so there is no transient ACTIVE checkpoint, capture promotion,
|
longer active, so there is no transient ACTIVE checkpoint, capture
|
||||||
first-PCL receipt, or invented STOP edge.
|
promotion, first-PCL receipt, or invented STOP edge.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_validate_mutation_request(
|
_validate_mutation_request(
|
||||||
@@ -2351,7 +2352,11 @@ class ActiveAcquisitionRecoveryCheckpointStore:
|
|||||||
self._clock(),
|
self._clock(),
|
||||||
floor=current.updated_at_utc,
|
floor=current.updated_at_utc,
|
||||||
)
|
)
|
||||||
assert origin_proof.original_project_id_sha256 is not None
|
active_project_id_sha256 = (
|
||||||
|
origin_proof.original_project_id_sha256
|
||||||
|
or origin_proof.reconciled_active_project_id_sha256
|
||||||
|
)
|
||||||
|
assert active_project_id_sha256 is not None
|
||||||
candidate = replace(
|
candidate = replace(
|
||||||
current,
|
current,
|
||||||
revision=revision,
|
revision=revision,
|
||||||
@@ -2360,9 +2365,7 @@ class ActiveAcquisitionRecoveryCheckpointStore:
|
|||||||
physical_lineage_head_revision=(
|
physical_lineage_head_revision=(
|
||||||
cessation_physical_proof.ledger_revision
|
cessation_physical_proof.ledger_revision
|
||||||
),
|
),
|
||||||
active_project_id_sha256=(
|
active_project_id_sha256=active_project_id_sha256,
|
||||||
origin_proof.original_project_id_sha256
|
|
||||||
),
|
|
||||||
current_evidence_session_id=(
|
current_evidence_session_id=(
|
||||||
cessation_status_proof.evidence_session_id
|
cessation_status_proof.evidence_session_id
|
||||||
),
|
),
|
||||||
@@ -2614,6 +2617,7 @@ def _require_binding_matches_checkpoint_values(
|
|||||||
compatibility_profile_id: str,
|
compatibility_profile_id: str,
|
||||||
binding: ActiveAcquisitionRecoveryTransportBinding,
|
binding: ActiveAcquisitionRecoveryTransportBinding,
|
||||||
allow_target_ipv4_change: bool = False,
|
allow_target_ipv4_change: bool = False,
|
||||||
|
allow_connection_mode_change: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
if (
|
if (
|
||||||
binding.logical_device_id != identity.logical_device_id
|
binding.logical_device_id != identity.logical_device_id
|
||||||
@@ -2621,7 +2625,10 @@ def _require_binding_matches_checkpoint_values(
|
|||||||
or binding.device_serial_sha256 != identity.device_serial_sha256
|
or binding.device_serial_sha256 != identity.device_serial_sha256
|
||||||
or binding.compatibility_profile_id != compatibility_profile_id
|
or binding.compatibility_profile_id != compatibility_profile_id
|
||||||
or binding.transport_ref != connection.transport_ref
|
or binding.transport_ref != connection.transport_ref
|
||||||
or binding.connection_mode != connection.connection_mode
|
or (
|
||||||
|
not allow_connection_mode_change
|
||||||
|
and binding.connection_mode != connection.connection_mode
|
||||||
|
)
|
||||||
or (
|
or (
|
||||||
not allow_target_ipv4_change
|
not allow_target_ipv4_change
|
||||||
and binding.target_ipv4 != connection.target_ipv4
|
and binding.target_ipv4 != connection.target_ipv4
|
||||||
@@ -2684,6 +2691,7 @@ def _require_physical_lineage_base(
|
|||||||
proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
|
proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
|
||||||
*,
|
*,
|
||||||
allow_target_ipv4_change: bool = False,
|
allow_target_ipv4_change: bool = False,
|
||||||
|
allow_connection_mode_change: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
if (
|
if (
|
||||||
proof.acquisition_id != checkpoint.acquisition_id
|
proof.acquisition_id != checkpoint.acquisition_id
|
||||||
@@ -2706,6 +2714,7 @@ def _require_physical_lineage_base(
|
|||||||
and proof.binding == checkpoint.current_binding
|
and proof.binding == checkpoint.current_binding
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
|
allow_connection_mode_change=allow_connection_mode_change,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -2994,22 +3003,40 @@ def _require_prepared_resolved_start_standby(
|
|||||||
physical_proof=cessation_physical_proof,
|
physical_proof=cessation_physical_proof,
|
||||||
)
|
)
|
||||||
baseline = origin_proof.baseline_status_proof
|
baseline = origin_proof.baseline_status_proof
|
||||||
if not (
|
exact_origin = bool(
|
||||||
origin_proof.origin_kind == "composite-resolved"
|
origin_proof.origin_kind == "composite-resolved"
|
||||||
and origin_proof.operation_id == checkpoint.original_start_operation_id
|
|
||||||
and origin_proof.acquisition_id == checkpoint.acquisition_id
|
|
||||||
and origin_proof.payload_sha256 == checkpoint.start_payload_sha256
|
|
||||||
and origin_proof.original_attempt_stage == "resolved"
|
and origin_proof.original_attempt_stage == "resolved"
|
||||||
and origin_proof.original_attempt_resolution == "start-active-observed"
|
and origin_proof.original_attempt_resolution == "start-active-observed"
|
||||||
and origin_proof.original_project_id_sha256 is not None
|
and origin_proof.original_project_id_sha256 is not None
|
||||||
|
and origin_proof.reconciled_active_project_id_sha256
|
||||||
|
in {None, origin_proof.original_project_id_sha256}
|
||||||
and origin_proof.project_evidence_strength == "exact-vendor-project-id"
|
and origin_proof.project_evidence_strength == "exact-vendor-project-id"
|
||||||
|
)
|
||||||
|
ambiguous_origin = bool(
|
||||||
|
origin_proof.origin_kind == "ambiguous-reconciled"
|
||||||
|
and origin_proof.original_attempt_stage in {"dispatching", "observing"}
|
||||||
|
and origin_proof.original_attempt_resolution is None
|
||||||
|
and origin_proof.original_project_id_sha256 is None
|
||||||
|
and origin_proof.reconciled_active_project_id_sha256 is not None
|
||||||
|
and origin_proof.project_evidence_strength
|
||||||
|
== "edge-correlated-vendor-project-id"
|
||||||
|
)
|
||||||
|
settlement_project_id_sha256 = (
|
||||||
|
origin_proof.original_project_id_sha256
|
||||||
|
or origin_proof.reconciled_active_project_id_sha256
|
||||||
|
)
|
||||||
|
if not (
|
||||||
|
(exact_origin or ambiguous_origin)
|
||||||
|
and origin_proof.operation_id == checkpoint.original_start_operation_id
|
||||||
|
and origin_proof.acquisition_id == checkpoint.acquisition_id
|
||||||
|
and origin_proof.payload_sha256 == checkpoint.start_payload_sha256
|
||||||
and origin_proof.automatic_replay_allowed is False
|
and origin_proof.automatic_replay_allowed is False
|
||||||
and origin_proof.ledger_revision == cessation_physical_proof.ledger_revision
|
and origin_proof.ledger_revision == cessation_physical_proof.ledger_revision
|
||||||
and origin_proof.physical_proof_id == cessation_physical_proof.proof_id
|
and origin_proof.physical_proof_id == cessation_physical_proof.proof_id
|
||||||
and origin_proof.reconciliation_id == reconciliation_id
|
and origin_proof.reconciliation_id == reconciliation_id
|
||||||
and origin_proof.original_attempt_sha256
|
and origin_proof.original_attempt_sha256
|
||||||
== reconciliation_original_attempt_sha256
|
== reconciliation_original_attempt_sha256
|
||||||
and origin_proof.original_project_id_sha256
|
and settlement_project_id_sha256
|
||||||
== reconciliation_original_project_id_sha256
|
== reconciliation_original_project_id_sha256
|
||||||
and baseline.binding == checkpoint.prepared_binding
|
and baseline.binding == checkpoint.prepared_binding
|
||||||
and baseline.evidence_session_id
|
and baseline.evidence_session_id
|
||||||
@@ -3029,7 +3056,7 @@ def _require_prepared_resolved_start_standby(
|
|||||||
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
||||||
"restart standby settlement requires new runtime, control and evidence sessions"
|
"restart standby settlement requires new runtime, control and evidence sessions"
|
||||||
)
|
)
|
||||||
if not (
|
exact_terminal_lineage = bool(
|
||||||
cessation_physical_proof.operation_id
|
cessation_physical_proof.operation_id
|
||||||
== checkpoint.original_start_operation_id
|
== checkpoint.original_start_operation_id
|
||||||
and cessation_physical_proof.action == "start"
|
and cessation_physical_proof.action == "start"
|
||||||
@@ -3044,9 +3071,30 @@ def _require_prepared_resolved_start_standby(
|
|||||||
== "physical-standby-observed"
|
== "physical-standby-observed"
|
||||||
and cessation_physical_proof.composite_complete
|
and cessation_physical_proof.composite_complete
|
||||||
and cessation_physical_proof.stop_fence == "none"
|
and cessation_physical_proof.stop_fence == "none"
|
||||||
|
)
|
||||||
|
ambiguous_terminal_lineage = bool(
|
||||||
|
cessation_physical_proof.operation_id
|
||||||
|
== checkpoint.original_start_operation_id
|
||||||
|
and cessation_physical_proof.action == "start"
|
||||||
|
and cessation_physical_proof.resolution
|
||||||
|
== "physical-standby-observed"
|
||||||
|
and cessation_physical_proof.payload_sha256
|
||||||
|
== checkpoint.start_payload_sha256
|
||||||
|
and cessation_physical_proof.original_start_payload_sha256
|
||||||
|
== checkpoint.start_payload_sha256
|
||||||
|
and cessation_physical_proof.reconciliation_kind
|
||||||
|
== "resolved-active-cessation"
|
||||||
|
and cessation_physical_proof.reconciliation_resolution
|
||||||
|
== "physical-standby-observed"
|
||||||
|
and not cessation_physical_proof.composite_complete
|
||||||
|
and cessation_physical_proof.stop_fence == "none"
|
||||||
|
)
|
||||||
|
if not (
|
||||||
|
(exact_origin and exact_terminal_lineage)
|
||||||
|
or (ambiguous_origin and ambiguous_terminal_lineage)
|
||||||
):
|
):
|
||||||
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
||||||
"resolved START standby settlement lacks exact terminal lineage"
|
"reconciled START standby settlement lacks exact terminal lineage"
|
||||||
)
|
)
|
||||||
if _validated_timestamp(
|
if _validated_timestamp(
|
||||||
cessation_status_proof.observed_at_utc,
|
cessation_status_proof.observed_at_utc,
|
||||||
@@ -3215,11 +3263,13 @@ def _require_active_cessation(
|
|||||||
*,
|
*,
|
||||||
status_proof: ActiveAcquisitionRecoveryStatusProof,
|
status_proof: ActiveAcquisitionRecoveryStatusProof,
|
||||||
physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
|
physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
|
||||||
|
allow_reconciled_transport_change: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
_require_active_cessation_shape(
|
_require_active_cessation_shape(
|
||||||
checkpoint,
|
checkpoint,
|
||||||
status_proof=status_proof,
|
status_proof=status_proof,
|
||||||
physical_proof=physical_proof,
|
physical_proof=physical_proof,
|
||||||
|
allow_reconciled_transport_change=allow_reconciled_transport_change,
|
||||||
)
|
)
|
||||||
if (
|
if (
|
||||||
checkpoint.physical_lineage_head_revision is None
|
checkpoint.physical_lineage_head_revision is None
|
||||||
@@ -3236,12 +3286,23 @@ def _require_active_cessation_shape(
|
|||||||
*,
|
*,
|
||||||
status_proof: ActiveAcquisitionRecoveryStatusProof,
|
status_proof: ActiveAcquisitionRecoveryStatusProof,
|
||||||
physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
|
physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
|
||||||
|
allow_reconciled_transport_change: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
if status_proof.session_state not in {"ready", "scan_over"}:
|
if status_proof.session_state not in {"ready", "scan_over"}:
|
||||||
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
||||||
"active cessation requires fresh READY/SCAN_OVER"
|
"active cessation requires fresh READY/SCAN_OVER"
|
||||||
)
|
)
|
||||||
_require_status_binding(checkpoint, status_proof)
|
if allow_reconciled_transport_change:
|
||||||
|
_require_binding_matches_checkpoint_values(
|
||||||
|
identity=checkpoint.identity,
|
||||||
|
connection=checkpoint.connection,
|
||||||
|
compatibility_profile_id=checkpoint.compatibility_profile_id,
|
||||||
|
binding=status_proof.binding,
|
||||||
|
allow_target_ipv4_change=True,
|
||||||
|
allow_connection_mode_change=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_require_status_binding(checkpoint, status_proof)
|
||||||
_require_matching_observed_proofs(
|
_require_matching_observed_proofs(
|
||||||
status_proof=status_proof,
|
status_proof=status_proof,
|
||||||
physical_proof=physical_proof,
|
physical_proof=physical_proof,
|
||||||
@@ -3342,8 +3403,24 @@ def _require_active_reconciled_standby_shape(
|
|||||||
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
|
||||||
"restart standby settlement requires fresh READY/SCAN_OVER"
|
"restart standby settlement requires fresh READY/SCAN_OVER"
|
||||||
)
|
)
|
||||||
_require_status_binding(checkpoint, cessation_status_proof)
|
# This is a terminal, read-only settlement of the same pinned K1. The
|
||||||
_require_physical_lineage_base(checkpoint, cessation_physical_proof)
|
# scanner may become observable through Bridge after a Quick Connect STOP
|
||||||
|
# became ambiguous (or vice versa), so the verified route may change
|
||||||
|
# without granting START, STOP, replay, or network-mutation authority.
|
||||||
|
_require_binding_matches_checkpoint_values(
|
||||||
|
identity=checkpoint.identity,
|
||||||
|
connection=checkpoint.connection,
|
||||||
|
compatibility_profile_id=checkpoint.compatibility_profile_id,
|
||||||
|
binding=cessation_status_proof.binding,
|
||||||
|
allow_target_ipv4_change=True,
|
||||||
|
allow_connection_mode_change=True,
|
||||||
|
)
|
||||||
|
_require_physical_lineage_base(
|
||||||
|
checkpoint,
|
||||||
|
cessation_physical_proof,
|
||||||
|
allow_target_ipv4_change=True,
|
||||||
|
allow_connection_mode_change=True,
|
||||||
|
)
|
||||||
_require_matching_observed_proofs(
|
_require_matching_observed_proofs(
|
||||||
status_proof=cessation_status_proof,
|
status_proof=cessation_status_proof,
|
||||||
physical_proof=cessation_physical_proof,
|
physical_proof=cessation_physical_proof,
|
||||||
@@ -4642,6 +4719,12 @@ def _validate_checkpoint_semantics(
|
|||||||
checkpoint.state != "prepared"
|
checkpoint.state != "prepared"
|
||||||
and checkpoint.last_gap_recovered_at_utc is not None
|
and checkpoint.last_gap_recovered_at_utc is not None
|
||||||
),
|
),
|
||||||
|
allow_connection_mode_change=(
|
||||||
|
checkpoint.state == "ceased"
|
||||||
|
and bool(checkpoint.transition_receipts)
|
||||||
|
and checkpoint.transition_receipts[-1].kind
|
||||||
|
== "cease-active-reconciled-standby"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
if checkpoint.last_gap_failed_binding is not None:
|
if checkpoint.last_gap_failed_binding is not None:
|
||||||
_require_binding_matches_checkpoint_values(
|
_require_binding_matches_checkpoint_values(
|
||||||
@@ -4944,7 +5027,16 @@ def _validate_checkpoint_semantics(
|
|||||||
cessation_physical = checkpoint.cessation_physical_proof
|
cessation_physical = checkpoint.cessation_physical_proof
|
||||||
if checkpoint.physical_lineage_head_revision != cessation_physical.ledger_revision:
|
if checkpoint.physical_lineage_head_revision != cessation_physical.ledger_revision:
|
||||||
raise ValueError("ceased physical lineage head does not match cessation proof")
|
raise ValueError("ceased physical lineage head does not match cessation proof")
|
||||||
_require_physical_lineage_base(checkpoint, cessation_physical)
|
_require_physical_lineage_base(
|
||||||
|
checkpoint,
|
||||||
|
cessation_physical,
|
||||||
|
allow_target_ipv4_change=(
|
||||||
|
receipts[-1].kind == "cease-active-reconciled-standby"
|
||||||
|
),
|
||||||
|
allow_connection_mode_change=(
|
||||||
|
receipts[-1].kind == "cease-active-reconciled-standby"
|
||||||
|
),
|
||||||
|
)
|
||||||
if resolved_start_reconciliation_id is not None:
|
if resolved_start_reconciliation_id is not None:
|
||||||
if not (
|
if not (
|
||||||
checkpoint.prepared_resolution_proof == cessation_physical
|
checkpoint.prepared_resolution_proof == cessation_physical
|
||||||
@@ -4983,7 +5075,10 @@ def _validate_checkpoint_semantics(
|
|||||||
checkpoint.current_evidence_session_id
|
checkpoint.current_evidence_session_id
|
||||||
!= checkpoint.cessation_status_proof.evidence_session_id
|
!= checkpoint.cessation_status_proof.evidence_session_id
|
||||||
or checkpoint.active_project_id_sha256
|
or checkpoint.active_project_id_sha256
|
||||||
!= origin_proof.original_project_id_sha256
|
!= (
|
||||||
|
origin_proof.original_project_id_sha256
|
||||||
|
or origin_proof.reconciled_active_project_id_sha256
|
||||||
|
)
|
||||||
or receipts[-1].kind
|
or receipts[-1].kind
|
||||||
!= "cease-prepared-resolved-start-standby"
|
!= "cease-prepared-resolved-start-standby"
|
||||||
):
|
):
|
||||||
@@ -5048,6 +5143,9 @@ def _validate_checkpoint_semantics(
|
|||||||
checkpoint,
|
checkpoint,
|
||||||
status_proof=status,
|
status_proof=status,
|
||||||
physical_proof=cessation_physical,
|
physical_proof=cessation_physical,
|
||||||
|
allow_reconciled_transport_change=(
|
||||||
|
receipts[-1].kind == "cease-active-reconciled-standby"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
if checkpoint.current_evidence_session_id != status.evidence_session_id:
|
if checkpoint.current_evidence_session_id != status.evidence_session_id:
|
||||||
raise ValueError("active cessation evidence session is inconsistent")
|
raise ValueError("active cessation evidence session is inconsistent")
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import queue
|
import queue
|
||||||
import signal
|
import signal
|
||||||
@@ -27,6 +28,8 @@ from k1link.web.camera_archive import (
|
|||||||
|
|
||||||
CameraSourceId = Literal["sensor.camera.left", "sensor.camera.right"]
|
CameraSourceId = Literal["sensor.camera.left", "sensor.camera.right"]
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
CAMERA_SOURCE_PATHS: Final[dict[CameraSourceId, str]] = {
|
CAMERA_SOURCE_PATHS: Final[dict[CameraSourceId, str]] = {
|
||||||
"sensor.camera.left": "/live/chn_left_main",
|
"sensor.camera.left": "/live/chn_left_main",
|
||||||
"sensor.camera.right": "/live/chn_right_main",
|
"sensor.camera.right": "/live/chn_right_main",
|
||||||
@@ -48,7 +51,11 @@ MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS: Final = 64
|
|||||||
# media fragment. This remains a strict per-reader bound and matches the
|
# media fragment. This remains a strict per-reader bound and matches the
|
||||||
# frontend's reviewed 12 MiB receive envelope.
|
# frontend's reviewed 12 MiB receive envelope.
|
||||||
MAX_CAMERA_PREVIEW_QUEUED_BYTES: Final = 12 * 1024 * 1024
|
MAX_CAMERA_PREVIEW_QUEUED_BYTES: Final = 12 * 1024 * 1024
|
||||||
MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS: Final = 3.0
|
# The browser can pause WebSocket consumption for just over three seconds while
|
||||||
|
# the live 3D workspace commits a large reactive update. The byte and segment
|
||||||
|
# caps above remain the hard memory/latency fence; this age fence only avoids
|
||||||
|
# discarding an otherwise healthy preview at that measured UI pause boundary.
|
||||||
|
MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS: Final = 6.0
|
||||||
MAX_CAMERA_PREVIEW_CONSUMERS: Final = 8
|
MAX_CAMERA_PREVIEW_CONSUMERS: Final = 8
|
||||||
CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS: Final = 3.0
|
CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS: Final = 3.0
|
||||||
CAMERA_DRAIN_TIMEOUT_SECONDS: Final = 5.0
|
CAMERA_DRAIN_TIMEOUT_SECONDS: Final = 5.0
|
||||||
@@ -143,6 +150,7 @@ class _CameraPreviewSegmentQueue:
|
|||||||
self._queued_bytes = 0
|
self._queued_bytes = 0
|
||||||
self._closed = False
|
self._closed = False
|
||||||
self._clock = clock
|
self._clock = clock
|
||||||
|
self._last_rejection_reason: str | None = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def queued_bytes(self) -> int:
|
def queued_bytes(self) -> int:
|
||||||
@@ -158,14 +166,19 @@ class _CameraPreviewSegmentQueue:
|
|||||||
payload_size = len(segment[1])
|
payload_size = len(segment[1])
|
||||||
with self._condition:
|
with self._condition:
|
||||||
if self._closed:
|
if self._closed:
|
||||||
|
self._last_rejection_reason = "queue-closed"
|
||||||
return False
|
return False
|
||||||
now = self._clock()
|
now = self._clock()
|
||||||
if self._segments and now - self._segments[0][0] > MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS:
|
if self._segments and now - self._segments[0][0] > MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS:
|
||||||
|
self._last_rejection_reason = "queue-age"
|
||||||
return False
|
return False
|
||||||
if len(self._segments) >= MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS:
|
if len(self._segments) >= MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS:
|
||||||
|
self._last_rejection_reason = "queue-segments"
|
||||||
return False
|
return False
|
||||||
if self._queued_bytes + payload_size > MAX_CAMERA_PREVIEW_QUEUED_BYTES:
|
if self._queued_bytes + payload_size > MAX_CAMERA_PREVIEW_QUEUED_BYTES:
|
||||||
|
self._last_rejection_reason = "queue-bytes"
|
||||||
return False
|
return False
|
||||||
|
self._last_rejection_reason = None
|
||||||
self._segments.append((now, segment))
|
self._segments.append((now, segment))
|
||||||
self._queued_bytes += payload_size
|
self._queued_bytes += payload_size
|
||||||
self._condition.notify()
|
self._condition.notify()
|
||||||
@@ -188,6 +201,23 @@ class _CameraPreviewSegmentQueue:
|
|||||||
return segment
|
return segment
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def diagnostic_snapshot(self) -> dict[str, int | str | None]:
|
||||||
|
"""Return bounded observer facts without changing delivery state."""
|
||||||
|
|
||||||
|
with self._condition:
|
||||||
|
now = self._clock()
|
||||||
|
oldest_age_ms = (
|
||||||
|
max(0, int((now - self._segments[0][0]) * 1_000))
|
||||||
|
if self._segments
|
||||||
|
else 0
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"queued_bytes": self._queued_bytes,
|
||||||
|
"queued_segments": len(self._segments),
|
||||||
|
"oldest_age_ms": oldest_age_ms,
|
||||||
|
"rejection_reason": self._last_rejection_reason,
|
||||||
|
}
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
with self._condition:
|
with self._condition:
|
||||||
self._segments.clear()
|
self._segments.clear()
|
||||||
@@ -1269,11 +1299,34 @@ class XgridsK1CameraGateway:
|
|||||||
*,
|
*,
|
||||||
failure_code: str = "consumer-too-slow",
|
failure_code: str = "consumer-too-slow",
|
||||||
) -> None:
|
) -> None:
|
||||||
|
queue_diagnostic = delivery.segments.diagnostic_snapshot()
|
||||||
with self._lock:
|
with self._lock:
|
||||||
if self._producer is producer and producer.deliveries.get(id(delivery)) is delivery:
|
if self._producer is producer and producer.deliveries.get(id(delivery)) is delivery:
|
||||||
producer.deliveries.pop(id(delivery), None)
|
producer.deliveries.pop(id(delivery), None)
|
||||||
delivery.failure_code = failure_code
|
delivery.failure_code = failure_code
|
||||||
self._revision += 1
|
self._revision += 1
|
||||||
|
logger.warning(
|
||||||
|
"K1 camera preview browser lease retired: generation=%s "
|
||||||
|
"failure=%s rejection=%s queued_segments=%s queued_bytes=%s "
|
||||||
|
"oldest_age_ms=%s",
|
||||||
|
delivery.generation,
|
||||||
|
failure_code,
|
||||||
|
queue_diagnostic["rejection_reason"],
|
||||||
|
queue_diagnostic["queued_segments"],
|
||||||
|
queue_diagnostic["queued_bytes"],
|
||||||
|
queue_diagnostic["oldest_age_ms"],
|
||||||
|
extra={
|
||||||
|
"event_code": "k1_camera_preview_delivery_retired",
|
||||||
|
"failure_code": failure_code,
|
||||||
|
"camera_generation": delivery.generation,
|
||||||
|
"camera_queue_rejection_reason": queue_diagnostic["rejection_reason"],
|
||||||
|
"camera_queue_segments": queue_diagnostic["queued_segments"],
|
||||||
|
"camera_queue_bytes": queue_diagnostic["queued_bytes"],
|
||||||
|
"camera_queue_oldest_age_ms": queue_diagnostic["oldest_age_ms"],
|
||||||
|
"device_write_performed": False,
|
||||||
|
"automatic_retry": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
_close_segment_queue(delivery)
|
_close_segment_queue(delivery)
|
||||||
|
|
||||||
def _mark_producer_failure(
|
def _mark_producer_failure(
|
||||||
|
|||||||
@@ -10019,7 +10019,9 @@ class XgridsK1CompatibilityService:
|
|||||||
"retryable": terminal.retryable,
|
"retryable": terminal.retryable,
|
||||||
"safe_to_retry": terminal.safe_to_retry,
|
"safe_to_retry": terminal.safe_to_retry,
|
||||||
"side_effect_status": (
|
"side_effect_status": (
|
||||||
"none" if terminal.side_effect_status == "none" else "confirmed"
|
terminal.side_effect_status
|
||||||
|
if terminal.side_effect_status in {"none", "unknown"}
|
||||||
|
else "confirmed"
|
||||||
),
|
),
|
||||||
"durable_replay": True,
|
"durable_replay": True,
|
||||||
},
|
},
|
||||||
@@ -10748,6 +10750,7 @@ class XgridsK1CompatibilityService:
|
|||||||
quick_connect_host_profile_id(selected_device_name) if quick_connect else None
|
quick_connect_host_profile_id(selected_device_name) if quick_connect else None
|
||||||
)
|
)
|
||||||
operation_stage = "device-ap-activation" if quick_connect else "ble-provisioning-write"
|
operation_stage = "device-ap-activation" if quick_connect else "ble-provisioning-write"
|
||||||
|
quick_host_association: Mapping[str, Any] | None = None
|
||||||
|
|
||||||
# Resolve the durable identity expectation while the previous
|
# Resolve the durable identity expectation while the previous
|
||||||
# control session is still intact. A corrupt/unavailable pin store
|
# control session is still intact. A corrupt/unavailable pin store
|
||||||
@@ -11018,6 +11021,7 @@ class XgridsK1CompatibilityService:
|
|||||||
self._duplicate_network_process_fence_descriptor
|
self._duplicate_network_process_fence_descriptor
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
quick_host_association = association
|
||||||
except HostWifiProfileError as exc:
|
except HostWifiProfileError as exc:
|
||||||
write_json_atomic(
|
write_json_atomic(
|
||||||
session_dir / "host-wifi-association.redacted.json",
|
session_dir / "host-wifi-association.redacted.json",
|
||||||
@@ -11249,8 +11253,12 @@ class XgridsK1CompatibilityService:
|
|||||||
# decide whether that separately authorized host mutation is
|
# decide whether that separately authorized host mutation is
|
||||||
# necessary at all.
|
# necessary at all.
|
||||||
host_route_class: str | None = None
|
host_route_class: str | None = None
|
||||||
host_association: Mapping[str, Any] | None = None
|
host_association: Mapping[str, Any] | None = quick_host_association
|
||||||
host_wifi_association_outcome: str | None = None
|
host_wifi_association_outcome: str | None = (
|
||||||
|
str(quick_host_association.get("outcome") or "unknown")
|
||||||
|
if quick_host_association is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
if request.connection_mode == "bridge" and request.allow_host_wifi_switch:
|
if request.connection_mode == "bridge" and request.allow_host_wifi_switch:
|
||||||
host_route_class = _host_route_class(ipv4)
|
host_route_class = _host_route_class(ipv4)
|
||||||
if (
|
if (
|
||||||
@@ -11758,11 +11766,6 @@ class XgridsK1CompatibilityService:
|
|||||||
and reconciliation.original_attempt.action == "stop"
|
and reconciliation.original_attempt.action == "stop"
|
||||||
and reconciliation.original_attempt.stage == "observing"
|
and reconciliation.original_attempt.stage == "observing"
|
||||||
and reconciliation.original_attempt.resolution is None
|
and reconciliation.original_attempt.resolution is None
|
||||||
and reconciliation.original_attempt.publish_call_returned is True
|
|
||||||
and reconciliation.original_attempt.packet_id is not None
|
|
||||||
and reconciliation.original_attempt.qos2_completed
|
|
||||||
and reconciliation.original_attempt.application_response is not None
|
|
||||||
and reconciliation.original_attempt.application_response.success
|
|
||||||
and reconciliation.observation.source
|
and reconciliation.observation.source
|
||||||
== "explicit-read-only-reconciliation"
|
== "explicit-read-only-reconciliation"
|
||||||
and reconciliation.observation.session_state in {"ready", "scan_over"}
|
and reconciliation.observation.session_state in {"ready", "scan_over"}
|
||||||
@@ -12932,25 +12935,35 @@ class XgridsK1CompatibilityService:
|
|||||||
raise ActiveAcquisitionRecoveryCheckpointError(
|
raise ActiveAcquisitionRecoveryCheckpointError(
|
||||||
"ambiguous START origin has the wrong physical classification"
|
"ambiguous START origin has the wrong physical classification"
|
||||||
)
|
)
|
||||||
if reconciliation.kind == "resolved-active-rebind":
|
prior_active_project_id_sha256 = next(
|
||||||
prior_active_project_id_sha256 = next(
|
(
|
||||||
(
|
item.observation.project_id_sha256
|
||||||
item.observation.project_id_sha256
|
for item in reversed(record.reconciliations[:-1])
|
||||||
for item in reversed(record.reconciliations[:-1])
|
if item.original_attempt_sha256
|
||||||
if item.original_attempt_sha256
|
== reconciliation.original_attempt_sha256
|
||||||
== reconciliation.original_attempt_sha256
|
and item.resolution == "physical-active-observed"
|
||||||
and item.resolution == "physical-active-observed"
|
and item.observation.session_state == "scanning"
|
||||||
),
|
and item.observation.project_bound
|
||||||
None,
|
and item.observation.init_ready
|
||||||
|
and not item.observation.mqtt_retained
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if reconciliation.kind == "resolved-active-rebind" and (
|
||||||
|
prior_active_project_id_sha256 is None
|
||||||
|
or reconciliation.observation.project_id_sha256
|
||||||
|
!= prior_active_project_id_sha256
|
||||||
|
):
|
||||||
|
raise ActiveAcquisitionRecoveryCheckpointError(
|
||||||
|
"ambiguous START rebind changed or lacks its edge-correlated project"
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
reconciliation.kind == "resolved-active-cessation"
|
||||||
|
and prior_active_project_id_sha256 is None
|
||||||
|
):
|
||||||
|
raise ActiveAcquisitionRecoveryCheckpointError(
|
||||||
|
"ambiguous START cessation lacks its prior active project proof"
|
||||||
)
|
)
|
||||||
if (
|
|
||||||
prior_active_project_id_sha256 is None
|
|
||||||
or reconciliation.observation.project_id_sha256
|
|
||||||
!= prior_active_project_id_sha256
|
|
||||||
):
|
|
||||||
raise ActiveAcquisitionRecoveryCheckpointError(
|
|
||||||
"ambiguous START rebind changed or lacks its edge-correlated project"
|
|
||||||
)
|
|
||||||
origin_kind = "ambiguous-reconciled"
|
origin_kind = "ambiguous-reconciled"
|
||||||
project_strength = "edge-correlated-vendor-project-id"
|
project_strength = "edge-correlated-vendor-project-id"
|
||||||
original_project_id_sha256 = None
|
original_project_id_sha256 = None
|
||||||
@@ -12980,6 +12993,8 @@ class XgridsK1CompatibilityService:
|
|||||||
reconciled_active_project_id_sha256=(
|
reconciled_active_project_id_sha256=(
|
||||||
reconciliation.observation.project_id_sha256
|
reconciliation.observation.project_id_sha256
|
||||||
if active_observation
|
if active_observation
|
||||||
|
else prior_active_project_id_sha256
|
||||||
|
if ambiguous_origin
|
||||||
else None
|
else None
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -13151,7 +13166,7 @@ class XgridsK1CompatibilityService:
|
|||||||
else status.observed_at_utc
|
else status.observed_at_utc
|
||||||
)
|
)
|
||||||
if checkpoint.state == "active":
|
if checkpoint.state == "active":
|
||||||
store.cease_active_reconciled_standby(
|
settled_checkpoint = store.cease_active_reconciled_standby(
|
||||||
transition_id=self._active_acquisition_checkpoint_transition_id(
|
transition_id=self._active_acquisition_checkpoint_transition_id(
|
||||||
"cease-active-reconciled-standby",
|
"cease-active-reconciled-standby",
|
||||||
checkpoint.acquisition_id,
|
checkpoint.acquisition_id,
|
||||||
@@ -13167,6 +13182,25 @@ class XgridsK1CompatibilityService:
|
|||||||
cessation_status_proof=status_proof,
|
cessation_status_proof=status_proof,
|
||||||
cessation_physical_proof=physical_proof,
|
cessation_physical_proof=physical_proof,
|
||||||
)
|
)
|
||||||
|
logger.info(
|
||||||
|
"active acquisition checkpoint closed from verified standby",
|
||||||
|
extra={
|
||||||
|
"event_code": (
|
||||||
|
"active_acquisition_checkpoint_reconciled_standby_settled"
|
||||||
|
),
|
||||||
|
"acquisition_id": checkpoint.acquisition_id,
|
||||||
|
"checkpoint_revision_before": checkpoint.revision,
|
||||||
|
"checkpoint_revision_after": settled_checkpoint.revision,
|
||||||
|
"connection_mode_before": (
|
||||||
|
checkpoint.current_binding.connection_mode
|
||||||
|
),
|
||||||
|
"connection_mode_after": binding.connection_mode,
|
||||||
|
"reconciliation_id": reconciliation_id,
|
||||||
|
"observed_session_state": status.session_state,
|
||||||
|
"device_write_performed": False,
|
||||||
|
"automatic_retry": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
elif checkpoint.state == "prepared" and (
|
elif checkpoint.state == "prepared" and (
|
||||||
reconciliation.kind == "resolved-active-cessation"
|
reconciliation.kind == "resolved-active-cessation"
|
||||||
):
|
):
|
||||||
@@ -13176,14 +13210,13 @@ class XgridsK1CompatibilityService:
|
|||||||
reconciliation=reconciliation,
|
reconciliation=reconciliation,
|
||||||
physical_proof=physical_proof,
|
physical_proof=physical_proof,
|
||||||
)
|
)
|
||||||
original_project_id_sha256 = (
|
settlement_project_id_sha256 = (
|
||||||
reconciliation.original_attempt.last_status.project_id_sha256
|
origin_proof.original_project_id_sha256
|
||||||
if reconciliation.original_attempt.last_status is not None
|
or origin_proof.reconciled_active_project_id_sha256
|
||||||
else None
|
|
||||||
)
|
)
|
||||||
if original_project_id_sha256 is None:
|
if settlement_project_id_sha256 is None:
|
||||||
raise ActiveAcquisitionRecoveryCheckpointError(
|
raise ActiveAcquisitionRecoveryCheckpointError(
|
||||||
"resolved START standby lacks its original vendor project"
|
"reconciled START standby lacks its proven vendor project"
|
||||||
)
|
)
|
||||||
store.cease_prepared_resolved_start_standby(
|
store.cease_prepared_resolved_start_standby(
|
||||||
transition_id=self._active_acquisition_checkpoint_transition_id(
|
transition_id=self._active_acquisition_checkpoint_transition_id(
|
||||||
@@ -13206,7 +13239,7 @@ class XgridsK1CompatibilityService:
|
|||||||
reconciliation.original_attempt_sha256
|
reconciliation.original_attempt_sha256
|
||||||
),
|
),
|
||||||
reconciliation_original_project_id_sha256=(
|
reconciliation_original_project_id_sha256=(
|
||||||
original_project_id_sha256
|
settlement_project_id_sha256
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@@ -13361,7 +13394,7 @@ class XgridsK1CompatibilityService:
|
|||||||
)
|
)
|
||||||
and reconciliation.kind == "prepared-stop-classification"
|
and reconciliation.kind == "prepared-stop-classification"
|
||||||
)
|
)
|
||||||
dispatched_then_observed_standby = bool(
|
ambiguous_stop_observed_standby = bool(
|
||||||
record is not None
|
record is not None
|
||||||
and record.resolution == "physical-standby-observed"
|
and record.resolution == "physical-standby-observed"
|
||||||
and reconciliation is not None
|
and reconciliation is not None
|
||||||
@@ -13371,11 +13404,6 @@ class XgridsK1CompatibilityService:
|
|||||||
and reconciliation.original_attempt.action == "stop"
|
and reconciliation.original_attempt.action == "stop"
|
||||||
and reconciliation.original_attempt.stage == "observing"
|
and reconciliation.original_attempt.stage == "observing"
|
||||||
and reconciliation.original_attempt.resolution is None
|
and reconciliation.original_attempt.resolution is None
|
||||||
and reconciliation.original_attempt.publish_call_returned is True
|
|
||||||
and reconciliation.original_attempt.packet_id is not None
|
|
||||||
and reconciliation.original_attempt.qos2_completed
|
|
||||||
and reconciliation.original_attempt.application_response is not None
|
|
||||||
and reconciliation.original_attempt.application_response.success
|
|
||||||
)
|
)
|
||||||
if not (
|
if not (
|
||||||
ledger_snapshot.status == "resolved"
|
ledger_snapshot.status == "resolved"
|
||||||
@@ -13383,7 +13411,7 @@ class XgridsK1CompatibilityService:
|
|||||||
and record.action == "stop"
|
and record.action == "stop"
|
||||||
and record.stage == "resolved"
|
and record.stage == "resolved"
|
||||||
and reconciliation is not None
|
and reconciliation is not None
|
||||||
and (definitely_undispatched or dispatched_then_observed_standby)
|
and (definitely_undispatched or ambiguous_stop_observed_standby)
|
||||||
and reconciliation.resolution == "physical-standby-observed"
|
and reconciliation.resolution == "physical-standby-observed"
|
||||||
and reconciliation.observation.source
|
and reconciliation.observation.source
|
||||||
== "explicit-read-only-reconciliation"
|
== "explicit-read-only-reconciliation"
|
||||||
@@ -15573,13 +15601,47 @@ class XgridsK1CompatibilityService:
|
|||||||
else:
|
else:
|
||||||
restart_outcome = None
|
restart_outcome = None
|
||||||
if allow_receiver_rehydrate:
|
if allow_receiver_rehydrate:
|
||||||
restart_outcome = (
|
try:
|
||||||
await self._rehydrate_active_acquisition_after_restart(
|
restart_outcome = (
|
||||||
token=checkpoint_trust_token,
|
await self._rehydrate_active_acquisition_after_restart(
|
||||||
reconciliation_id=reconciliation_id,
|
token=checkpoint_trust_token,
|
||||||
reconciled_record=reconciled_record,
|
reconciliation_id=reconciliation_id,
|
||||||
|
reconciled_record=reconciled_record,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except ActiveAcquisitionRecoveryCheckpointError as exc:
|
||||||
|
# Receiver resurrection is optional after an exact
|
||||||
|
# read-only SCANNING proof. A stale/corrupt local
|
||||||
|
# checkpoint must revoke that broader authority, but
|
||||||
|
# it must not discard the narrower, already durable
|
||||||
|
# STOP-only authority derived from DeviceInfo plus a
|
||||||
|
# fresh non-retained DeviceStatus.
|
||||||
|
reason_code = str(
|
||||||
|
getattr(
|
||||||
|
exc,
|
||||||
|
"reason_code",
|
||||||
|
(
|
||||||
|
"active-acquisition-recovery-"
|
||||||
|
"checkpoint-rehydrate-failed"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self._mark_active_acquisition_checkpoint_untrusted(
|
||||||
|
trust="unavailable",
|
||||||
|
reason_code=reason_code,
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
"K1 checkpoint rejected receiver rehydration; "
|
||||||
|
"continuing with proven STOP-only control",
|
||||||
|
extra={
|
||||||
|
"event_code": (
|
||||||
|
"k1_restart_rehydration_checkpoint_rejected"
|
||||||
|
),
|
||||||
|
"reason_code": reason_code,
|
||||||
|
"device_write_performed": False,
|
||||||
|
"automatic_retry": False,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
)
|
|
||||||
if restart_outcome is None:
|
if restart_outcome is None:
|
||||||
checkpoint_lineage = (
|
checkpoint_lineage = (
|
||||||
self._restart_stop_only_checkpoint_lineage(
|
self._restart_stop_only_checkpoint_lineage(
|
||||||
|
|||||||
@@ -37,11 +37,11 @@ NetworkProvisioningIdempotencyStage = Literal["prepared", "unresolved", "termina
|
|||||||
NetworkProvisioningIdempotencyDisposition = Literal["admitted", "terminal-replay"]
|
NetworkProvisioningIdempotencyDisposition = Literal["admitted", "terminal-replay"]
|
||||||
NetworkProvisioningIdempotencyStatus = Literal["empty", "ready", "blocked", "corrupt"]
|
NetworkProvisioningIdempotencyStatus = Literal["empty", "ready", "blocked", "corrupt"]
|
||||||
NetworkProvisioningTerminalOutcome = Literal["succeeded", "failed", "cancelled"]
|
NetworkProvisioningTerminalOutcome = Literal["succeeded", "failed", "cancelled"]
|
||||||
NetworkProvisioningSideEffectStatus = Literal["none", "applied", "reconciled"]
|
NetworkProvisioningSideEffectStatus = Literal["none", "applied", "reconciled", "unknown"]
|
||||||
|
|
||||||
_STAGES = frozenset({"prepared", "unresolved", "terminal"})
|
_STAGES = frozenset({"prepared", "unresolved", "terminal"})
|
||||||
_OUTCOMES = frozenset({"succeeded", "failed", "cancelled"})
|
_OUTCOMES = frozenset({"succeeded", "failed", "cancelled"})
|
||||||
_SIDE_EFFECT_STATUSES = frozenset({"none", "applied", "reconciled"})
|
_SIDE_EFFECT_STATUSES = frozenset({"none", "applied", "reconciled", "unknown"})
|
||||||
_SAFE_ACTION = re.compile(r"^[a-z][a-z0-9._-]{0,95}$")
|
_SAFE_ACTION = re.compile(r"^[a-z][a-z0-9._-]{0,95}$")
|
||||||
_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$")
|
_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$")
|
||||||
_SAFE_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,127}$")
|
_SAFE_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,127}$")
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ _EXTRA_FIELDS: Final = (
|
|||||||
"stale_session_retired",
|
"stale_session_retired",
|
||||||
"stream_id",
|
"stream_id",
|
||||||
"ui_build_id",
|
"ui_build_id",
|
||||||
|
"expected_ui_build_id",
|
||||||
"document_instance_id",
|
"document_instance_id",
|
||||||
"viewer_instance_id",
|
"viewer_instance_id",
|
||||||
"lifecycle_generation",
|
"lifecycle_generation",
|
||||||
@@ -63,6 +64,17 @@ _EXTRA_FIELDS: Final = (
|
|||||||
"viewer_range_max_ns",
|
"viewer_range_max_ns",
|
||||||
"stalled_for_ms",
|
"stalled_for_ms",
|
||||||
"recovery_attempt",
|
"recovery_attempt",
|
||||||
|
"camera_queue_bytes",
|
||||||
|
"camera_queue_segments",
|
||||||
|
"camera_queue_oldest_age_ms",
|
||||||
|
"camera_queue_rejection_reason",
|
||||||
|
"camera_retry_count",
|
||||||
|
"camera_generation",
|
||||||
|
"websocket_close_code",
|
||||||
|
"transport_epoch",
|
||||||
|
"camera_append_error_name",
|
||||||
|
"camera_media_source_state",
|
||||||
|
"camera_video_error_code",
|
||||||
"preferred_port",
|
"preferred_port",
|
||||||
"selected_port",
|
"selected_port",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
from fastapi import APIRouter, Response
|
from fastapi import APIRouter, Request, Response
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
@@ -17,12 +18,27 @@ LiveViewerEventCode = Literal[
|
|||||||
"live_receiver_recovery_exhausted",
|
"live_receiver_recovery_exhausted",
|
||||||
"live_receiver_active_store_admitted",
|
"live_receiver_active_store_admitted",
|
||||||
"live_receiver_error",
|
"live_receiver_error",
|
||||||
|
"live_camera_transport_restart_requested",
|
||||||
|
"live_camera_transport_playing",
|
||||||
]
|
]
|
||||||
LiveViewerFailureStage = Literal[
|
LiveViewerFailureStage = Literal[
|
||||||
"recording-open-timeout",
|
"recording-open-timeout",
|
||||||
"viewer-start",
|
"viewer-start",
|
||||||
"module-load",
|
"module-load",
|
||||||
"receiver-stalled",
|
"receiver-stalled",
|
||||||
|
"camera-first-media-timeout",
|
||||||
|
"camera-first-playable-timeout",
|
||||||
|
"camera-queue-capacity",
|
||||||
|
"camera-append-quota",
|
||||||
|
"camera-append-error",
|
||||||
|
"camera-source-buffer-error",
|
||||||
|
"camera-fragment-decode",
|
||||||
|
"camera-websocket-error",
|
||||||
|
"camera-websocket-close",
|
||||||
|
"camera-heartbeat-reopen",
|
||||||
|
"camera-visibility-reopen",
|
||||||
|
"camera-network-online-reopen",
|
||||||
|
"camera-page-restore-reopen",
|
||||||
]
|
]
|
||||||
|
|
||||||
LIVE_VIEWER_DIAGNOSTIC_SCHEMA = "missioncore.live-viewer-diagnostic/v2"
|
LIVE_VIEWER_DIAGNOSTIC_SCHEMA = "missioncore.live-viewer-diagnostic/v2"
|
||||||
@@ -61,6 +77,17 @@ class LiveViewerDiagnosticEvent(BaseModel):
|
|||||||
viewer_range_max_ns: int | None = Field(default=None, ge=0)
|
viewer_range_max_ns: int | None = Field(default=None, ge=0)
|
||||||
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
|
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
|
||||||
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
|
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
|
||||||
|
camera_queue_bytes: int | None = Field(default=None, ge=0, le=16_777_216)
|
||||||
|
camera_queue_segments: int | None = Field(default=None, ge=0, le=128)
|
||||||
|
camera_retry_count: int | None = Field(default=None, ge=1, le=4)
|
||||||
|
websocket_close_code: int | None = Field(default=None, ge=0, le=4_999)
|
||||||
|
transport_epoch: int | None = Field(default=None, ge=1)
|
||||||
|
camera_append_error_name: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
pattern=r"^[A-Za-z][A-Za-z0-9]{0,63}$",
|
||||||
|
)
|
||||||
|
camera_media_source_state: Literal["closed", "open", "ended"] | None = None
|
||||||
|
camera_video_error_code: int | None = Field(default=None, ge=0, le=4)
|
||||||
|
|
||||||
|
|
||||||
def build_viewer_diagnostics_router(
|
def build_viewer_diagnostics_router(
|
||||||
@@ -70,7 +97,7 @@ def build_viewer_diagnostics_router(
|
|||||||
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
|
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
|
||||||
|
|
||||||
@router.get("/client-contract")
|
@router.get("/client-contract")
|
||||||
def get_live_viewer_client_contract() -> JSONResponse:
|
def get_live_viewer_client_contract(request: Request) -> JSONResponse:
|
||||||
expected = expected_ui_build_id()
|
expected = expected_ui_build_id()
|
||||||
if expected is None:
|
if expected is None:
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
@@ -81,6 +108,24 @@ def build_viewer_diagnostics_router(
|
|||||||
},
|
},
|
||||||
headers={"Cache-Control": "no-store"},
|
headers={"Cache-Control": "no-store"},
|
||||||
)
|
)
|
||||||
|
loaded = request.headers.get(UI_BUILD_HEADER)
|
||||||
|
if (
|
||||||
|
loaded not in {None, "development", expected}
|
||||||
|
and re.fullmatch(_UI_BUILD_ID_PATTERN, loaded) is not None
|
||||||
|
):
|
||||||
|
logger.info(
|
||||||
|
"Mission Core UI build drift observed; automatic page reload suppressed: "
|
||||||
|
"loaded=%s expected=%s",
|
||||||
|
loaded,
|
||||||
|
expected,
|
||||||
|
extra={
|
||||||
|
"event_code": "ui_build_drift_reload_suppressed",
|
||||||
|
"ui_build_id": loaded,
|
||||||
|
"expected_ui_build_id": expected,
|
||||||
|
"device_write_performed": False,
|
||||||
|
"automatic_retry": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
content={
|
content={
|
||||||
"schema_version": LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA,
|
"schema_version": LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA,
|
||||||
@@ -142,6 +187,14 @@ def build_viewer_diagnostics_router(
|
|||||||
"viewer_range_max_ns": event.viewer_range_max_ns,
|
"viewer_range_max_ns": event.viewer_range_max_ns,
|
||||||
"stalled_for_ms": event.stalled_for_ms,
|
"stalled_for_ms": event.stalled_for_ms,
|
||||||
"recovery_attempt": event.recovery_attempt,
|
"recovery_attempt": event.recovery_attempt,
|
||||||
|
"camera_queue_bytes": event.camera_queue_bytes,
|
||||||
|
"camera_queue_segments": event.camera_queue_segments,
|
||||||
|
"camera_retry_count": event.camera_retry_count,
|
||||||
|
"websocket_close_code": event.websocket_close_code,
|
||||||
|
"transport_epoch": event.transport_epoch,
|
||||||
|
"camera_append_error_name": event.camera_append_error_name,
|
||||||
|
"camera_media_source_state": event.camera_media_source_state,
|
||||||
|
"camera_video_error_code": event.camera_video_error_code,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
return Response(status_code=204, headers={UI_BUILD_HEADER: expected})
|
return Response(status_code=204, headers={UI_BUILD_HEADER: expected})
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from k1link.device_plugins.xgrids_k1.protocol.normalizer import normalize_k1_mes
|
|||||||
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
|
from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage
|
||||||
from k1link.device_plugins.xgrids_k1.viewer.runtime import VisualizationRuntime
|
from k1link.device_plugins.xgrids_k1.viewer.runtime import VisualizationRuntime
|
||||||
from k1link.viewer.rerun_bridge import (
|
from k1link.viewer.rerun_bridge import (
|
||||||
|
LIVE_GRPC_BUFFER_LIMIT,
|
||||||
RerunBridge,
|
RerunBridge,
|
||||||
RerunSceneSettings,
|
RerunSceneSettings,
|
||||||
_live_time_panel,
|
_live_time_panel,
|
||||||
@@ -37,8 +38,10 @@ class FakeRecording:
|
|||||||
self.blueprints: list[object] = []
|
self.blueprints: list[object] = []
|
||||||
self.disconnected = False
|
self.disconnected = False
|
||||||
self.flush_count = 0
|
self.flush_count = 0
|
||||||
|
self.serve_grpc_options: dict[str, object] | None = None
|
||||||
|
|
||||||
def serve_grpc(self, **_: object) -> str:
|
def serve_grpc(self, **options: object) -> str:
|
||||||
|
self.serve_grpc_options = options
|
||||||
return "rerun+http://127.0.0.1:9876/proxy"
|
return "rerun+http://127.0.0.1:9876/proxy"
|
||||||
|
|
||||||
def log(self, path: str, entity: object, *, static: bool = False) -> None:
|
def log(self, path: str, entity: object, *, static: bool = False) -> None:
|
||||||
@@ -299,6 +302,10 @@ def test_legacy_points_and_pose_are_logged_to_rerun(
|
|||||||
"stream_time",
|
"stream_time",
|
||||||
}
|
}
|
||||||
assert recording.flush_count == 1
|
assert recording.flush_count == 1
|
||||||
|
assert recording.serve_grpc_options is not None
|
||||||
|
assert recording.serve_grpc_options["server_memory_limit"] == "32MiB"
|
||||||
|
assert recording.serve_grpc_options["newest_first"] is False
|
||||||
|
assert LIVE_GRPC_BUFFER_LIMIT == "32MiB"
|
||||||
|
|
||||||
bridge.close()
|
bridge.close()
|
||||||
assert recording.disconnected is True
|
assert recording.disconnected is True
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import pytest
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
from fastapi.routing import APIRoute
|
from fastapi.routing import APIRoute
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
from starlette.requests import Request
|
||||||
|
|
||||||
from k1link.web.runtime_diagnostics import (
|
from k1link.web.runtime_diagnostics import (
|
||||||
SCANNER_LOGGER_NAME,
|
SCANNER_LOGGER_NAME,
|
||||||
@@ -34,6 +35,13 @@ def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
|||||||
raise AssertionError(f"{method} {path} route is missing")
|
raise AssertionError(f"{method} {path} route is missing")
|
||||||
|
|
||||||
|
|
||||||
|
def _request(*, ui_build_id: str | None = None) -> Request:
|
||||||
|
headers = []
|
||||||
|
if ui_build_id is not None:
|
||||||
|
headers.append((b"x-missioncore-ui-build", ui_build_id.encode("ascii")))
|
||||||
|
return Request({"type": "http", "method": "GET", "path": "/", "headers": headers})
|
||||||
|
|
||||||
|
|
||||||
def test_private_scanner_diagnostics_are_durable_structured_and_bounded(
|
def test_private_scanner_diagnostics_are_durable_structured_and_bounded(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -76,6 +84,11 @@ def test_private_scanner_diagnostics_are_durable_structured_and_bounded(
|
|||||||
"device_write_performed": False,
|
"device_write_performed": False,
|
||||||
"preferred_port": 9876,
|
"preferred_port": 9876,
|
||||||
"selected_port": 9877,
|
"selected_port": 9877,
|
||||||
|
"camera_queue_bytes": 12_000_000,
|
||||||
|
"camera_queue_segments": 96,
|
||||||
|
"camera_retry_count": 3,
|
||||||
|
"websocket_close_code": 4_008,
|
||||||
|
"transport_epoch": 7,
|
||||||
"unapproved_secret_field": "must-not-be-written",
|
"unapproved_secret_field": "must-not-be-written",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -118,6 +131,11 @@ def test_private_scanner_diagnostics_are_durable_structured_and_bounded(
|
|||||||
assert document["device_write_performed"] is False
|
assert document["device_write_performed"] is False
|
||||||
assert document["preferred_port"] == 9876
|
assert document["preferred_port"] == 9876
|
||||||
assert document["selected_port"] == 9877
|
assert document["selected_port"] == 9877
|
||||||
|
assert document["camera_queue_bytes"] == 12_000_000
|
||||||
|
assert document["camera_queue_segments"] == 96
|
||||||
|
assert document["camera_retry_count"] == 3
|
||||||
|
assert document["websocket_close_code"] == 4_008
|
||||||
|
assert document["transport_epoch"] == 7
|
||||||
assert "unapproved_secret_field" not in document
|
assert "unapproved_secret_field" not in document
|
||||||
|
|
||||||
parent = logging.getLogger(SCANNER_LOGGER_NAME)
|
parent = logging.getLogger(SCANNER_LOGGER_NAME)
|
||||||
@@ -180,6 +198,40 @@ def test_live_viewer_diagnostic_endpoint_accepts_only_bounded_events(
|
|||||||
)
|
)
|
||||||
assert fallback.failure_stage is None
|
assert fallback.failure_stage is None
|
||||||
|
|
||||||
|
camera_restart = LiveViewerDiagnosticEvent(
|
||||||
|
schema_version="missioncore.live-viewer-diagnostic/v2",
|
||||||
|
event_code="live_camera_transport_restart_requested",
|
||||||
|
ui_build_id=expected_build,
|
||||||
|
document_instance_id="00000000-0000-4000-8000-000000000001",
|
||||||
|
viewer_instance_id="00000000-0000-4000-8000-000000000003",
|
||||||
|
lifecycle_generation=2,
|
||||||
|
failure_stage="camera-queue-capacity",
|
||||||
|
stream_id="camera-preview-2",
|
||||||
|
camera_queue_bytes=12_000_000,
|
||||||
|
camera_queue_segments=96,
|
||||||
|
camera_retry_count=3,
|
||||||
|
websocket_close_code=4_008,
|
||||||
|
transport_epoch=7,
|
||||||
|
camera_append_error_name="InvalidStateError",
|
||||||
|
camera_media_source_state="open",
|
||||||
|
camera_video_error_code=3,
|
||||||
|
)
|
||||||
|
with caplog.at_level(
|
||||||
|
logging.INFO,
|
||||||
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
||||||
|
):
|
||||||
|
camera_response = endpoint(camera_restart)
|
||||||
|
assert camera_response.status_code == 204
|
||||||
|
assert caplog.records[-1].failure_stage == "camera-queue-capacity"
|
||||||
|
assert caplog.records[-1].camera_queue_bytes == 12_000_000
|
||||||
|
assert caplog.records[-1].camera_queue_segments == 96
|
||||||
|
assert caplog.records[-1].camera_retry_count == 3
|
||||||
|
assert caplog.records[-1].websocket_close_code == 4_008
|
||||||
|
assert caplog.records[-1].transport_epoch == 7
|
||||||
|
assert caplog.records[-1].camera_append_error_name == "InvalidStateError"
|
||||||
|
assert caplog.records[-1].camera_media_source_state == "open"
|
||||||
|
assert caplog.records[-1].camera_video_error_code == 3
|
||||||
|
|
||||||
|
|
||||||
def test_live_viewer_diagnostic_rejects_stale_build_before_logging(
|
def test_live_viewer_diagnostic_rejects_stale_build_before_logging(
|
||||||
caplog: pytest.LogCaptureFixture,
|
caplog: pytest.LogCaptureFixture,
|
||||||
@@ -212,7 +264,7 @@ def test_live_viewer_client_contract_is_no_store_and_exact_build() -> None:
|
|||||||
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build)
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build)
|
||||||
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
||||||
|
|
||||||
response = endpoint()
|
response = endpoint(_request(ui_build_id=expected_build))
|
||||||
|
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert response.headers["cache-control"] == "no-store"
|
assert response.headers["cache-control"] == "no-store"
|
||||||
@@ -225,11 +277,33 @@ def test_live_viewer_client_contract_is_no_store_and_exact_build() -> None:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_viewer_client_contract_logs_suppressed_stale_build_reload(
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
loaded_build = "/assets/index-abcdefgh.js"
|
||||||
|
expected_build = "/assets/index-ijklmnop.js"
|
||||||
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build)
|
||||||
|
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
||||||
|
|
||||||
|
with caplog.at_level(
|
||||||
|
logging.INFO,
|
||||||
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
||||||
|
):
|
||||||
|
response = endpoint(_request(ui_build_id=loaded_build))
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert caplog.records[-1].event_code == "ui_build_drift_reload_suppressed"
|
||||||
|
assert caplog.records[-1].ui_build_id == loaded_build
|
||||||
|
assert caplog.records[-1].expected_ui_build_id == expected_build
|
||||||
|
assert caplog.records[-1].device_write_performed is False
|
||||||
|
assert caplog.records[-1].automatic_retry is False
|
||||||
|
|
||||||
|
|
||||||
def test_live_viewer_client_contract_no_dist_is_retryable_without_reload_header() -> None:
|
def test_live_viewer_client_contract_no_dist_is_retryable_without_reload_header() -> None:
|
||||||
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: None)
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: None)
|
||||||
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
||||||
|
|
||||||
response = endpoint()
|
response = endpoint(_request())
|
||||||
|
|
||||||
assert response.status_code == 503
|
assert response.status_code == 503
|
||||||
assert response.headers["cache-control"] == "no-store"
|
assert response.headers["cache-control"] == "no-store"
|
||||||
|
|||||||
@@ -19833,6 +19833,8 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
|||||||
assert state["connection_mode"] == "quick-connect"
|
assert state["connection_mode"] == "quick-connect"
|
||||||
assert state["k1_ip"] == "192.168.56.1"
|
assert state["k1_ip"] == "192.168.56.1"
|
||||||
assert state["compatibility"]["attestation"]["topology"] == "device-ap"
|
assert state["compatibility"]["attestation"]["topology"] == "device-ap"
|
||||||
|
assert state["last_operation"]["result"]["host_wifi_association_performed"] is True
|
||||||
|
assert state["last_operation"]["result"]["host_wifi_association_outcome"] == "associated"
|
||||||
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*"))
|
||||||
assert len(quick_sessions) == 1
|
assert len(quick_sessions) == 1
|
||||||
assert not (quick_sessions[0] / "provisioning.sensitive.json").exists()
|
assert not (quick_sessions[0] / "provisioning.sensitive.json").exists()
|
||||||
@@ -19843,6 +19845,8 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host(
|
|||||||
assert '"credentials_resolved_by_plugin": true' in redacted_manifest
|
assert '"credentials_resolved_by_plugin": true' in redacted_manifest
|
||||||
assert "credential_provider_id" in redacted_manifest
|
assert "credential_provider_id" in redacted_manifest
|
||||||
assert "device_ap_activation_profile_id" in redacted_manifest
|
assert "device_ap_activation_profile_id" in redacted_manifest
|
||||||
|
assert '"host_wifi_association_performed": true' in redacted_manifest
|
||||||
|
assert '"host_wifi_association_outcome": "associated"' in redacted_manifest
|
||||||
assert (quick_sessions[0] / "ap-activation.redacted.json").exists()
|
assert (quick_sessions[0] / "ap-activation.redacted.json").exists()
|
||||||
assert (
|
assert (
|
||||||
service._camera_target_for_session( # noqa: SLF001
|
service._camera_target_for_session( # noqa: SLF001
|
||||||
@@ -29487,6 +29491,114 @@ _RECOVERY_ACQUISITION_ID = "acquisition-persisted-active-k1"
|
|||||||
_RECOVERY_START_OPERATION_ID = "physical-start-persisted-active-k1"
|
_RECOVERY_START_OPERATION_ID = "physical-start-persisted-active-k1"
|
||||||
|
|
||||||
|
|
||||||
|
def _persist_ambiguous_start_with_prepared_checkpoint(
|
||||||
|
service: XgridsK1CompatibilityService,
|
||||||
|
) -> None:
|
||||||
|
"""Persist the exact pre-crash shape: PREPARED token + ambiguous START."""
|
||||||
|
|
||||||
|
identity = PhysicalCommandIdentity(
|
||||||
|
vendor_device_id_sha256=_RECOVERY_VENDOR_HASH,
|
||||||
|
device_serial_sha256=_RECOVERY_SERIAL_HASH,
|
||||||
|
)
|
||||||
|
connection = PhysicalCommandConnectionBinding(
|
||||||
|
intent_id="ambiguous-start-intent",
|
||||||
|
transport_ref="test-ble-transport",
|
||||||
|
connection_mode="bridge",
|
||||||
|
target_ipv4="192.168.68.52",
|
||||||
|
target_port=facade_module.CONTROL_MQTT_PORT,
|
||||||
|
host_path_epoch=1,
|
||||||
|
control_session_id="ambiguous-start-control",
|
||||||
|
producer_generation=1,
|
||||||
|
)
|
||||||
|
observed_at_utc = datetime.now(UTC).isoformat(timespec="milliseconds").replace(
|
||||||
|
"+00:00",
|
||||||
|
"Z",
|
||||||
|
)
|
||||||
|
baseline = PhysicalCommandStatusEvidence(
|
||||||
|
source="live-control-session",
|
||||||
|
vendor_device_id_sha256=_RECOVERY_VENDOR_HASH,
|
||||||
|
device_serial_sha256=_RECOVERY_SERIAL_HASH,
|
||||||
|
control_session_id=connection.control_session_id,
|
||||||
|
host_path_epoch=connection.host_path_epoch,
|
||||||
|
producer_generation=connection.producer_generation,
|
||||||
|
session_state="ready",
|
||||||
|
session_state_code=300,
|
||||||
|
project_bound=False,
|
||||||
|
project_id_sha256=None,
|
||||||
|
init_ready=False,
|
||||||
|
status_message_sha256="1" * 64,
|
||||||
|
mqtt_retained=False,
|
||||||
|
observed_at_utc=observed_at_utc,
|
||||||
|
)
|
||||||
|
operation_id = "physical-start-persisted-ambiguous-k1"
|
||||||
|
acquisition_id = "acquisition-persisted-ambiguous-k1"
|
||||||
|
payload_sha256 = "2" * 64
|
||||||
|
ledger = service._physical_command_ledger # noqa: SLF001
|
||||||
|
ledger.prepare(
|
||||||
|
operation_id=operation_id,
|
||||||
|
parent_operation_id=None,
|
||||||
|
acquisition_id=acquisition_id,
|
||||||
|
action="start",
|
||||||
|
identity=identity,
|
||||||
|
connection=connection,
|
||||||
|
compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID,
|
||||||
|
payload_sha256=payload_sha256,
|
||||||
|
baseline_status=baseline,
|
||||||
|
)
|
||||||
|
store = service._active_acquisition_checkpoint # noqa: SLF001
|
||||||
|
assert store is not None
|
||||||
|
store.prepare(
|
||||||
|
transition_id="prepare-persisted-ambiguous-start",
|
||||||
|
predecessor_revision=0,
|
||||||
|
acquisition_id=acquisition_id,
|
||||||
|
original_start_operation_id=operation_id,
|
||||||
|
start_payload_sha256=payload_sha256,
|
||||||
|
identity=ActiveAcquisitionRecoveryIdentity(
|
||||||
|
logical_device_id="known-k1",
|
||||||
|
vendor_device_id_sha256=_RECOVERY_VENDOR_HASH,
|
||||||
|
device_serial_sha256=_RECOVERY_SERIAL_HASH,
|
||||||
|
),
|
||||||
|
connection=ActiveAcquisitionRecoveryConnection(
|
||||||
|
transport_ref=connection.transport_ref,
|
||||||
|
connection_mode=connection.connection_mode,
|
||||||
|
target_ipv4=connection.target_ipv4,
|
||||||
|
target_port=connection.target_port,
|
||||||
|
),
|
||||||
|
compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID,
|
||||||
|
project_name="AMBIGUOUS_RECOVERY",
|
||||||
|
project_name_wire_sha256=active_acquisition_project_name_sha256(
|
||||||
|
"AMBIGUOUS_RECOVERY"
|
||||||
|
),
|
||||||
|
original_evidence_session_id="evidence-before-crash",
|
||||||
|
duration_seconds=None,
|
||||||
|
requested_streams=("spatial.point-cloud.live", "camera.rgb.live"),
|
||||||
|
evidence_policy="required",
|
||||||
|
mount_type="handheld",
|
||||||
|
gnss_mode="none",
|
||||||
|
prepared_binding=ActiveAcquisitionRecoveryTransportBinding(
|
||||||
|
runtime_instance_id="runtime-before-crash",
|
||||||
|
intent_id=connection.intent_id,
|
||||||
|
transport_ref=connection.transport_ref,
|
||||||
|
connection_mode=connection.connection_mode,
|
||||||
|
target_ipv4=connection.target_ipv4,
|
||||||
|
target_port=connection.target_port,
|
||||||
|
host_path_epoch=connection.host_path_epoch,
|
||||||
|
control_session_id=connection.control_session_id,
|
||||||
|
producer_generation=connection.producer_generation,
|
||||||
|
logical_device_id="known-k1",
|
||||||
|
compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID,
|
||||||
|
vendor_device_id_sha256=_RECOVERY_VENDOR_HASH,
|
||||||
|
device_serial_sha256=_RECOVERY_SERIAL_HASH,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
ledger.mark_dispatching(operation_id)
|
||||||
|
ledger.mark_observing(
|
||||||
|
operation_id,
|
||||||
|
publish_call_returned=True,
|
||||||
|
packet_id=41,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _persist_resolved_active_start_for_restart(
|
def _persist_resolved_active_start_for_restart(
|
||||||
service: XgridsK1CompatibilityService,
|
service: XgridsK1CompatibilityService,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -30448,6 +30560,54 @@ def test_explicit_verify_reconciles_scanning_as_active_without_unblocking_mode_c
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_verify_scanning_checkpoint_rehydrate_failure_keeps_stop_only(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""A rejected local checkpoint cannot erase proven physical STOP authority."""
|
||||||
|
|
||||||
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
|
coordinator = _VerifyPhysicalRecoveryCoordinator(
|
||||||
|
observed_session_state="scanning",
|
||||||
|
reconciliation_ready=True,
|
||||||
|
)
|
||||||
|
_install_synthetic_verify_recovery(service, coordinator)
|
||||||
|
rehydrate_calls: list[str] = []
|
||||||
|
|
||||||
|
async def reject_receiver_rehydration(**_: object) -> None:
|
||||||
|
rehydrate_calls.append("rejected")
|
||||||
|
raise facade_module.ActiveAcquisitionRecoveryCheckpointError(
|
||||||
|
"synthetic stale prepared checkpoint"
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service,
|
||||||
|
"_rehydrate_active_acquisition_after_restart",
|
||||||
|
reject_receiver_rehydration,
|
||||||
|
)
|
||||||
|
operation_id = "op-00000000-0000-4000-8000-000000001217"
|
||||||
|
|
||||||
|
state = asyncio.run(
|
||||||
|
service.verify_connection(
|
||||||
|
_retained_physical_recovery_verify_request(operation_id=operation_id)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert rehydrate_calls == ["rejected"]
|
||||||
|
assert state["last_operation"]["status"] == "succeeded"
|
||||||
|
assert state["physical_command"]["record"]["resolution"] == (
|
||||||
|
"physical-active-observed"
|
||||||
|
)
|
||||||
|
assert state["acquisition"]["acquisition_id"] == "acq-persisted-start"
|
||||||
|
assert state["acquisition"]["state"] == "failed"
|
||||||
|
assert state["acquisition"]["result"]["recovery_only"] is True
|
||||||
|
assert state["application_control_session"]["state"] == "scanning"
|
||||||
|
assert state["application_control_session"]["can_stop"] is True
|
||||||
|
assert state["connection_policy"]["actions"]["stop-acquisition"]["allowed"] is True
|
||||||
|
assert runtime.start_calls == []
|
||||||
|
assert runtime.stop_calls == 0
|
||||||
|
|
||||||
|
|
||||||
def test_explicit_verify_scanning_cleans_terminal_camera_residual_for_stop_only_shell(
|
def test_explicit_verify_scanning_cleans_terminal_camera_residual_for_stop_only_shell(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -30934,6 +31094,113 @@ def test_active_reconciliation_survives_post_commit_adoption_failure_and_next_re
|
|||||||
]["mode_selection"]
|
]["mode_selection"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepared_ambiguous_start_active_then_ready_settles_checkpoint_without_command(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""Fresh SCANNING then READY closes the pre-crash token without replay."""
|
||||||
|
|
||||||
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
|
_persist_ambiguous_start_with_prepared_checkpoint(service)
|
||||||
|
coordinator = service._physical_command_coordinator # noqa: SLF001
|
||||||
|
observed_base = datetime.now(UTC) + timedelta(seconds=1)
|
||||||
|
|
||||||
|
def bind_fresh_status(
|
||||||
|
*,
|
||||||
|
suffix: str,
|
||||||
|
generation: int,
|
||||||
|
session_state: str,
|
||||||
|
) -> None:
|
||||||
|
observed_at_utc = (
|
||||||
|
observed_base + timedelta(seconds=generation)
|
||||||
|
).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||||
|
coordinator.application_response(
|
||||||
|
ApplicationMqttResponseEvidence(
|
||||||
|
operation_key=f"bootstrap:{suffix}:DeviceInfoRequest",
|
||||||
|
response_topic="lixel/application/response/device_info",
|
||||||
|
payload_sha256=hashlib.sha256(
|
||||||
|
f"device-info:{suffix}".encode()
|
||||||
|
).hexdigest(),
|
||||||
|
modeling_action=None,
|
||||||
|
result_code=None,
|
||||||
|
success=None,
|
||||||
|
observed_at_utc=observed_at_utc,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
coordinator.bind_control_session(
|
||||||
|
PhysicalCommandRuntimeBinding(
|
||||||
|
vendor_device_id_sha256=_RECOVERY_VENDOR_HASH,
|
||||||
|
device_serial_sha256=_RECOVERY_SERIAL_HASH,
|
||||||
|
compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID,
|
||||||
|
intent_id=f"recovery-{suffix}-intent",
|
||||||
|
transport_ref="test-ble-transport",
|
||||||
|
connection_mode="bridge",
|
||||||
|
target_ipv4="192.168.68.52",
|
||||||
|
target_port=facade_module.CONTROL_MQTT_PORT,
|
||||||
|
host_path_epoch=generation,
|
||||||
|
control_session_id=f"recovery-{suffix}-control",
|
||||||
|
producer_generation=generation,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
scanning = session_state == "scanning"
|
||||||
|
coordinator.device_status(
|
||||||
|
ApplicationMqttDeviceStatusEvidence(
|
||||||
|
vendor_device_id_sha256=_RECOVERY_VENDOR_HASH,
|
||||||
|
device_serial_sha256=_RECOVERY_SERIAL_HASH,
|
||||||
|
session_state=session_state, # type: ignore[arg-type]
|
||||||
|
session_state_code=MODELING_STATE_BASE + (302 if scanning else 300),
|
||||||
|
project_bound=scanning,
|
||||||
|
project_id_sha256=_RECOVERY_PROJECT_HASH if scanning else None,
|
||||||
|
init_ready=scanning,
|
||||||
|
status_message_sha256=hashlib.sha256(
|
||||||
|
f"status:{suffix}:{session_state}".encode()
|
||||||
|
).hexdigest(),
|
||||||
|
mqtt_retained=False,
|
||||||
|
observed_at_utc=observed_at_utc,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
bind_fresh_status(suffix="active", generation=2, session_state="scanning")
|
||||||
|
active_record = coordinator.reconcile_unresolved(
|
||||||
|
reconciliation_id="reconcile-ambiguous-active"
|
||||||
|
)
|
||||||
|
assert active_record["resolution"] == "physical-active-observed"
|
||||||
|
token = service._validate_active_acquisition_checkpoint_lineage() # noqa: SLF001
|
||||||
|
assert token is not None and token.checkpoint_state == "prepared"
|
||||||
|
|
||||||
|
bind_fresh_status(suffix="ready", generation=3, session_state="ready")
|
||||||
|
standby_record = coordinator.reconcile_resolved_active(
|
||||||
|
reconciliation_id="reconcile-ambiguous-ready"
|
||||||
|
)
|
||||||
|
assert standby_record["resolution"] == "physical-standby-observed"
|
||||||
|
assert standby_record["reconciliations"][-1]["kind"] == (
|
||||||
|
"resolved-active-cessation"
|
||||||
|
)
|
||||||
|
|
||||||
|
settled = service._settle_restart_checkpoint_after_verified_standby( # noqa: SLF001
|
||||||
|
token=token,
|
||||||
|
reconciliation_id="reconcile-ambiguous-ready",
|
||||||
|
reconciled_record=standby_record,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert settled is True
|
||||||
|
store = service._active_acquisition_checkpoint # noqa: SLF001
|
||||||
|
assert store is not None
|
||||||
|
checkpoint = store.snapshot().checkpoint
|
||||||
|
assert checkpoint is not None and checkpoint.state == "ceased"
|
||||||
|
assert checkpoint.active_project_id_sha256 == _RECOVERY_PROJECT_HASH
|
||||||
|
assert checkpoint.activated_at_utc is None
|
||||||
|
assert checkpoint.first_published_pcl_proof is None
|
||||||
|
assert checkpoint.reconciled_start_origin_proof is not None
|
||||||
|
assert checkpoint.reconciled_start_origin_proof.origin_kind == (
|
||||||
|
"ambiguous-reconciled"
|
||||||
|
)
|
||||||
|
assert checkpoint.reconciled_start_origin_proof.reconciled_active_project_id_sha256 == (
|
||||||
|
_RECOVERY_PROJECT_HASH
|
||||||
|
)
|
||||||
|
assert runtime.start_calls == []
|
||||||
|
assert runtime.stop_calls == 0
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("proof_kind", "observed_session_state"),
|
("proof_kind", "observed_session_state"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -126,12 +126,14 @@ def _connection(
|
|||||||
control_session_id: str,
|
control_session_id: str,
|
||||||
host_path_epoch: int,
|
host_path_epoch: int,
|
||||||
producer_generation: int,
|
producer_generation: int,
|
||||||
|
connection_mode: str = "bridge",
|
||||||
|
target_ipv4: str = "192.168.68.52",
|
||||||
) -> PhysicalCommandConnectionBinding:
|
) -> PhysicalCommandConnectionBinding:
|
||||||
return PhysicalCommandConnectionBinding(
|
return PhysicalCommandConnectionBinding(
|
||||||
intent_id="intent-checkpoint-integration",
|
intent_id="intent-checkpoint-integration",
|
||||||
transport_ref="transport-checkpoint-integration",
|
transport_ref="transport-checkpoint-integration",
|
||||||
connection_mode="bridge",
|
connection_mode=connection_mode, # type: ignore[arg-type]
|
||||||
target_ipv4="192.168.68.52",
|
target_ipv4=target_ipv4,
|
||||||
target_port=1883,
|
target_port=1883,
|
||||||
host_path_epoch=host_path_epoch,
|
host_path_epoch=host_path_epoch,
|
||||||
control_session_id=control_session_id,
|
control_session_id=control_session_id,
|
||||||
@@ -460,10 +462,22 @@ def test_repeated_reset_reopen_fresh_ready_ceases_old_active_checkpoint(
|
|||||||
(False, True),
|
(False, True),
|
||||||
ids=("process-restart", "same-process-new-control-epoch"),
|
ids=("process-restart", "same-process-new-control-epoch"),
|
||||||
)
|
)
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"acknowledged_stop",
|
||||||
|
(True, False),
|
||||||
|
ids=("acknowledged-stop", "ambiguous-stop"),
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"cross_mode",
|
||||||
|
(False, True),
|
||||||
|
ids=("same-mode", "quick-to-bridge"),
|
||||||
|
)
|
||||||
def test_ready_after_dispatched_stop_ceases_active_checkpoint(
|
def test_ready_after_dispatched_stop_ceases_active_checkpoint(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
same_process: bool,
|
same_process: bool,
|
||||||
|
acknowledged_stop: bool,
|
||||||
|
cross_mode: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Fresh READY settles a dispatched STOP across either recovery boundary."""
|
"""Fresh READY settles a dispatched STOP across either recovery boundary."""
|
||||||
|
|
||||||
@@ -472,6 +486,8 @@ def test_ready_after_dispatched_stop_ceases_active_checkpoint(
|
|||||||
control_session_id="checkpoint-dispatched-stop-original-control",
|
control_session_id="checkpoint-dispatched-stop-original-control",
|
||||||
host_path_epoch=1,
|
host_path_epoch=1,
|
||||||
producer_generation=1,
|
producer_generation=1,
|
||||||
|
connection_mode="quick-connect" if cross_mode else "bridge",
|
||||||
|
target_ipv4="192.168.56.1" if cross_mode else "192.168.68.52",
|
||||||
)
|
)
|
||||||
_prepare_and_activate_checkpoint(service, original)
|
_prepare_and_activate_checkpoint(service, original)
|
||||||
ledger = service._physical_command_ledger # noqa: SLF001
|
ledger = service._physical_command_ledger # noqa: SLF001
|
||||||
@@ -500,21 +516,22 @@ def test_ready_after_dispatched_stop_ceases_active_checkpoint(
|
|||||||
publish_call_returned=True,
|
publish_call_returned=True,
|
||||||
packet_id=42,
|
packet_id=42,
|
||||||
)
|
)
|
||||||
ledger.mark_qos2_completed(FIRST_STOP_OPERATION_ID, packet_id=42)
|
if acknowledged_stop:
|
||||||
ledger.record_application_response(
|
ledger.mark_qos2_completed(FIRST_STOP_OPERATION_ID, packet_id=42)
|
||||||
FIRST_STOP_OPERATION_ID,
|
ledger.record_application_response(
|
||||||
PhysicalCommandApplicationResponse(
|
FIRST_STOP_OPERATION_ID,
|
||||||
operation_id=FIRST_STOP_OPERATION_ID,
|
PhysicalCommandApplicationResponse(
|
||||||
action="stop",
|
operation_id=FIRST_STOP_OPERATION_ID,
|
||||||
control_session_id=original.control_session_id,
|
action="stop",
|
||||||
host_path_epoch=original.host_path_epoch,
|
control_session_id=original.control_session_id,
|
||||||
producer_generation=original.producer_generation,
|
host_path_epoch=original.host_path_epoch,
|
||||||
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
|
producer_generation=original.producer_generation,
|
||||||
success=True,
|
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
|
||||||
payload_sha256="3" * 64,
|
success=True,
|
||||||
observed_at_utc="2026-08-13T12:01:01.000Z",
|
payload_sha256="3" * 64,
|
||||||
),
|
observed_at_utc="2026-08-13T12:01:01.000Z",
|
||||||
)
|
),
|
||||||
|
)
|
||||||
if not same_process:
|
if not same_process:
|
||||||
service._snapshot_runtime_id = ( # noqa: SLF001
|
service._snapshot_runtime_id = ( # noqa: SLF001
|
||||||
"snapshot-runtime-checkpoint-dispatched-stop-successor"
|
"snapshot-runtime-checkpoint-dispatched-stop-successor"
|
||||||
@@ -527,6 +544,8 @@ def test_ready_after_dispatched_stop_ceases_active_checkpoint(
|
|||||||
# behind an ACTIVE checkpoint, as observed in the live Bridge flow.
|
# behind an ACTIVE checkpoint, as observed in the live Bridge flow.
|
||||||
host_path_epoch=(original.host_path_epoch if same_process else 2),
|
host_path_epoch=(original.host_path_epoch if same_process else 2),
|
||||||
producer_generation=2,
|
producer_generation=2,
|
||||||
|
connection_mode="bridge",
|
||||||
|
target_ipv4="192.168.68.52",
|
||||||
)
|
)
|
||||||
coordinator = service._physical_command_coordinator # noqa: SLF001
|
coordinator = service._physical_command_coordinator # noqa: SLF001
|
||||||
coordinator.application_response(
|
coordinator.application_response(
|
||||||
|
|||||||
@@ -537,6 +537,7 @@ def _cease_prepared_resolved_start_standby_kwargs(
|
|||||||
prepared_binding: ActiveAcquisitionRecoveryTransportBinding,
|
prepared_binding: ActiveAcquisitionRecoveryTransportBinding,
|
||||||
*,
|
*,
|
||||||
terminal_state: str,
|
terminal_state: str,
|
||||||
|
origin_kind: str = "composite-resolved",
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
binding = _reconciled_binding()
|
binding = _reconciled_binding()
|
||||||
status = _status(
|
status = _status(
|
||||||
@@ -545,16 +546,21 @@ def _cease_prepared_resolved_start_standby_kwargs(
|
|||||||
evidence_session_id="evidence-restarted",
|
evidence_session_id="evidence-restarted",
|
||||||
observed_at="2026-08-13T12:00:03.000Z",
|
observed_at="2026-08-13T12:00:03.000Z",
|
||||||
)
|
)
|
||||||
|
composite = origin_kind == "composite-resolved"
|
||||||
physical = ActiveAcquisitionRecoveryPhysicalLineageProof(
|
physical = ActiveAcquisitionRecoveryPhysicalLineageProof(
|
||||||
ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA,
|
ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA,
|
||||||
ledger_revision=21,
|
ledger_revision=21,
|
||||||
proof_id="physical-composite-resolved-standby",
|
proof_id=f"physical-{origin_kind}-standby",
|
||||||
operation_id=START_OPERATION_ID,
|
operation_id=START_OPERATION_ID,
|
||||||
original_start_operation_id=START_OPERATION_ID,
|
original_start_operation_id=START_OPERATION_ID,
|
||||||
parent_operation_id=None,
|
parent_operation_id=None,
|
||||||
acquisition_id=ACQUISITION_ID,
|
acquisition_id=ACQUISITION_ID,
|
||||||
action="start",
|
action="start",
|
||||||
resolution="start-active-observed",
|
resolution=(
|
||||||
|
"start-active-observed"
|
||||||
|
if composite
|
||||||
|
else "physical-standby-observed"
|
||||||
|
),
|
||||||
payload_sha256=START_PAYLOAD_SHA256,
|
payload_sha256=START_PAYLOAD_SHA256,
|
||||||
original_start_payload_sha256=START_PAYLOAD_SHA256,
|
original_start_payload_sha256=START_PAYLOAD_SHA256,
|
||||||
reconciliation_kind="resolved-active-cessation",
|
reconciliation_kind="resolved-active-cessation",
|
||||||
@@ -562,18 +568,22 @@ def _cease_prepared_resolved_start_standby_kwargs(
|
|||||||
status_message_sha256=status.status_message_sha256,
|
status_message_sha256=status.status_message_sha256,
|
||||||
observed_session_state=status.session_state,
|
observed_session_state=status.session_state,
|
||||||
binding=binding,
|
binding=binding,
|
||||||
composite_complete=True,
|
composite_complete=composite,
|
||||||
edge_terminal=True,
|
edge_terminal=True,
|
||||||
late_start_excluded=True,
|
late_start_excluded=True,
|
||||||
stop_fence="none",
|
stop_fence="none",
|
||||||
observed_at_utc=status.observed_at_utc,
|
observed_at_utc=status.observed_at_utc,
|
||||||
)
|
)
|
||||||
origin = _origin(
|
origin = _origin(
|
||||||
origin_kind="composite-resolved",
|
origin_kind=origin_kind,
|
||||||
prepared_binding=prepared_binding,
|
prepared_binding=prepared_binding,
|
||||||
physical_proof=physical,
|
physical_proof=physical,
|
||||||
)
|
)
|
||||||
assert origin.original_project_id_sha256 is not None
|
settlement_project_id_sha256 = (
|
||||||
|
origin.original_project_id_sha256
|
||||||
|
or origin.reconciled_active_project_id_sha256
|
||||||
|
)
|
||||||
|
assert settlement_project_id_sha256 is not None
|
||||||
return {
|
return {
|
||||||
"transition_id": (
|
"transition_id": (
|
||||||
f"transition-cease-prepared-resolved-start-{terminal_state}"
|
f"transition-cease-prepared-resolved-start-{terminal_state}"
|
||||||
@@ -590,7 +600,7 @@ def _cease_prepared_resolved_start_standby_kwargs(
|
|||||||
origin.original_attempt_sha256
|
origin.original_attempt_sha256
|
||||||
),
|
),
|
||||||
"reconciliation_original_project_id_sha256": (
|
"reconciliation_original_project_id_sha256": (
|
||||||
origin.original_project_id_sha256
|
settlement_project_id_sha256
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -981,6 +991,52 @@ def test_cease_prepared_resolved_start_standby_is_terminal_without_activation(
|
|||||||
assert restarted.snapshot().checkpoint == ceased
|
assert restarted.snapshot().checkpoint == ceased
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("terminal_state", ("ready", "scan_over"))
|
||||||
|
def test_cease_prepared_ambiguous_active_start_standby_is_terminal_without_activation(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
terminal_state: str,
|
||||||
|
) -> None:
|
||||||
|
"""Two read-only proofs close an ambiguous START without inventing STOP/PCL."""
|
||||||
|
|
||||||
|
prepared_binding = _binding()
|
||||||
|
store = _store(tmp_path, monkeypatch)
|
||||||
|
prepared = _prepare(store, prepared_binding)
|
||||||
|
kwargs = _cease_prepared_resolved_start_standby_kwargs(
|
||||||
|
prepared_binding,
|
||||||
|
terminal_state=terminal_state,
|
||||||
|
origin_kind="ambiguous-reconciled",
|
||||||
|
)
|
||||||
|
|
||||||
|
ceased = store.cease_prepared_resolved_start_standby(**kwargs)
|
||||||
|
|
||||||
|
assert ceased.state == "ceased"
|
||||||
|
assert ceased.revision == prepared.revision + 1
|
||||||
|
assert ceased.active_project_id_sha256 == ACTIVE_PROJECT_ID_SHA256
|
||||||
|
assert ceased.activated_at_utc is None
|
||||||
|
assert ceased.activation_status_proof is None
|
||||||
|
assert ceased.activation_physical_proof is None
|
||||||
|
assert ceased.current_active_status_proof is None
|
||||||
|
assert ceased.current_active_physical_proof is None
|
||||||
|
assert ceased.first_published_pcl_proof is None
|
||||||
|
assert ceased.reconciled_start_origin_proof is not None
|
||||||
|
assert ceased.reconciled_start_origin_proof.origin_kind == (
|
||||||
|
"ambiguous-reconciled"
|
||||||
|
)
|
||||||
|
assert ceased.reconciled_start_origin_proof.original_project_id_sha256 is None
|
||||||
|
assert ceased.reconciled_start_origin_proof.reconciled_active_project_id_sha256 == (
|
||||||
|
ACTIVE_PROJECT_ID_SHA256
|
||||||
|
)
|
||||||
|
assert ceased.cessation_status_proof == kwargs["cessation_status_proof"]
|
||||||
|
assert ceased.cessation_physical_proof == kwargs["cessation_physical_proof"]
|
||||||
|
assert (
|
||||||
|
ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository")
|
||||||
|
.snapshot()
|
||||||
|
.checkpoint
|
||||||
|
== ceased
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_cease_prepared_resolved_start_standby_rejects_inexact_restart_proofs(
|
def test_cease_prepared_resolved_start_standby_rejects_inexact_restart_proofs(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|||||||
@@ -801,6 +801,16 @@ def test_preview_queue_enforces_exact_fragment_byte_and_age_bounds() -> None:
|
|||||||
assert age_bounded.offer(("media", b"too-old")) is False
|
assert age_bounded.offer(("media", b"too-old")) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_preview_queue_survives_measured_ui_pause() -> None:
|
||||||
|
now = [100.0]
|
||||||
|
preview_queue = _CameraPreviewSegmentQueue(clock=lambda: now[0])
|
||||||
|
|
||||||
|
assert preview_queue.offer(("media", b"first")) is True
|
||||||
|
now[0] += 3.2
|
||||||
|
|
||||||
|
assert preview_queue.offer(("media", b"next")) is True
|
||||||
|
|
||||||
|
|
||||||
def test_camera_router_closes_lagging_preview_with_explicit_private_code(
|
def test_camera_router_closes_lagging_preview_with_explicit_private_code(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|||||||
@@ -470,6 +470,63 @@ def test_reset_attempt_cutoff_uses_operation_identity_not_local_stage_sequence(
|
|||||||
assert projected["status"] == "failed"
|
assert projected["status"] == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_quick_connect_unknown_write_can_reset_to_bridge_without_device_restart(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service = _service(monkeypatch, tmp_path)
|
||||||
|
operation_id = "op-00000000-0000-4000-8000-000000000103"
|
||||||
|
journal = service._require_network_provisioning_idempotency_journal() # noqa: SLF001
|
||||||
|
admitted = journal.begin(
|
||||||
|
idempotency_key="network-provision:quick-to-bridge-reset",
|
||||||
|
action=facade_module.ACTION_NETWORK_PROVISION,
|
||||||
|
operation_id=operation_id,
|
||||||
|
request_binding_sha256=hashlib.sha256(b"quick-to-bridge-reset").hexdigest(),
|
||||||
|
)
|
||||||
|
journal.mark_unresolved(
|
||||||
|
operation_id,
|
||||||
|
expected_revision=admitted.record.revision,
|
||||||
|
)
|
||||||
|
prepared = service._network_mutation_ledger.prepare( # noqa: SLF001
|
||||||
|
operation_id=operation_id,
|
||||||
|
transport_ref="RESET-QUICK-K1-UUID",
|
||||||
|
intended_mode="quick-connect",
|
||||||
|
write_mode="with_response",
|
||||||
|
baseline_status=NetworkStatusEvidence(
|
||||||
|
mode="WIFI_AP",
|
||||||
|
ipv4="192.168.56.1",
|
||||||
|
status_code=1,
|
||||||
|
reserved=0,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
service._network_mutation_ledger.mark_dispatching( # noqa: SLF001
|
||||||
|
operation_id,
|
||||||
|
expected_revision=prepared.revision,
|
||||||
|
)
|
||||||
|
|
||||||
|
reset = service.select_connection_mode(
|
||||||
|
_reset(
|
||||||
|
mode="bridge",
|
||||||
|
revision=0,
|
||||||
|
reset_id="op-reset-quick-to-bridge-without-device-restart-01",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reset["desired_connection_mode"] == "bridge"
|
||||||
|
assert reset["desired_connection_mode_revision"] == 1
|
||||||
|
assert reset["connection_scenario_reset"]["network_write_performed"] is False
|
||||||
|
network_record = service._network_mutation_ledger.snapshot().record # noqa: SLF001
|
||||||
|
assert network_record is not None
|
||||||
|
assert network_record.resolution == "superseded"
|
||||||
|
terminal_record = journal.snapshot().records[-1]
|
||||||
|
assert terminal_record.operation_id == operation_id
|
||||||
|
assert terminal_record.stage == "terminal"
|
||||||
|
assert terminal_record.terminal is not None
|
||||||
|
assert terminal_record.terminal.outcome == "cancelled"
|
||||||
|
assert terminal_record.terminal.side_effect_status == "unknown"
|
||||||
|
assert terminal_record.terminal.safe_to_retry is False
|
||||||
|
|
||||||
|
|
||||||
def test_scenario_reset_preserves_physical_audit_and_unrelated_plugin_state(
|
def test_scenario_reset_preserves_physical_audit_and_unrelated_plugin_state(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
|
|||||||
Reference in New Issue
Block a user