fix(k1): stabilize live recovery and media admission
This commit is contained in:
@@ -41,7 +41,9 @@ export interface CameraStartupWatchdog {
|
|||||||
* Supervise only the disposable browser transport. One fixed first-media
|
* Supervise only the disposable browser transport. One fixed first-media
|
||||||
* deadline covers both an MSE that never opens and an open-but-silent socket.
|
* deadline covers both an MSE that never opens and an open-but-silent socket.
|
||||||
* The first non-empty fragment then starts a separate first-playable-frame
|
* The first non-empty fragment then starts a separate first-playable-frame
|
||||||
* deadline; later fragments deliberately do not extend it.
|
* deadline. Every subsequent fragment extends that deadline: a browser that is
|
||||||
|
* still receiving the authoritative stream must not destroy its partial MSE
|
||||||
|
* decode solely because the main thread or decoder needed longer to start.
|
||||||
*/
|
*/
|
||||||
export function createCameraStartupWatchdog({
|
export function createCameraStartupWatchdog({
|
||||||
schedule,
|
schedule,
|
||||||
@@ -83,7 +85,7 @@ export function createCameraStartupWatchdog({
|
|||||||
arm("first-media", firstMediaTimeoutMs);
|
arm("first-media", firstMediaTimeoutMs);
|
||||||
},
|
},
|
||||||
markMediaReceived() {
|
markMediaReceived() {
|
||||||
if (mediaReceived || playing) return;
|
if (playing) return;
|
||||||
mediaReceived = true;
|
mediaReceived = true;
|
||||||
arm("first-playable-frame", firstPlayableFrameTimeoutMs);
|
arm("first-playable-frame", firstPlayableFrameTimeoutMs);
|
||||||
},
|
},
|
||||||
@@ -189,6 +191,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id));
|
const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id));
|
||||||
const activeAuthorityRef = useRef(recoveryAuthorityIdentity);
|
const activeAuthorityRef = useRef(recoveryAuthorityIdentity);
|
||||||
const recoveryPendingAuthorityRef = useRef<string | null>(null);
|
const recoveryPendingAuthorityRef = useRef<string | null>(null);
|
||||||
|
const transportHealthyRef = useRef(false);
|
||||||
const transportEpochRef = useRef(0);
|
const transportEpochRef = useRef(0);
|
||||||
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
|
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
|
||||||
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
|
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
|
||||||
@@ -243,6 +246,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
let receivedMedia = false;
|
let receivedMedia = false;
|
||||||
let failed = false;
|
let failed = false;
|
||||||
let startupWatchdog: CameraStartupWatchdog | null = null;
|
let startupWatchdog: CameraStartupWatchdog | null = null;
|
||||||
|
transportHealthyRef.current = false;
|
||||||
const recovering = Boolean(
|
const recovering = Boolean(
|
||||||
activeAuthorityRef.current
|
activeAuthorityRef.current
|
||||||
&& recoveryPendingAuthorityRef.current === activeAuthorityRef.current,
|
&& recoveryPendingAuthorityRef.current === activeAuthorityRef.current,
|
||||||
@@ -254,6 +258,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
if (!transportIsCurrent() || failed) return;
|
if (!transportIsCurrent() || failed) return;
|
||||||
startupWatchdog?.clear();
|
startupWatchdog?.clear();
|
||||||
failed = true;
|
failed = true;
|
||||||
|
transportHealthyRef.current = false;
|
||||||
recoveryPendingAuthorityRef.current = null;
|
recoveryPendingAuthorityRef.current = null;
|
||||||
setStatus("error");
|
setStatus("error");
|
||||||
setMessage(copy);
|
setMessage(copy);
|
||||||
@@ -280,6 +285,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
}
|
}
|
||||||
startupWatchdog?.clear();
|
startupWatchdog?.clear();
|
||||||
failed = true;
|
failed = true;
|
||||||
|
transportHealthyRef.current = false;
|
||||||
queue.length = 0;
|
queue.length = 0;
|
||||||
queuedBytes = 0;
|
queuedBytes = 0;
|
||||||
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
|
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
|
||||||
@@ -320,6 +326,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
const onPlaying = () => {
|
const onPlaying = () => {
|
||||||
if (!transportIsCurrent() || failed) return;
|
if (!transportIsCurrent() || failed) return;
|
||||||
startupWatchdog?.markPlaying();
|
startupWatchdog?.markPlaying();
|
||||||
|
transportHealthyRef.current = true;
|
||||||
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
|
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
|
||||||
recoveryPendingAuthorityRef.current = null;
|
recoveryPendingAuthorityRef.current = null;
|
||||||
setStatus("playing");
|
setStatus("playing");
|
||||||
@@ -464,6 +471,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
const disposeTransport = () => {
|
const disposeTransport = () => {
|
||||||
if (disposed) return;
|
if (disposed) return;
|
||||||
disposed = true;
|
disposed = true;
|
||||||
|
transportHealthyRef.current = false;
|
||||||
startupWatchdog?.clear();
|
startupWatchdog?.clear();
|
||||||
if (retryTimer !== undefined) window.clearTimeout(retryTimer);
|
if (retryTimer !== undefined) window.clearTimeout(retryTimer);
|
||||||
queue.length = 0;
|
queue.length = 0;
|
||||||
@@ -518,6 +526,7 @@ export function MseFmp4WebSocketPlayer({
|
|||||||
now: Date.now(),
|
now: Date.now(),
|
||||||
documentVisible: document.visibilityState === "visible",
|
documentVisible: document.visibilityState === "visible",
|
||||||
networkOnline: navigator.onLine !== false,
|
networkOnline: navigator.onLine !== false,
|
||||||
|
transportHealthy: transportHealthyRef.current,
|
||||||
});
|
});
|
||||||
recovery = decision.state;
|
recovery = decision.state;
|
||||||
if (!decision.reopen) return;
|
if (!decision.reopen) return;
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import {
|
|||||||
initialLiveReceiverRecoveryState,
|
initialLiveReceiverRecoveryState,
|
||||||
initialLiveReceiverWatchdogState,
|
initialLiveReceiverWatchdogState,
|
||||||
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||||
LIVE_RECEIVER_OPEN_MAX_AGE_MS,
|
LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS,
|
||||||
requestLiveReceiverRecovery,
|
requestLiveReceiverRecovery,
|
||||||
} from "../core/observation/liveReceiverWatchdog";
|
} from "../core/observation/liveReceiverWatchdog";
|
||||||
import {
|
import {
|
||||||
@@ -1091,25 +1091,6 @@ export function RerunViewport({
|
|||||||
};
|
};
|
||||||
const clearRecordedAdmissionWatchdog = () => recordedOpenWatchdog?.clear();
|
const clearRecordedAdmissionWatchdog = () => recordedOpenWatchdog?.clear();
|
||||||
const clearLiveRecordingOpenTimer = diagnosticLifecycle.clearAdmissionTimeout;
|
const clearLiveRecordingOpenTimer = diagnosticLifecycle.clearAdmissionTimeout;
|
||||||
function refreshOpeningLiveReceiver(openForMs: number) {
|
|
||||||
if (disposed || recordingOpened) return;
|
|
||||||
diagnosticLifecycle.post({
|
|
||||||
eventCode: "live_receiver_restart_requested",
|
|
||||||
failureStage: "recording-open-timeout",
|
|
||||||
streamId: liveStreamIdRef.current,
|
|
||||||
backendActivitySequence: liveActivitySequenceRef.current,
|
|
||||||
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
|
||||||
stalledForMs: Math.round(openForMs),
|
|
||||||
recoveryAttempt: liveRecoveryRef.current.attempts || null,
|
|
||||||
});
|
|
||||||
setStatus("loading");
|
|
||||||
onStatusChange?.(
|
|
||||||
"loading",
|
|
||||||
"Живой визуализатор обновляет приёмник продолжающегося потока.",
|
|
||||||
);
|
|
||||||
disposeViewer?.();
|
|
||||||
setRetryNonce((nonce) => nonce + 1);
|
|
||||||
}
|
|
||||||
const armLiveRecordingOpenTimer = () => {
|
const armLiveRecordingOpenTimer = () => {
|
||||||
clearLiveRecordingOpenTimer();
|
clearLiveRecordingOpenTimer();
|
||||||
diagnosticLifecycle.armAdmissionTimeout(() => {
|
diagnosticLifecycle.armAdmissionTimeout(() => {
|
||||||
@@ -1129,17 +1110,9 @@ export function RerunViewport({
|
|||||||
armLiveRecordingOpenTimer();
|
armLiveRecordingOpenTimer();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (observed.signal === "refresh-receiver") {
|
|
||||||
// Publication is healthy, so this is presentation-only maintenance:
|
|
||||||
// refresh the aged native receiver without spending (or clearing)
|
|
||||||
// recovery debt.
|
|
||||||
recordingOpenTimedOut = true;
|
|
||||||
refreshOpeningLiveReceiver(observed.openForMs);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
recordingOpenTimedOut = true;
|
recordingOpenTimedOut = true;
|
||||||
requestLiveRecovery("recording-open-timeout");
|
requestLiveRecovery("recording-open-timeout");
|
||||||
}, LIVE_RECEIVER_OPEN_MAX_AGE_MS);
|
}, LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS);
|
||||||
};
|
};
|
||||||
const clearLiveRecordingDiscoveryTimer = diagnosticLifecycle.clearAdmissionInterval;
|
const clearLiveRecordingDiscoveryTimer = diagnosticLifecycle.clearAdmissionInterval;
|
||||||
const clearLiveRecoveryRetryTimer = () => {
|
const clearLiveRecoveryRetryTimer = () => {
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ export interface CameraPlaybackRecoveryContext {
|
|||||||
now: number;
|
now: number;
|
||||||
documentVisible: boolean;
|
documentVisible: boolean;
|
||||||
networkOnline: boolean;
|
networkOnline: boolean;
|
||||||
|
transportHealthy?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CameraPlaybackRecoveryDecision {
|
export interface CameraPlaybackRecoveryDecision {
|
||||||
@@ -186,7 +187,8 @@ export function reduceCameraPlaybackRecovery(
|
|||||||
} else if (event.type === "page-restore") {
|
} else if (event.type === "page-restore") {
|
||||||
candidate = event.persisted;
|
candidate = event.persisted;
|
||||||
} else if (event.type === "heartbeat") {
|
} else if (event.type === "heartbeat") {
|
||||||
candidate = now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
|
candidate = context.transportHealthy !== true
|
||||||
|
&& now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
|
||||||
}
|
}
|
||||||
|
|
||||||
const outsideCooldown = current.lastReopenAt === null
|
const outsideCooldown = current.lastReopenAt === null
|
||||||
|
|||||||
@@ -31,7 +31,6 @@ export interface LiveReceiverOpenWatchdogState {
|
|||||||
|
|
||||||
export type LiveReceiverOpenWatchdogSignal =
|
export type LiveReceiverOpenWatchdogSignal =
|
||||||
| "wait-for-store"
|
| "wait-for-store"
|
||||||
| "refresh-receiver"
|
|
||||||
| "restart-receiver";
|
| "restart-receiver";
|
||||||
|
|
||||||
export interface LiveReceiverOpenWatchdogResult {
|
export interface LiveReceiverOpenWatchdogResult {
|
||||||
@@ -60,10 +59,11 @@ export interface LiveReceiverRecoveryRequest {
|
|||||||
export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
|
export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
|
||||||
export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
|
export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
|
||||||
// The bridge publishes its URL only after StoreInfo, blueprint and static
|
// The bridge publishes its URL only after StoreInfo, blueprint and static
|
||||||
// scene data have been flushed. A receiver that still has not admitted that
|
// scene data have been flushed. Poll at the original four-second boundary,
|
||||||
// store after one operator-visible four-second window is wedged, not merely
|
// but never discard a receiver while this exact acquisition is still proving
|
||||||
// slow; keeping it for 48 seconds made a healthy live scan look blank.
|
// fresh publication progress: doing so throws away its partial store replay and
|
||||||
export const LIVE_RECEIVER_OPEN_MAX_AGE_MS = 4_000;
|
// can keep a healthy long-running stream blank indefinitely.
|
||||||
|
export const LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS = 4_000;
|
||||||
const LIVE_RECEIVER_RECOVERY_DELAYS_MS = [400, 1_000, 2_000, 5_000] as const;
|
const LIVE_RECEIVER_RECOVERY_DELAYS_MS = [400, 1_000, 2_000, 5_000] as const;
|
||||||
|
|
||||||
export function liveReceiverRecoveryRetryDelay(attempt: number): number {
|
export function liveReceiverRecoveryRetryDelay(attempt: number): number {
|
||||||
@@ -216,11 +216,10 @@ export function initialLiveReceiverOpenWatchdogState(
|
|||||||
/**
|
/**
|
||||||
* Keep one still-opening Rerun receiver alive while the backend is proving
|
* Keep one still-opening Rerun receiver alive while the backend is proving
|
||||||
* fresh publication progress. Recreating the WASM receiver on a fixed timer
|
* fresh publication progress. Recreating the WASM receiver on a fixed timer
|
||||||
* can repeatedly discard an otherwise healthy late StoreInfo replay. Rolling
|
* can repeatedly discard an otherwise healthy late StoreInfo replay. Preserve
|
||||||
* patience is nevertheless bounded: a receiver that has not admitted a store
|
* that receiver for as long as the backend proves fresh publication. A true
|
||||||
* by the absolute open-age limit is refreshed without consuming the recovery
|
* lack of backend progress delegates to the bounded restart policy on the next
|
||||||
* budget. A true lack of backend progress delegates to the bounded restart
|
* check. Recovery debt is cleared only after viewer admission, never merely
|
||||||
* policy. Recovery debt is cleared only after viewer admission, never merely
|
|
||||||
* because the backend counter advanced.
|
* because the backend counter advanced.
|
||||||
*/
|
*/
|
||||||
export function advanceLiveReceiverOpenWatchdog(
|
export function advanceLiveReceiverOpenWatchdog(
|
||||||
@@ -228,7 +227,6 @@ export function advanceLiveReceiverOpenWatchdog(
|
|||||||
recoveryState: LiveReceiverRecoveryState,
|
recoveryState: LiveReceiverRecoveryState,
|
||||||
backendActivitySequence: number | null,
|
backendActivitySequence: number | null,
|
||||||
nowMs = Date.now(),
|
nowMs = Date.now(),
|
||||||
maxOpenAgeMs = LIVE_RECEIVER_OPEN_MAX_AGE_MS,
|
|
||||||
): LiveReceiverOpenWatchdogResult {
|
): LiveReceiverOpenWatchdogResult {
|
||||||
const sequence = validBackendActivitySequence(backendActivitySequence);
|
const sequence = validBackendActivitySequence(backendActivitySequence);
|
||||||
const previous = current.lastBackendActivitySequence;
|
const previous = current.lastBackendActivitySequence;
|
||||||
@@ -247,9 +245,7 @@ export function advanceLiveReceiverOpenWatchdog(
|
|||||||
return {
|
return {
|
||||||
state,
|
state,
|
||||||
recoveryState,
|
recoveryState,
|
||||||
signal: openForMs >= maxOpenAgeMs
|
signal: "wait-for-store",
|
||||||
? "refresh-receiver"
|
|
||||||
: "wait-for-store",
|
|
||||||
openForMs,
|
openForMs,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -129,6 +129,26 @@ export function admitLiveDefaultPresentations(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A restored workspace may hide cameras that the device plugin has not selected.
|
||||||
|
* It must still admit the exact selected delivery in a fresh browser document,
|
||||||
|
* or a replacement delivery for a camera already presented in this acquisition.
|
||||||
|
* An explicit close remains a separate acquisition-scoped fence in
|
||||||
|
* `admitLiveDefaultPresentations` and de-selects the source in the plugin first.
|
||||||
|
*/
|
||||||
|
export function restoredLayoutMayAdmitLiveDefault(
|
||||||
|
sources: readonly ObservationSourceDescriptor[],
|
||||||
|
presentedAcquisitionSources: ReadonlySet<string>,
|
||||||
|
): boolean {
|
||||||
|
const selectedSources = sources.filter(automaticLivePresentationIdentity);
|
||||||
|
if (selectedSources.length === 0) return false;
|
||||||
|
if (presentedAcquisitionSources.size === 0) return true;
|
||||||
|
return selectedSources.some((source) => {
|
||||||
|
const lineage = livePresentationCloseFence(source);
|
||||||
|
return Boolean(lineage && presentedAcquisitionSources.has(lineage));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function catalogIdentity(sources: readonly ObservationSourceDescriptor[]): string {
|
function catalogIdentity(sources: readonly ObservationSourceDescriptor[]): string {
|
||||||
return sources
|
return sources
|
||||||
.map((source) => [
|
.map((source) => [
|
||||||
@@ -170,6 +190,7 @@ export function useObservationLayout(
|
|||||||
const restoredLayoutAuthorityRef = useRef(false);
|
const restoredLayoutAuthorityRef = useRef(false);
|
||||||
const admittedLivePresentationIdentitiesRef = useRef(new Set<string>());
|
const admittedLivePresentationIdentitiesRef = useRef(new Set<string>());
|
||||||
const closedLiveAcquisitionSourcesRef = useRef(new Set<string>());
|
const closedLiveAcquisitionSourcesRef = useRef(new Set<string>());
|
||||||
|
const presentedLiveAcquisitionSourcesRef = useRef(new Set<string>());
|
||||||
const initializedCatalog = useRef<string | null>(null);
|
const initializedCatalog = useRef<string | null>(null);
|
||||||
const sourceIdList = sources.map((source) => source.id).sort();
|
const sourceIdList = sources.map((source) => source.id).sort();
|
||||||
const sourceIdsIdentity = sourceIdList.join("\u0000");
|
const sourceIdsIdentity = sourceIdList.join("\u0000");
|
||||||
@@ -182,6 +203,12 @@ export function useObservationLayout(
|
|||||||
|
|
||||||
const commitVisibleIds = useCallback((next: readonly string[]) => {
|
const commitVisibleIds = useCallback((next: readonly string[]) => {
|
||||||
const unique = [...new Set(next)];
|
const unique = [...new Set(next)];
|
||||||
|
const visible = new Set(unique);
|
||||||
|
for (const source of sourcesRef.current) {
|
||||||
|
if (!visible.has(source.id) || !automaticLivePresentationIdentity(source)) continue;
|
||||||
|
const lineage = livePresentationCloseFence(source);
|
||||||
|
if (lineage) presentedLiveAcquisitionSourcesRef.current.add(lineage);
|
||||||
|
}
|
||||||
visibleIdsRef.current = unique;
|
visibleIdsRef.current = unique;
|
||||||
setVisibleIds(unique);
|
setVisibleIds(unique);
|
||||||
}, []);
|
}, []);
|
||||||
@@ -352,11 +379,18 @@ export function useObservationLayout(
|
|||||||
.filter((candidate): candidate is string => candidate !== null)
|
.filter((candidate): candidate is string => candidate !== null)
|
||||||
.sort());
|
.sort());
|
||||||
|
|
||||||
useEffect(() => {
|
const admitAutomaticLivePresentations = useCallback(() => {
|
||||||
if (restoredLayoutAuthorityRef.current) return;
|
const currentSources = sourcesRef.current;
|
||||||
|
if (
|
||||||
|
restoredLayoutAuthorityRef.current
|
||||||
|
&& !restoredLayoutMayAdmitLiveDefault(
|
||||||
|
currentSources,
|
||||||
|
presentedLiveAcquisitionSourcesRef.current,
|
||||||
|
)
|
||||||
|
) return;
|
||||||
const admission = admitLiveDefaultPresentations(
|
const admission = admitLiveDefaultPresentations(
|
||||||
visibleIdsRef.current,
|
visibleIdsRef.current,
|
||||||
sources,
|
currentSources,
|
||||||
admittedLivePresentationIdentitiesRef.current,
|
admittedLivePresentationIdentitiesRef.current,
|
||||||
closedLiveAcquisitionSourcesRef.current,
|
closedLiveAcquisitionSourcesRef.current,
|
||||||
);
|
);
|
||||||
@@ -367,7 +401,11 @@ export function useObservationLayout(
|
|||||||
commitVisibleIds(admission.visibleIds);
|
commitVisibleIds(admission.visibleIds);
|
||||||
clearPresentation(admission.removedIds, false);
|
clearPresentation(admission.removedIds, false);
|
||||||
persistLiveLayout();
|
persistLiveLayout();
|
||||||
}, [clearPresentation, commitVisibleIds, persistLiveLayout, selectedDeliveryIdentity]);
|
}, [clearPresentation, commitVisibleIds, persistLiveLayout]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
admitAutomaticLivePresentations();
|
||||||
|
}, [admitAutomaticLivePresentations, selectedDeliveryIdentity]);
|
||||||
|
|
||||||
const markPending = useCallback((source: ObservationSourceDescriptor, pending: boolean) => {
|
const markPending = useCallback((source: ObservationSourceDescriptor, pending: boolean) => {
|
||||||
const groupId = source.activation?.groupId;
|
const groupId = source.activation?.groupId;
|
||||||
@@ -545,7 +583,10 @@ export function useObservationLayout(
|
|||||||
});
|
});
|
||||||
restoredLayoutAuthorityRef.current = true;
|
restoredLayoutAuthorityRef.current = true;
|
||||||
applyDesiredSnapshot(desiredSnapshotRef.current, "reset");
|
applyDesiredSnapshot(desiredSnapshotRef.current, "reset");
|
||||||
}, [applyDesiredSnapshot]);
|
// The selected delivery may already have arrived before the saved layout.
|
||||||
|
// Re-run admission here so restore ordering cannot strand its camera window.
|
||||||
|
admitAutomaticLivePresentations();
|
||||||
|
}, [admitAutomaticLivePresentations, applyDesiredSnapshot]);
|
||||||
|
|
||||||
const visibleSourceIds = useMemo(() => new Set(visibleIds), [visibleIds]);
|
const visibleSourceIds = useMemo(() => new Set(visibleIds), [visibleIds]);
|
||||||
const pendingSourceIds = useMemo(() => new Set(pendingIds), [pendingIds]);
|
const pendingSourceIds = useMemo(() => new Set(pendingIds), [pendingIds]);
|
||||||
|
|||||||
@@ -235,12 +235,7 @@ function SpatialWorkspace({
|
|||||||
Boolean(observationLayout.maximizedFloatingSourceId);
|
Boolean(observationLayout.maximizedFloatingSourceId);
|
||||||
const timeline = state?.observationTimeline;
|
const timeline = state?.observationTimeline;
|
||||||
const viewportRef = useRef<HTMLDivElement>(null);
|
const viewportRef = useRef<HTMLDivElement>(null);
|
||||||
const intentionalSourceEnd = !recordedSource && [
|
const intentionalSourceEnd = !recordedSource && state?.sourceMode === "idle" && ["awaiting_external_stop", "stopping", "finalizing", "completed",].includes(state?.acquisition?.state ?? "");
|
||||||
"awaiting_external_stop",
|
|
||||||
"stopping",
|
|
||||||
"finalizing",
|
|
||||||
"completed",
|
|
||||||
].includes(state?.acquisition?.state ?? "");
|
|
||||||
const presentedViewerStatus = intentionalSourceEnd
|
const presentedViewerStatus = intentionalSourceEnd
|
||||||
? "idle"
|
? "idle"
|
||||||
: rerunPresentationStatus(
|
: rerunPresentationStatus(
|
||||||
|
|||||||
@@ -620,6 +620,53 @@ test("canonical K1 preparation stops before START and guards every async stage",
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("K1 control-phase polling fails closed instead of stacking timed-out state reads", async () => {
|
||||||
|
const hookSource = await readFile(
|
||||||
|
new URL(
|
||||||
|
"../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts",
|
||||||
|
import.meta.url,
|
||||||
|
),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const controlPollingLoop = hookSource.slice(
|
||||||
|
hookSource.indexOf("async function waitForControlPhase"),
|
||||||
|
hookSource.indexOf("async function waitForPhysicalReconciliationProof"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.match(hookSource, /const CONTROL_PHASE_WAIT_TIMEOUT_MS = 120_000/);
|
||||||
|
assert.match(controlPollingLoop, /xgridsK1Api\.getState\(\)/);
|
||||||
|
assert.doesNotMatch(controlPollingLoop, /ApiRequestTimeoutError/);
|
||||||
|
assert.doesNotMatch(controlPollingLoop, /catch \(error\)/);
|
||||||
|
assert.doesNotMatch(
|
||||||
|
controlPollingLoop,
|
||||||
|
/(?:openApplicationControlSession|enterApplicationWorkspace|prepareAcquisition|startAcquisition|stopAcquisition)\(/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("K1 frontend coalesces concurrent read-only state requests without caching them", async () => {
|
||||||
|
const apiSource = await readFile(
|
||||||
|
new URL(
|
||||||
|
"../../../plugins/xgrids-k1/frontend/src/api.ts",
|
||||||
|
import.meta.url,
|
||||||
|
),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const singleFlight = apiSource.slice(
|
||||||
|
apiSource.indexOf("let stateReadInFlight"),
|
||||||
|
apiSource.indexOf("export const xgridsK1Api"),
|
||||||
|
);
|
||||||
|
const getState = apiSource.slice(
|
||||||
|
apiSource.indexOf("async getState"),
|
||||||
|
apiSource.indexOf("scanBle("),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.match(singleFlight, /if \(stateReadInFlight\) return stateReadInFlight/);
|
||||||
|
assert.match(singleFlight, /invokeState\(xgridsK1Actions\.stateRead\)/);
|
||||||
|
assert.match(singleFlight, /stateReadInFlight === request[\s\S]*?stateReadInFlight = null/);
|
||||||
|
assert.match(getState, /return readStateSingleFlight\(\)/);
|
||||||
|
assert.doesNotMatch(singleFlight, /setTimeout|setInterval|retry/i);
|
||||||
|
});
|
||||||
|
|
||||||
test("K1 control-session CAS is exact, integer-only and mapped for each mutation family", () => {
|
test("K1 control-session CAS is exact, integer-only and mapped for each mutation family", () => {
|
||||||
const state = {
|
const state = {
|
||||||
application_control_session: {
|
application_control_session: {
|
||||||
|
|||||||
@@ -244,13 +244,16 @@ test("connecting Rerun authority requires an exact recovery generation lease", (
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("opening receiver gets bounded rolling patience while backend publication advances", () => {
|
test("opening receiver preserves partial store replay while backend publication advances", () => {
|
||||||
let openState = initialLiveReceiverOpenWatchdogState(0, 0);
|
let openState = initialLiveReceiverOpenWatchdogState(0, 0);
|
||||||
let recoveryState = initialLiveReceiverRecoveryState();
|
let recoveryState = initialLiveReceiverRecoveryState();
|
||||||
|
|
||||||
const samples = [
|
const samples = [
|
||||||
[134, 3_999, "wait-for-store"],
|
[134, 3_999, "wait-for-store"],
|
||||||
[266, 4_000, "refresh-receiver"],
|
[266, 4_000, "wait-for-store"],
|
||||||
|
[380, 8_000, "wait-for-store"],
|
||||||
|
[486, 12_000, "wait-for-store"],
|
||||||
|
[700, 120_000, "wait-for-store"],
|
||||||
];
|
];
|
||||||
for (const [backendActivitySequence, nowMs, expectedSignal] of samples) {
|
for (const [backendActivitySequence, nowMs, expectedSignal] of samples) {
|
||||||
const observed = advanceLiveReceiverOpenWatchdog(
|
const observed = advanceLiveReceiverOpenWatchdog(
|
||||||
@@ -302,17 +305,17 @@ test("fresh backend progress preserves earlier restart debt until viewer admissi
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("aged active receiver refresh does not spend or erase restart debt", () => {
|
test("aged active receiver remains intact and preserves restart debt", () => {
|
||||||
const openState = initialLiveReceiverOpenWatchdogState(486, 0);
|
const openState = initialLiveReceiverOpenWatchdogState(486, 0);
|
||||||
const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
|
const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
|
||||||
const observed = advanceLiveReceiverOpenWatchdog(
|
const observed = advanceLiveReceiverOpenWatchdog(
|
||||||
openState,
|
openState,
|
||||||
consumedRestart.state,
|
consumedRestart.state,
|
||||||
900,
|
900,
|
||||||
4_000,
|
12_000,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert.equal(observed.signal, "refresh-receiver");
|
assert.equal(observed.signal, "wait-for-store");
|
||||||
assert.deepEqual(observed.recoveryState, consumedRestart.state);
|
assert.deepEqual(observed.recoveryState, consumedRestart.state);
|
||||||
assert.equal(observed.openForMs, 4_000);
|
assert.equal(observed.openForMs, 12_000);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1199,7 +1199,7 @@ test("camera startup watchdog replaces an open-but-silent MSE or WebSocket", ()
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("first media starts one non-sliding first-playable-frame deadline", () => {
|
test("continuing media extends the first-playable-frame deadline", () => {
|
||||||
const scheduled = new Map();
|
const scheduled = new Map();
|
||||||
const cancelled = [];
|
const cancelled = [];
|
||||||
const timeouts = [];
|
const timeouts = [];
|
||||||
@@ -1228,8 +1228,14 @@ test("first media starts one non-sliding first-playable-frame deadline", () => {
|
|||||||
assert.equal(scheduled.get(playableHandle).timeoutMs, CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS);
|
assert.equal(scheduled.get(playableHandle).timeoutMs, CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS);
|
||||||
|
|
||||||
watchdog.markMediaReceived();
|
watchdog.markMediaReceived();
|
||||||
assert.equal(nextHandle - 1, playableHandle, "later fragments must not extend the deadline");
|
const extendedPlayableHandle = nextHandle - 1;
|
||||||
scheduled.get(playableHandle).callback();
|
assert.notEqual(extendedPlayableHandle, playableHandle);
|
||||||
|
assert.deepEqual(cancelled, [10, playableHandle]);
|
||||||
|
assert.equal(
|
||||||
|
scheduled.get(extendedPlayableHandle).timeoutMs,
|
||||||
|
CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS,
|
||||||
|
);
|
||||||
|
scheduled.get(extendedPlayableHandle).callback();
|
||||||
assert.deepEqual(timeouts, ["first-playable-frame"]);
|
assert.deepEqual(timeouts, ["first-playable-frame"]);
|
||||||
assert.match(
|
assert.match(
|
||||||
cameraStartupWatchdogRecoveryMessage("first-playable-frame"),
|
cameraStartupWatchdogRecoveryMessage("first-playable-frame"),
|
||||||
@@ -1306,6 +1312,7 @@ test("a laptop sleep gap reopens only the exact authoritative live camera transp
|
|||||||
now: 7_000,
|
now: 7_000,
|
||||||
documentVisible: true,
|
documentVisible: true,
|
||||||
networkOnline: true,
|
networkOnline: true,
|
||||||
|
transportHealthy: false,
|
||||||
});
|
});
|
||||||
assert.equal(wake.reopen, true);
|
assert.equal(wake.reopen, true);
|
||||||
assert.equal(wake.state.lastReopenAt, 7_000);
|
assert.equal(wake.state.lastReopenAt, 7_000);
|
||||||
@@ -1369,15 +1376,16 @@ test("visibility, pageshow and online wake burst owns one replacement decoder",
|
|||||||
assert.equal(secondWake.state.lastReopenAt, 18_000);
|
assert.equal(secondWake.state.lastReopenAt, 18_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("healthy visible camera heartbeats do not churn its WebSocket or decoder", () => {
|
test("healthy camera survives a long main-thread heartbeat gap without decoder churn", () => {
|
||||||
const authority = "camera-authority-acquisition-healthy";
|
const authority = "camera-authority-acquisition-healthy";
|
||||||
let state = initialCameraPlaybackRecoveryState(authority, 1_000);
|
let state = initialCameraPlaybackRecoveryState(authority, 1_000);
|
||||||
for (const now of [2_000, 3_000, 4_000, 5_000]) {
|
for (const now of [2_000, 3_000, 4_000, 12_000, 25_000]) {
|
||||||
const heartbeat = reduceCameraPlaybackRecovery(state, { type: "heartbeat" }, {
|
const heartbeat = reduceCameraPlaybackRecovery(state, { type: "heartbeat" }, {
|
||||||
activeAuthorityIdentity: authority,
|
activeAuthorityIdentity: authority,
|
||||||
now,
|
now,
|
||||||
documentVisible: true,
|
documentVisible: true,
|
||||||
networkOnline: true,
|
networkOnline: true,
|
||||||
|
transportHealthy: true,
|
||||||
});
|
});
|
||||||
assert.equal(heartbeat.reopen, false);
|
assert.equal(heartbeat.reopen, false);
|
||||||
state = heartbeat.state;
|
state = heartbeat.state;
|
||||||
|
|||||||
@@ -295,6 +295,25 @@ test("raw replay exercises the same streaming receiver lifecycle as a live scan"
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("pending K1 STOP keeps the live Rerun source mounted until local capture ends", async () => {
|
||||||
|
const source = await readFile(
|
||||||
|
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
source,
|
||||||
|
/const intentionalSourceEnd = !recordedSource &&\s*state\?\.sourceMode === "idle" && \[/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
source,
|
||||||
|
/"awaiting_external_stop",\s*"stopping",\s*"finalizing",\s*"completed",/,
|
||||||
|
);
|
||||||
|
assert.doesNotMatch(
|
||||||
|
source,
|
||||||
|
/const intentionalSourceEnd = !recordedSource && \[\s*"awaiting_external_stop"/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test("the spatial header reports raw replay as an active source", async () => {
|
test("the spatial header reports raw replay as an active source", async () => {
|
||||||
const source = await readFile(
|
const source = await readFile(
|
||||||
new URL("../src/App.tsx", import.meta.url),
|
new URL("../src/App.tsx", import.meta.url),
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ let WorkspaceLayoutContractError;
|
|||||||
let admitLiveDefaultPresentations;
|
let admitLiveDefaultPresentations;
|
||||||
let automaticLivePresentationIdentity;
|
let automaticLivePresentationIdentity;
|
||||||
let livePresentationCloseFence;
|
let livePresentationCloseFence;
|
||||||
|
let restoredLayoutMayAdmitLiveDefault;
|
||||||
let observationPresentationSourceAfterLayoutApply;
|
let observationPresentationSourceAfterLayoutApply;
|
||||||
let visibleSourceIdsAfterRecordedCatalogActivation;
|
let visibleSourceIdsAfterRecordedCatalogActivation;
|
||||||
|
|
||||||
@@ -39,6 +40,7 @@ before(async () => {
|
|||||||
admitLiveDefaultPresentations,
|
admitLiveDefaultPresentations,
|
||||||
automaticLivePresentationIdentity,
|
automaticLivePresentationIdentity,
|
||||||
livePresentationCloseFence,
|
livePresentationCloseFence,
|
||||||
|
restoredLayoutMayAdmitLiveDefault,
|
||||||
observationPresentationSourceAfterLayoutApply,
|
observationPresentationSourceAfterLayoutApply,
|
||||||
visibleSourceIdsAfterRecordedCatalogActivation,
|
visibleSourceIdsAfterRecordedCatalogActivation,
|
||||||
} = await server.ssrLoadModule("/src/core/observation/useObservationLayout.ts"));
|
} = await server.ssrLoadModule("/src/core/observation/useObservationLayout.ts"));
|
||||||
@@ -308,6 +310,67 @@ test("a sequential live acquisition re-arms the same camera without reopening a
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("a restored layout admits a selected delivery after reload and only same-acquisition successors", () => {
|
||||||
|
const camera = (acquisitionId, deliveryId) => ({
|
||||||
|
id: "k1:sensor.camera.right",
|
||||||
|
sourceId: "sensor.camera.right",
|
||||||
|
modality: "video",
|
||||||
|
availability: "streaming",
|
||||||
|
transport: "websocket",
|
||||||
|
previewUrl: null,
|
||||||
|
delivery: {
|
||||||
|
id: deliveryId,
|
||||||
|
kind: "mse-fmp4-websocket",
|
||||||
|
url: `/camera-preview/${deliveryId}`,
|
||||||
|
mediaType: 'video/mp4; codecs="avc1.641028"',
|
||||||
|
},
|
||||||
|
activation: {
|
||||||
|
groupId: "k1:device-session-reused:camera.preview.decoder",
|
||||||
|
maxActive: 1,
|
||||||
|
selected: true,
|
||||||
|
controllable: true,
|
||||||
|
},
|
||||||
|
binding: {
|
||||||
|
deviceId: "k1-a",
|
||||||
|
deviceSessionId: "device-session-reused",
|
||||||
|
acquisitionId,
|
||||||
|
},
|
||||||
|
capabilities: { defaultVisible: true, overlay: true },
|
||||||
|
});
|
||||||
|
const original = camera("acquisition-a", "camera-preview-2");
|
||||||
|
const successor = camera("acquisition-a", "camera-preview-3");
|
||||||
|
const unrelated = camera("acquisition-b", "camera-preview-3");
|
||||||
|
const presented = new Set([livePresentationCloseFence(original)]);
|
||||||
|
|
||||||
|
assert.equal(restoredLayoutMayAdmitLiveDefault([original], new Set()), true);
|
||||||
|
assert.equal(restoredLayoutMayAdmitLiveDefault([successor], presented), true);
|
||||||
|
assert.equal(restoredLayoutMayAdmitLiveDefault([unrelated], presented), false);
|
||||||
|
assert.equal(restoredLayoutMayAdmitLiveDefault([{
|
||||||
|
...original,
|
||||||
|
activation: { ...original.activation, selected: false },
|
||||||
|
}], new Set()), false);
|
||||||
|
|
||||||
|
const admitted = admitLiveDefaultPresentations(
|
||||||
|
[],
|
||||||
|
[successor],
|
||||||
|
new Set([automaticLivePresentationIdentity(original)]),
|
||||||
|
new Set(),
|
||||||
|
);
|
||||||
|
assert.deepEqual(admitted.visibleIds, [successor.id]);
|
||||||
|
assert.deepEqual(admitted.admittedIdentities, [
|
||||||
|
automaticLivePresentationIdentity(successor),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const deliberatelyClosed = admitLiveDefaultPresentations(
|
||||||
|
[],
|
||||||
|
[successor],
|
||||||
|
new Set([automaticLivePresentationIdentity(original)]),
|
||||||
|
presented,
|
||||||
|
);
|
||||||
|
assert.deepEqual(deliberatelyClosed.visibleIds, []);
|
||||||
|
assert.deepEqual(deliberatelyClosed.admittedIdentities, []);
|
||||||
|
});
|
||||||
|
|
||||||
test("workspace layout API uses the canonical endpoint and optimistic revision", async () => {
|
test("workspace layout API uses the canonical endpoint and optimistic revision", async () => {
|
||||||
const calls = [];
|
const calls = [];
|
||||||
const current = decodeObservationWorkspaceLayoutProfile(wireProfile());
|
const current = decodeObservationWorkspaceLayoutProfile(wireProfile());
|
||||||
|
|||||||
@@ -1875,6 +1875,20 @@ function invokeState(actionId: string, input: object = {}): Promise<XgridsK1Stat
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let stateReadInFlight: Promise<XgridsK1State> | null = null;
|
||||||
|
|
||||||
|
function readStateSingleFlight(): Promise<XgridsK1State> {
|
||||||
|
if (stateReadInFlight) return stateReadInFlight;
|
||||||
|
|
||||||
|
const request = invokeState(xgridsK1Actions.stateRead);
|
||||||
|
stateReadInFlight = request;
|
||||||
|
const release = () => {
|
||||||
|
if (stateReadInFlight === request) stateReadInFlight = null;
|
||||||
|
};
|
||||||
|
void request.then(release, release);
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
export const xgridsK1Api = {
|
export const xgridsK1Api = {
|
||||||
async getHealth(): Promise<HealthResponse> {
|
async getHealth(): Promise<HealthResponse> {
|
||||||
const payload = await requestJson(
|
const payload = await requestJson(
|
||||||
@@ -1890,7 +1904,7 @@ export const xgridsK1Api = {
|
|||||||
},
|
},
|
||||||
|
|
||||||
async getState(): Promise<XgridsK1State> {
|
async getState(): Promise<XgridsK1State> {
|
||||||
return invokeState(xgridsK1Actions.stateRead);
|
return readStateSingleFlight();
|
||||||
},
|
},
|
||||||
|
|
||||||
scanBle(
|
scanBle(
|
||||||
|
|||||||
@@ -67,10 +67,12 @@ function RecoveryState({
|
|||||||
{copy.showSpinner ? <ActivityIndicator size="compact" /> : null}
|
{copy.showSpinner ? <ActivityIndicator size="compact" /> : null}
|
||||||
<div className="active-stream-recovery__copy">
|
<div className="active-stream-recovery__copy">
|
||||||
<strong>{copy.title}</strong>
|
<strong>{copy.title}</strong>
|
||||||
|
<span className="active-stream-recovery__detail">
|
||||||
<span>{copy.detail}</span>
|
<span>{copy.detail}</span>
|
||||||
{presentation?.progressLabel ? (
|
{presentation?.progressLabel ? (
|
||||||
<small>{presentation.progressLabel}</small>
|
<small>{presentation.progressLabel}</small>
|
||||||
) : null}
|
) : null}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -599,6 +599,12 @@ box-sizing: border-box;
|
|||||||
gap: 0.28rem;
|
gap: 0.28rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.active-stream-recovery__detail {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.28rem;
|
||||||
|
}
|
||||||
|
|
||||||
.active-stream-recovery__copy strong {
|
.active-stream-recovery__copy strong {
|
||||||
color: var(--nodedc-text-primary);
|
color: var(--nodedc-text-primary);
|
||||||
font-size: 0.72rem;
|
font-size: 0.72rem;
|
||||||
@@ -1023,25 +1029,83 @@ box-sizing: border-box;
|
|||||||
min-width: 0;
|
min-width: 0;
|
||||||
grid-template-columns: minmax(0, 1fr) auto;
|
grid-template-columns: minmax(0, 1fr) auto;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 0.65rem;
|
column-gap: 0.75rem;
|
||||||
margin-top: 0;
|
margin-top: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.active-stream-recovery--compact .active-stream-recovery__state {
|
.active-stream-recovery--compact .active-stream-recovery__state {
|
||||||
|
grid-column: 1;
|
||||||
|
grid-row: 1;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.65rem;
|
||||||
background: transparent;
|
background: transparent;
|
||||||
padding: 0;
|
padding: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.active-stream-recovery--compact
|
||||||
|
.active-stream-recovery__state
|
||||||
|
> .nodedc-activity-indicator {
|
||||||
|
margin-top: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.active-stream-recovery--compact .active-stream-recovery__copy {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
justify-items: center;
|
||||||
|
gap: 0.1rem;
|
||||||
|
overflow: hidden;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.active-stream-recovery--compact .active-stream-recovery__copy strong {
|
||||||
|
max-width: 100%;
|
||||||
|
overflow: hidden;
|
||||||
|
font-size: 0.62rem;
|
||||||
|
font-weight: 600;
|
||||||
|
line-height: 1.35;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.active-stream-recovery--compact .active-stream-recovery__detail {
|
||||||
|
display: flex;
|
||||||
|
min-width: 0;
|
||||||
|
max-width: 100%;
|
||||||
|
align-items: baseline;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 0.4rem;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.active-stream-recovery--compact .active-stream-recovery__detail > span {
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
font-size: 0.56rem;
|
||||||
|
line-height: 1.35;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.active-stream-recovery--compact .active-stream-recovery__copy small {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
font-size: 0.54rem;
|
||||||
|
line-height: 1.35;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
.active-stream-recovery--compact .active-stream-recovery__actions {
|
.active-stream-recovery--compact .active-stream-recovery__actions {
|
||||||
max-width: 15rem;
|
grid-column: 2;
|
||||||
|
grid-row: 1;
|
||||||
|
align-self: start;
|
||||||
|
max-width: none;
|
||||||
grid-template-columns: auto;
|
grid-template-columns: auto;
|
||||||
|
justify-items: end;
|
||||||
gap: 0.25rem;
|
gap: 0.25rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.active-stream-recovery--compact .active-stream-recovery__actions p {
|
.active-stream-recovery--compact .active-stream-recovery__actions p {
|
||||||
font-size: 0.5rem;
|
display: none;
|
||||||
line-height: 1.35;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.xgrids-k1-spatial-controls__phase {
|
.xgrids-k1-spatial-controls__phase {
|
||||||
@@ -1155,14 +1219,10 @@ box-sizing: border-box;
|
|||||||
}
|
}
|
||||||
|
|
||||||
.active-stream-recovery--compact {
|
.active-stream-recovery--compact {
|
||||||
grid-template-columns: minmax(0, 1fr);
|
grid-template-columns: minmax(0, 1fr) auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
.active-stream-recovery--compact .active-stream-recovery__actions {
|
.active-stream-recovery--compact .active-stream-recovery__actions {
|
||||||
max-width: none;
|
grid-column: 2;
|
||||||
}
|
|
||||||
|
|
||||||
.active-stream-recovery--compact .active-stream-recovery__actions p {
|
|
||||||
display: none;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -248,7 +248,7 @@ export function connectionActionAuthoritySnapshot(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const CONTROL_STATE_READ_INTERVAL_MS = 250;
|
const CONTROL_STATE_READ_INTERVAL_MS = 250;
|
||||||
const CONTROL_PHASE_WAIT_TIMEOUT_MS = 35_000;
|
const CONTROL_PHASE_WAIT_TIMEOUT_MS = 120_000;
|
||||||
|
|
||||||
function controlPhase(state: XgridsK1State): XgridsApplicationControlPhase {
|
function controlPhase(state: XgridsK1State): XgridsApplicationControlPhase {
|
||||||
return state.application_control_session?.state ?? "idle";
|
return state.application_control_session?.state ?? "idle";
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl"
|
|||||||
RUNNER_NAME = RUNNER.name
|
RUNNER_NAME = RUNNER.name
|
||||||
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
|
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
|
||||||
EXPECTED_BASELINE_SHA256 = "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8"
|
EXPECTED_BASELINE_SHA256 = "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8"
|
||||||
EXPECTED_WHEEL_SHA256 = "9a60efa68eadf2267fffe3dbb89fb58d5e69e672fa3cdf7529474c9f47416acb"
|
EXPECTED_WHEEL_SHA256 = "411e1a040b708a118827dfd6898ef1f9911e42a86fa58b9e476688cfa046fce1"
|
||||||
PAYLOAD_FILES = (
|
PAYLOAD_FILES = (
|
||||||
RUNNER_NAME,
|
RUNNER_NAME,
|
||||||
WHEEL_NAME,
|
WHEEL_NAME,
|
||||||
|
|||||||
@@ -1868,9 +1868,10 @@ class ReadOnlyConnectionMonitor:
|
|||||||
return None
|
return None
|
||||||
previous_path = baseline.host_path
|
previous_path = baseline.host_path
|
||||||
intent = baseline.intent
|
intent = baseline.intent
|
||||||
exact_verified_control = bool(
|
if intent is None:
|
||||||
intent is not None
|
return None
|
||||||
and previous_path.available
|
exact_reachable_route = bool(
|
||||||
|
previous_path.available
|
||||||
and previous_path.fingerprint is not None
|
and previous_path.fingerprint is not None
|
||||||
and previous_path.kernel_route_fingerprint is not None
|
and previous_path.kernel_route_fingerprint is not None
|
||||||
and baseline.device_network.state == "applied"
|
and baseline.device_network.state == "applied"
|
||||||
@@ -1880,6 +1881,9 @@ class ReadOnlyConnectionMonitor:
|
|||||||
and baseline.endpoint.intent_id == intent.intent_id
|
and baseline.endpoint.intent_id == intent.intent_id
|
||||||
and baseline.endpoint.host_path_epoch == previous_path.epoch
|
and baseline.endpoint.host_path_epoch == previous_path.epoch
|
||||||
and baseline.endpoint.tcp_state == "reachable"
|
and baseline.endpoint.tcp_state == "reachable"
|
||||||
|
)
|
||||||
|
exact_verified_control = bool(
|
||||||
|
exact_reachable_route
|
||||||
and baseline.device_identity.state == "verified"
|
and baseline.device_identity.state == "verified"
|
||||||
and baseline.device_identity.intent_id == intent.intent_id
|
and baseline.device_identity.intent_id == intent.intent_id
|
||||||
and baseline.device_identity.host_path_epoch == previous_path.epoch
|
and baseline.device_identity.host_path_epoch == previous_path.epoch
|
||||||
@@ -1892,7 +1896,11 @@ class ReadOnlyConnectionMonitor:
|
|||||||
and baseline.lease.target == target
|
and baseline.lease.target == target
|
||||||
and baseline.authority.control_allowed
|
and baseline.authority.control_allowed
|
||||||
)
|
)
|
||||||
if not exact_verified_control:
|
technical_timeout_on_exact_route = bool(
|
||||||
|
result.reason_code == "host-wifi-operation-timeout"
|
||||||
|
and exact_reachable_route
|
||||||
|
)
|
||||||
|
if not (exact_verified_control or technical_timeout_on_exact_route):
|
||||||
return None
|
return None
|
||||||
if (
|
if (
|
||||||
result.kernel_route_fingerprint is None
|
result.kernel_route_fingerprint is None
|
||||||
|
|||||||
@@ -278,6 +278,13 @@ HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS = 2.0
|
|||||||
# without weakening the sub-second association cache or skipping the final
|
# without weakening the sub-second association cache or skipping the final
|
||||||
# route/association continuity recheck.
|
# route/association continuity recheck.
|
||||||
CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS = 3.0
|
CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS = 3.0
|
||||||
|
# Active-stream recovery has already frozen an exact acquisition/START owner.
|
||||||
|
# Its endpoint probe may therefore use the same short read-only association
|
||||||
|
# budget as the monitor: the exact raw route bridge above preserves continuity
|
||||||
|
# across a technical CoreWLAN timeout, while fresh DeviceInfo still has to
|
||||||
|
# verify the K1 before MQTT publication resumes. Do not inherit the generic
|
||||||
|
# 30-second helper budget on every Wi-Fi return.
|
||||||
|
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS = 3.0
|
||||||
# Command checkpoints must not inherit either the monitor's three-second
|
# Command checkpoints must not inherit either the monitor's three-second
|
||||||
# contact budget or the host helper's generic 30-second budget. One command
|
# contact budget or the host helper's generic 30-second budget. One command
|
||||||
# may legitimately wait about three seconds for the service-lifetime
|
# may legitimately wait about three seconds for the service-lifetime
|
||||||
@@ -303,6 +310,9 @@ CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS = (
|
|||||||
2 * CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS + CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS
|
2 * CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS + CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS
|
||||||
)
|
)
|
||||||
CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS = 8.0
|
CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS = 8.0
|
||||||
|
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS = (
|
||||||
|
CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS + 1.0
|
||||||
|
)
|
||||||
CONNECTION_MONITOR_DEBOUNCEABLE_ASSOCIATION_REASONS = frozenset({"host-wifi-operation-timeout"})
|
CONNECTION_MONITOR_DEBOUNCEABLE_ASSOCIATION_REASONS = frozenset({"host-wifi-operation-timeout"})
|
||||||
LIVE_DATA_PLANE_STALL_SECONDS = 5.0
|
LIVE_DATA_PLANE_STALL_SECONDS = 5.0
|
||||||
LIVE_DATA_PLANE_LOST_SECONDS = LIVE_DATA_PLANE_STALL_SECONDS * 2.0
|
LIVE_DATA_PLANE_LOST_SECONDS = LIVE_DATA_PLANE_STALL_SECONDS * 2.0
|
||||||
@@ -2557,8 +2567,11 @@ class XgridsK1CompatibilityService:
|
|||||||
A privacy-limited observation has the stricter gate below: exact
|
A privacy-limited observation has the stricter gate below: exact
|
||||||
DeviceInfo, healthy control session, reachable lease and command
|
DeviceInfo, healthy control session, reachable lease and command
|
||||||
authority must all still match the same intent/mode/target/epoch. The
|
authority must all still match the same intent/mode/target/epoch. The
|
||||||
legacy timeout bridge may retain only its exact reachable route but
|
sole exception is an already-admitted active-stream recovery lineage:
|
||||||
cannot manufacture any missing command authority.
|
it may retain only this exact reachable route long enough to perform a
|
||||||
|
fresh read-only DeviceInfo/MQTT rebind. That route bridge grants no
|
||||||
|
control authority and cannot send START, STOP or a network mutation.
|
||||||
|
The legacy timeout bridge has the same route-only limitation.
|
||||||
|
|
||||||
A proven association change, a different route/interface/source, a
|
A proven association change, a different route/interface/source, a
|
||||||
stale or unreachable endpoint, or a different intent/target bypasses
|
stale or unreachable endpoint, or a different intent/target bypasses
|
||||||
@@ -2623,7 +2636,13 @@ class XgridsK1CompatibilityService:
|
|||||||
and snapshot.lease.target == endpoint_target
|
and snapshot.lease.target == endpoint_target
|
||||||
and snapshot.authority.control_allowed
|
and snapshot.authority.control_allowed
|
||||||
)
|
)
|
||||||
if not exact_verified_control:
|
exact_active_stream_rebind = (
|
||||||
|
self._exact_active_stream_recovery_route_rebind(
|
||||||
|
snapshot=snapshot,
|
||||||
|
endpoint_target=endpoint_target,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not (exact_verified_control or exact_active_stream_rebind):
|
||||||
return None
|
return None
|
||||||
return HostPathProbeResult(
|
return HostPathProbeResult(
|
||||||
available=True,
|
available=True,
|
||||||
@@ -2640,6 +2659,68 @@ class XgridsK1CompatibilityService:
|
|||||||
kernel_route_fingerprint=path.fingerprint,
|
kernel_route_fingerprint=path.fingerprint,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _exact_active_stream_recovery_route_rebind(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
snapshot: ConnectionSupervisorSnapshot,
|
||||||
|
endpoint_target: EndpointTarget,
|
||||||
|
) -> bool:
|
||||||
|
"""Allow one exact route-only bridge for an admitted live recovery.
|
||||||
|
|
||||||
|
Admission of ``_ActiveStreamRecoveryLineage`` already proved the
|
||||||
|
durable original START owner. This guard rechecks its process-local
|
||||||
|
acquisition/runtime coordinates without consulting or refreshing any
|
||||||
|
device authority. The subsequent recovery path must still establish
|
||||||
|
a fresh DeviceInfo identity and MQTT control proof before it can
|
||||||
|
resume publication.
|
||||||
|
"""
|
||||||
|
|
||||||
|
intent = snapshot.intent
|
||||||
|
with self._lock:
|
||||||
|
lineage = self._active_stream_recovery_lineage
|
||||||
|
acquisition = self._acquisition
|
||||||
|
if lineage is None or acquisition is None:
|
||||||
|
return False
|
||||||
|
exact_local_lineage = bool(
|
||||||
|
self._active_stream_recovery_state == "reconnecting"
|
||||||
|
and self._active_stream_recovery_generation
|
||||||
|
== lineage.recovery_generation
|
||||||
|
and self._snapshot_runtime_id == lineage.snapshot_runtime_id
|
||||||
|
and acquisition.acquisition_id == lineage.acquisition_id
|
||||||
|
and acquisition.device_id == lineage.device_id
|
||||||
|
and acquisition.device_session_id == lineage.device_session_id
|
||||||
|
and acquisition.state == "acquiring"
|
||||||
|
and self._acquisition_start_operation_id
|
||||||
|
in {None, lineage.start_operation_id}
|
||||||
|
and self._acquisition_session_lease is not None
|
||||||
|
and self._acquisition_out_dir is not None
|
||||||
|
and self._acquisition_out_dir.name == lineage.evidence_session_id
|
||||||
|
and self._selected_device_id == lineage.transport_ref
|
||||||
|
and self._device_id == lineage.device_id
|
||||||
|
and self._device_session_id == lineage.device_session_id
|
||||||
|
and self._connection_mode == lineage.connection_mode
|
||||||
|
and self._k1_ip == lineage.target_ipv4
|
||||||
|
)
|
||||||
|
if not exact_local_lineage:
|
||||||
|
return False
|
||||||
|
runtime = self.runtime.snapshot()
|
||||||
|
return bool(
|
||||||
|
runtime.get("phase") == "reconnecting"
|
||||||
|
and runtime.get("source_mode") == "live"
|
||||||
|
and runtime.get("producer_generation")
|
||||||
|
== lineage.runtime_producer_generation
|
||||||
|
and intent is not None
|
||||||
|
and intent.intent_id == lineage.intent_id
|
||||||
|
and intent.requested_mode == lineage.connection_mode
|
||||||
|
and snapshot.device_network.state == "applied"
|
||||||
|
and snapshot.device_network.intent_id == lineage.intent_id
|
||||||
|
and snapshot.device_network.transport_ref == lineage.transport_ref
|
||||||
|
and snapshot.device_network.connection_mode == lineage.connection_mode
|
||||||
|
and snapshot.device_network.target == endpoint_target
|
||||||
|
and endpoint_target
|
||||||
|
== EndpointTarget(lineage.target_ipv4, lineage.target_port)
|
||||||
|
)
|
||||||
|
|
||||||
async def _monitor_host_path(self, target: EndpointTarget) -> HostPathProbeResult:
|
async def _monitor_host_path(self, target: EndpointTarget) -> HostPathProbeResult:
|
||||||
if not self._connection_monitor_contact_gate.acquire(blocking=False):
|
if not self._connection_monitor_contact_gate.acquire(blocking=False):
|
||||||
raise ConnectionMonitorProbeSuperseded(
|
raise ConnectionMonitorProbeSuperseded(
|
||||||
@@ -2715,6 +2796,57 @@ class XgridsK1CompatibilityService:
|
|||||||
)
|
)
|
||||||
self._connection_supervisor.record_monitor_failure("connection-monitor-start-failed")
|
self._connection_supervisor.record_monitor_failure("connection-monitor-start-failed")
|
||||||
|
|
||||||
|
async def _refresh_aged_verify_transport_owned(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
application_control_session: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
"""Refresh only a nearly expired Verify transport with real I/O.
|
||||||
|
|
||||||
|
The caller owns the Verify lifecycle/network fences, so the background
|
||||||
|
monitor is deliberately superseded. If DeviceInfo or physical
|
||||||
|
reconciliation consumed most of the existing endpoint lease, perform
|
||||||
|
the same route/TCP/route observation before releasing those fences.
|
||||||
|
No cached timestamp, MQTT reconnect, BLE operation, or device command
|
||||||
|
is manufactured here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
verified_binding = application_control_session.get("verified_control")
|
||||||
|
if not isinstance(verified_binding, Mapping):
|
||||||
|
raise ConnectionVerificationError(
|
||||||
|
"Verify lost its exact control proof before transport refresh",
|
||||||
|
reason_code="connection-verify-final-binding-superseded",
|
||||||
|
)
|
||||||
|
target = EndpointTarget(
|
||||||
|
str(verified_binding["target_ipv4"]),
|
||||||
|
int(verified_binding["target_port"]),
|
||||||
|
)
|
||||||
|
intent_id = str(verified_binding["intent_id"])
|
||||||
|
host_path_epoch = int(verified_binding["host_path_epoch"])
|
||||||
|
if self._connection_supervisor.endpoint_observation_has_remaining_lease(
|
||||||
|
target=target,
|
||||||
|
intent_id=intent_id,
|
||||||
|
host_path_epoch=host_path_epoch,
|
||||||
|
minimum_remaining_seconds=(
|
||||||
|
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS
|
||||||
|
),
|
||||||
|
):
|
||||||
|
return
|
||||||
|
observation = await _run_blocking_operation_without_abandonment(
|
||||||
|
self._probe_control_endpoint,
|
||||||
|
target.ipv4,
|
||||||
|
association_timeout_seconds=(CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS),
|
||||||
|
)
|
||||||
|
if not observation.reachable:
|
||||||
|
raise ConnectionVerificationError(
|
||||||
|
"K1 endpoint не подтвердил связь после долгой read-only сверки",
|
||||||
|
reason_code="connection-verify-mqtt-unreachable",
|
||||||
|
)
|
||||||
|
self._reconcile_connection_supervisor(
|
||||||
|
application_control_session,
|
||||||
|
self.runtime.snapshot(),
|
||||||
|
)
|
||||||
|
|
||||||
def _observe_connection_transport(
|
def _observe_connection_transport(
|
||||||
self,
|
self,
|
||||||
target: str,
|
target: str,
|
||||||
@@ -2723,9 +2855,17 @@ class XgridsK1CompatibilityService:
|
|||||||
reachable: bool | None,
|
reachable: bool | None,
|
||||||
reason_code: str | None = None,
|
reason_code: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
supervisor = self._connection_supervisor.snapshot()
|
|
||||||
intent = supervisor.intent
|
|
||||||
endpoint = EndpointTarget(validate_private_ipv4(target), CONTROL_MQTT_PORT)
|
endpoint = EndpointTarget(validate_private_ipv4(target), CONTROL_MQTT_PORT)
|
||||||
|
# A completed route sample is fresher than the silence fallback. Read
|
||||||
|
# the exact applied target without expiring the previous observation
|
||||||
|
# between sampling and publication; the CAS below still rejects an
|
||||||
|
# intent/target change and the sample itself decides route continuity.
|
||||||
|
supervisor = self._connection_supervisor.association_timeout_retention_candidate(
|
||||||
|
expected_target=endpoint,
|
||||||
|
)
|
||||||
|
if supervisor is None:
|
||||||
|
return
|
||||||
|
intent = supervisor.intent
|
||||||
if (
|
if (
|
||||||
intent is None
|
intent is None
|
||||||
or supervisor.device_network.state != "applied"
|
or supervisor.device_network.state != "applied"
|
||||||
@@ -2734,7 +2874,14 @@ class XgridsK1CompatibilityService:
|
|||||||
or supervisor.device_network.target != endpoint
|
or supervisor.device_network.target != endpoint
|
||||||
):
|
):
|
||||||
return
|
return
|
||||||
epoch = self._connection_supervisor.observe_host_path(path)
|
epoch = self._connection_supervisor.observe_host_path_if_current(
|
||||||
|
expected_intent_id=intent.intent_id,
|
||||||
|
expected_target=endpoint,
|
||||||
|
expected_host_path_observation=supervisor.host_path,
|
||||||
|
result=path,
|
||||||
|
)
|
||||||
|
if epoch is None:
|
||||||
|
return
|
||||||
if path.available and reachable is not None:
|
if path.available and reachable is not None:
|
||||||
self._connection_supervisor.observe_endpoint(
|
self._connection_supervisor.observe_endpoint(
|
||||||
target=endpoint,
|
target=endpoint,
|
||||||
@@ -3637,8 +3784,15 @@ class XgridsK1CompatibilityService:
|
|||||||
pending = self._pending_local_control_retirement
|
pending = self._pending_local_control_retirement
|
||||||
if not pending:
|
if not pending:
|
||||||
return True
|
return True
|
||||||
|
control = self._application_control_session.snapshot()
|
||||||
try:
|
try:
|
||||||
self._retire_application_control_for_network_change()
|
self._retire_application_control_for_network_change(
|
||||||
|
# A failed worker may finish retiring after the first bounded
|
||||||
|
# close attempt. Retrying that host-only retirement must not
|
||||||
|
# remain permanently blocked by the terminal phase itself.
|
||||||
|
# The durable physical-command row is intentionally untouched.
|
||||||
|
allow_terminal_failure=control.get("state") == "failed",
|
||||||
|
)
|
||||||
except (AttributeError, RuntimeError):
|
except (AttributeError, RuntimeError):
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
@@ -11616,6 +11770,21 @@ class XgridsK1CompatibilityService:
|
|||||||
and not reconciliation.observation.init_ready
|
and not reconciliation.observation.init_ready
|
||||||
and not reconciliation.observation.mqtt_retained
|
and not reconciliation.observation.mqtt_retained
|
||||||
)
|
)
|
||||||
|
ordinary_terminal_stop = bool(
|
||||||
|
record.stage == "resolved"
|
||||||
|
and record.resolution == "stop-standby-observed"
|
||||||
|
and record.publish_call_returned is True
|
||||||
|
and record.packet_id is not None
|
||||||
|
and record.qos2_completed
|
||||||
|
and record.application_response is not None
|
||||||
|
and record.application_response.success
|
||||||
|
and record.last_status is not None
|
||||||
|
and record.last_status.source == "live-control-session"
|
||||||
|
and record.last_status.session_state in {"ready", "scan_over"}
|
||||||
|
and not record.last_status.project_bound
|
||||||
|
and not record.last_status.init_ready
|
||||||
|
and not record.last_status.mqtt_retained
|
||||||
|
)
|
||||||
if not (
|
if not (
|
||||||
persisted is not None
|
persisted is not None
|
||||||
and status is not None
|
and status is not None
|
||||||
@@ -11624,6 +11793,7 @@ class XgridsK1CompatibilityService:
|
|||||||
unresolved_stop
|
unresolved_stop
|
||||||
or classified_terminal_stop
|
or classified_terminal_stop
|
||||||
or reconciled_ambiguous_terminal_stop
|
or reconciled_ambiguous_terminal_stop
|
||||||
|
or ordinary_terminal_stop
|
||||||
)
|
)
|
||||||
and record.acquisition_id == checkpoint.acquisition_id
|
and record.acquisition_id == checkpoint.acquisition_id
|
||||||
and record.identity.vendor_device_id_sha256
|
and record.identity.vendor_device_id_sha256
|
||||||
@@ -13107,6 +13277,75 @@ class XgridsK1CompatibilityService:
|
|||||||
return False
|
return False
|
||||||
ledger_snapshot = self._physical_command_ledger.snapshot()
|
ledger_snapshot = self._physical_command_ledger.snapshot()
|
||||||
record = ledger_snapshot.record
|
record = ledger_snapshot.record
|
||||||
|
ordinary_terminal_stop = bool(
|
||||||
|
ledger_snapshot.status == "resolved"
|
||||||
|
and record is not None
|
||||||
|
and record.action == "stop"
|
||||||
|
and record.stage == "resolved"
|
||||||
|
and record.resolution == "stop-standby-observed"
|
||||||
|
and record.publish_call_returned is True
|
||||||
|
and record.packet_id is not None
|
||||||
|
and record.qos2_completed
|
||||||
|
and record.application_response is not None
|
||||||
|
and record.application_response.success
|
||||||
|
and record.last_status is not None
|
||||||
|
and record.last_status.source == "live-control-session"
|
||||||
|
and record.last_status.session_state in {"ready", "scan_over"}
|
||||||
|
and not record.last_status.project_bound
|
||||||
|
and not record.last_status.init_ready
|
||||||
|
and not record.last_status.mqtt_retained
|
||||||
|
and self._checkpoint_binding_matches_physical_connection(
|
||||||
|
token.checkpoint.current_binding,
|
||||||
|
record,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if ordinary_terminal_stop:
|
||||||
|
assert record is not None and record.last_status is not None
|
||||||
|
store = self._active_acquisition_checkpoint
|
||||||
|
if store is None or not self._active_acquisition_checkpoint_trust_token_is_current(
|
||||||
|
token
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
binding = token.checkpoint.current_binding
|
||||||
|
status_proof = self._checkpoint_status_proof(
|
||||||
|
status=record.last_status,
|
||||||
|
binding=binding,
|
||||||
|
evidence_session_id=(
|
||||||
|
token.checkpoint.current_evidence_session_id
|
||||||
|
),
|
||||||
|
)
|
||||||
|
physical_proof = self._checkpoint_physical_proof(
|
||||||
|
record=record,
|
||||||
|
binding=binding,
|
||||||
|
checkpoint=token.checkpoint,
|
||||||
|
)
|
||||||
|
store.cease(
|
||||||
|
transition_id=self._active_acquisition_checkpoint_transition_id(
|
||||||
|
"restart-ordinary-stop-cease",
|
||||||
|
record.operation_id,
|
||||||
|
record.revision,
|
||||||
|
token.checkpoint_revision,
|
||||||
|
),
|
||||||
|
expected_revision=token.checkpoint_revision,
|
||||||
|
expected_acquisition_id=token.acquisition_id,
|
||||||
|
expected_start_operation_id=token.root_start_operation_id,
|
||||||
|
physical_proof=physical_proof,
|
||||||
|
status_proof=status_proof,
|
||||||
|
)
|
||||||
|
self._set_active_acquisition_checkpoint_reason(None)
|
||||||
|
return True
|
||||||
|
except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc:
|
||||||
|
self._set_active_acquisition_checkpoint_reason(
|
||||||
|
str(
|
||||||
|
getattr(
|
||||||
|
exc,
|
||||||
|
"reason_code",
|
||||||
|
"restart-ordinary-stop-checkpoint-settlement-failed",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return False
|
||||||
reconciliation = (
|
reconciliation = (
|
||||||
record.reconciliations[-1]
|
record.reconciliations[-1]
|
||||||
if record is not None and record.reconciliations
|
if record is not None and record.reconciliations
|
||||||
@@ -14314,8 +14553,8 @@ class XgridsK1CompatibilityService:
|
|||||||
self._last_live_data_suspend_aware = time.time()
|
self._last_live_data_suspend_aware = time.time()
|
||||||
self._last_live_data_session_id = session_id
|
self._last_live_data_session_id = session_id
|
||||||
|
|
||||||
physical = self._physical_command_coordinator.snapshot()
|
now = time.monotonic()
|
||||||
control = self._application_control_session.snapshot()
|
runtime_recovery = runtime.get("connection_recovery")
|
||||||
with self._lock:
|
with self._lock:
|
||||||
acquisition = self._acquisition
|
acquisition = self._acquisition
|
||||||
out_dir = self._acquisition_out_dir
|
out_dir = self._acquisition_out_dir
|
||||||
@@ -14328,6 +14567,35 @@ class XgridsK1CompatibilityService:
|
|||||||
return
|
return
|
||||||
acquisition_id = acquisition.acquisition_id
|
acquisition_id = acquisition.acquisition_id
|
||||||
control_mode = acquisition.control_mode
|
control_mode = acquisition.control_mode
|
||||||
|
lineage = (acquisition_id, out_dir.name, producer_generation)
|
||||||
|
active_recovery_lineage = self._active_stream_recovery_lineage
|
||||||
|
post_recovery_restart_candidate = bool(
|
||||||
|
control_mode == "plugin-commanded"
|
||||||
|
and active_recovery_lineage is not None
|
||||||
|
and active_recovery_lineage.acquisition_id == acquisition_id
|
||||||
|
and active_recovery_lineage.evidence_session_id == out_dir.name
|
||||||
|
and active_recovery_lineage.runtime_producer_generation
|
||||||
|
== producer_generation
|
||||||
|
and isinstance(runtime_recovery, Mapping)
|
||||||
|
and runtime_recovery.get("state") == "recovered"
|
||||||
|
)
|
||||||
|
if not post_recovery_restart_candidate and (
|
||||||
|
self._camera_activation_lineage == lineage
|
||||||
|
or (
|
||||||
|
self._camera_activation_retry_lineage == lineage
|
||||||
|
and now < self._camera_activation_retry_not_before_monotonic
|
||||||
|
)
|
||||||
|
):
|
||||||
|
# The exact lineage already owns its one camera activation (or
|
||||||
|
# its bounded local retry delay). Repeating the durable physical
|
||||||
|
# ledger/checkpoint admission for every 10 Hz PCL blocks this
|
||||||
|
# publisher thread and evicts intervening pose frames. The
|
||||||
|
# data-plane liveness edge above remains per-frame; recovery
|
||||||
|
# deliberately bypasses this ordinary idempotence fast path.
|
||||||
|
return
|
||||||
|
|
||||||
|
physical = self._physical_command_coordinator.snapshot()
|
||||||
|
control = self._application_control_session.snapshot()
|
||||||
|
|
||||||
if control_mode == "plugin-commanded":
|
if control_mode == "plugin-commanded":
|
||||||
start_operation_id = (
|
start_operation_id = (
|
||||||
@@ -14364,21 +14632,10 @@ class XgridsK1CompatibilityService:
|
|||||||
# this local admission after the durable SCANNING proof arrives.
|
# this local admission after the durable SCANNING proof arrives.
|
||||||
return
|
return
|
||||||
|
|
||||||
lineage = (acquisition_id, out_dir.name, producer_generation)
|
|
||||||
now = time.monotonic()
|
|
||||||
camera = self.camera_preview.snapshot()
|
camera = self.camera_preview.snapshot()
|
||||||
with self._lock:
|
|
||||||
active_recovery_lineage = self._active_stream_recovery_lineage
|
|
||||||
runtime_recovery = runtime.get("connection_recovery")
|
|
||||||
if (
|
if (
|
||||||
control_mode == "plugin-commanded"
|
post_recovery_restart_candidate
|
||||||
and active_recovery_lineage is not None
|
and active_recovery_lineage is not None
|
||||||
and active_recovery_lineage.acquisition_id == acquisition_id
|
|
||||||
and active_recovery_lineage.evidence_session_id == out_dir.name
|
|
||||||
and active_recovery_lineage.runtime_producer_generation
|
|
||||||
== producer_generation
|
|
||||||
and isinstance(runtime_recovery, Mapping)
|
|
||||||
and runtime_recovery.get("state") == "recovered"
|
|
||||||
and self._enqueue_post_recovery_camera_restart(
|
and self._enqueue_post_recovery_camera_restart(
|
||||||
active_recovery_lineage,
|
active_recovery_lineage,
|
||||||
runtime=runtime,
|
runtime=runtime,
|
||||||
@@ -14888,6 +15145,10 @@ class XgridsK1CompatibilityService:
|
|||||||
or control_after.get("session_generation")
|
or control_after.get("session_generation")
|
||||||
!= control_before.get("session_generation")
|
!= control_before.get("session_generation")
|
||||||
)
|
)
|
||||||
|
control_stage = "post-device-info-transport-refresh"
|
||||||
|
await self._refresh_aged_verify_transport_owned(
|
||||||
|
application_control_session=control_after,
|
||||||
|
)
|
||||||
control_stage = "physical-reconciliation"
|
control_stage = "physical-reconciliation"
|
||||||
physical_reconciliation = await self._reconcile_physical_command_after_verify_owned(
|
physical_reconciliation = await self._reconcile_physical_command_after_verify_owned(
|
||||||
verify_operation_id=verify_operation_id,
|
verify_operation_id=verify_operation_id,
|
||||||
@@ -14895,6 +15156,12 @@ class XgridsK1CompatibilityService:
|
|||||||
if provisional_topology is not None:
|
if provisional_topology is not None:
|
||||||
control_stage = "semantic-topology-commit"
|
control_stage = "semantic-topology-commit"
|
||||||
self._commit_provisional_fresh_bridge_topology(provisional_topology)
|
self._commit_provisional_fresh_bridge_topology(provisional_topology)
|
||||||
|
control_stage = "post-reconciliation-transport-refresh"
|
||||||
|
await self._refresh_aged_verify_transport_owned(
|
||||||
|
application_control_session=dict(
|
||||||
|
self._application_control_session.snapshot()
|
||||||
|
),
|
||||||
|
)
|
||||||
control_stage = "final-control-binding"
|
control_stage = "final-control-binding"
|
||||||
ready = self._connection_supervisor.snapshot()
|
ready = self._connection_supervisor.snapshot()
|
||||||
if not (
|
if not (
|
||||||
@@ -23267,6 +23534,10 @@ class XgridsK1CompatibilityService:
|
|||||||
)
|
)
|
||||||
use_reconciliation = bool(
|
use_reconciliation = bool(
|
||||||
reconciliation is not None
|
reconciliation is not None
|
||||||
|
and self._physical_reconciliation_belongs_to_record(
|
||||||
|
reconciliation,
|
||||||
|
record,
|
||||||
|
)
|
||||||
and reconciliation.kind
|
and reconciliation.kind
|
||||||
in {
|
in {
|
||||||
"ambiguous-outcome",
|
"ambiguous-outcome",
|
||||||
@@ -24458,6 +24729,9 @@ class XgridsK1CompatibilityService:
|
|||||||
observation = await _run_blocking_operation_without_abandonment(
|
observation = await _run_blocking_operation_without_abandonment(
|
||||||
self._probe_control_endpoint,
|
self._probe_control_endpoint,
|
||||||
lineage.target_ipv4,
|
lineage.target_ipv4,
|
||||||
|
association_timeout_seconds=(
|
||||||
|
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS
|
||||||
|
),
|
||||||
)
|
)
|
||||||
if not observation.path.available or not observation.reachable:
|
if not observation.path.available or not observation.reachable:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -27846,6 +28120,15 @@ class XgridsK1CompatibilityService:
|
|||||||
self._physical_command_coordinator.snapshot()
|
self._physical_command_coordinator.snapshot()
|
||||||
) is None:
|
) is None:
|
||||||
return "not-applicable"
|
return "not-applicable"
|
||||||
|
# A live START worker cannot be settled. Avoid touching the exact
|
||||||
|
# mark_dispatching -> publish fence on every passive state read while
|
||||||
|
# it is still initializing; the terminal branch below re-samples the
|
||||||
|
# worker under that fence before committing any no-dispatch result.
|
||||||
|
if self._application_control_session.snapshot().get("state") not in {
|
||||||
|
"failed",
|
||||||
|
"closed",
|
||||||
|
}:
|
||||||
|
return "not-applicable"
|
||||||
if not self._k1_command_dispatch_gate.acquire(blocking=False):
|
if not self._k1_command_dispatch_gate.acquire(blocking=False):
|
||||||
return "defer"
|
return "defer"
|
||||||
try:
|
try:
|
||||||
@@ -28371,6 +28654,59 @@ class XgridsK1CompatibilityService:
|
|||||||
and durable_start_active_confirmed
|
and durable_start_active_confirmed
|
||||||
and durable_checkpoint_active_confirmed
|
and durable_checkpoint_active_confirmed
|
||||||
)
|
)
|
||||||
|
terminal_control_failure = (
|
||||||
|
terminal_control_proof.get("failure")
|
||||||
|
if isinstance(terminal_control_proof, Mapping)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
terminal_control_transport = (
|
||||||
|
terminal_control_proof.get("transport")
|
||||||
|
if isinstance(terminal_control_proof, Mapping)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
terminal_verified_control = (
|
||||||
|
terminal_control_proof.get("verified_control")
|
||||||
|
if isinstance(terminal_control_proof, Mapping)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
device_reported_scan_over_without_stop = bool(
|
||||||
|
current is not None
|
||||||
|
and current.control_mode == "plugin-commanded"
|
||||||
|
and current.state in {"starting", "awaiting_external_start", "acquiring"}
|
||||||
|
and current_stop_operation_id is None
|
||||||
|
and isinstance(terminal_control_proof, Mapping)
|
||||||
|
and terminal_control_proof.get("state") == "failed"
|
||||||
|
and isinstance(terminal_control_failure, Mapping)
|
||||||
|
and terminal_control_failure.get("failed_phase") == "scanning"
|
||||||
|
and terminal_control_failure.get("modeling_command_attempted") is True
|
||||||
|
and terminal_control_failure.get("stop_command_attempted") is False
|
||||||
|
and terminal_control_failure.get("diagnostic_snapshot_unavailable") == []
|
||||||
|
and terminal_control_failure.get("diagnostic_evidence_unavailable") == []
|
||||||
|
and isinstance(terminal_control_transport, Mapping)
|
||||||
|
and isinstance(terminal_control_transport.get("device_status_reports"), int)
|
||||||
|
and not isinstance(
|
||||||
|
terminal_control_transport.get("device_status_reports"),
|
||||||
|
bool,
|
||||||
|
)
|
||||||
|
and int(terminal_control_transport["device_status_reports"]) > 0
|
||||||
|
and terminal_control_transport.get("latest_device_session_state")
|
||||||
|
== "scan_over"
|
||||||
|
and terminal_control_transport.get("latest_system_error_code") is None
|
||||||
|
and isinstance(physical_command_proof, Mapping)
|
||||||
|
and physical_command_proof.get("runtime_bound") is True
|
||||||
|
and physical_command_proof.get("reconciliation_ready") is True
|
||||||
|
and physical_command_proof.get("observed_session_state") == "scan_over"
|
||||||
|
and self._matching_start_active_confirmed(
|
||||||
|
physical_command_proof,
|
||||||
|
acquisition_id=current_acquisition_id,
|
||||||
|
start_operation_id=canonical_start_operation_id,
|
||||||
|
verified_control=(
|
||||||
|
terminal_verified_control
|
||||||
|
if isinstance(terminal_verified_control, Mapping)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
# Camera-only transport loss is supervised by the backend producer
|
# Camera-only transport loss is supervised by the backend producer
|
||||||
# watchdog and exact camera CAS. Snapshot polling is observational: it
|
# watchdog and exact camera CAS. Snapshot polling is observational: it
|
||||||
# must not move an otherwise live MQTT/PCL runtime to ``reconnecting``.
|
# must not move an otherwise live MQTT/PCL runtime to ``reconnecting``.
|
||||||
@@ -28459,6 +28795,21 @@ class XgridsK1CompatibilityService:
|
|||||||
and stop_response_accepted_without_terminal_status
|
and stop_response_accepted_without_terminal_status
|
||||||
and self._operations.deadline_reached(current_stop_operation_id)
|
and self._operations.deadline_reached(current_stop_operation_id)
|
||||||
)
|
)
|
||||||
|
terminal_unknown_stop_control = bool(
|
||||||
|
current is not None
|
||||||
|
and current.control_mode == "plugin-commanded"
|
||||||
|
and current.state == "awaiting_external_stop"
|
||||||
|
and application_control_session.get("state") == "failed"
|
||||||
|
and application_control_session.get("outcome_unknown") is True
|
||||||
|
and isinstance(application_control_session.get("failure"), Mapping)
|
||||||
|
and application_control_session["failure"].get("stop_command_attempted") is True
|
||||||
|
and self._matching_unresolved_stop_stage(
|
||||||
|
physical_command_proof,
|
||||||
|
acquisition_id=current_acquisition_id,
|
||||||
|
stop_operation_id=current_stop_operation_id,
|
||||||
|
)
|
||||||
|
is not None
|
||||||
|
)
|
||||||
with self._lock:
|
with self._lock:
|
||||||
canonical_stop_standby_confirmed = bool(
|
canonical_stop_standby_confirmed = bool(
|
||||||
checkpoint_stop_completion_seen and checkpoint_stop_ceased
|
checkpoint_stop_completion_seen and checkpoint_stop_ceased
|
||||||
@@ -28504,7 +28855,11 @@ class XgridsK1CompatibilityService:
|
|||||||
and stop_response_deadline_reached
|
and stop_response_deadline_reached
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
if stop_response_deadline_reached:
|
if terminal_unknown_stop_control or stop_response_deadline_reached:
|
||||||
|
# Retire only the failed host-owned control socket. In particular,
|
||||||
|
# an UNKNOWN physical STOP keeps the capture, viewer and acquisition
|
||||||
|
# alive so an explicit read-only DeviceInfo/DeviceStatus dialogue
|
||||||
|
# can classify SCANNING or READY without a second command.
|
||||||
self._retire_local_control_after_stop_timeout()
|
self._retire_local_control_after_stop_timeout()
|
||||||
if terminal_unknown_stop_cleanup:
|
if terminal_unknown_stop_cleanup:
|
||||||
try:
|
try:
|
||||||
@@ -28858,6 +29213,39 @@ class XgridsK1CompatibilityService:
|
|||||||
# read-only classification and seal the old STOP as
|
# read-only classification and seal the old STOP as
|
||||||
# non-replayable/none.
|
# non-replayable/none.
|
||||||
pass
|
pass
|
||||||
|
elif device_reported_scan_over_without_stop:
|
||||||
|
# This is not an inference from a quiet camera or point stream.
|
||||||
|
# The continuously DeviceInfo-bound control socket decoded a
|
||||||
|
# fresh, non-retained SCAN_OVER from this exact K1, and the
|
||||||
|
# physical coordinator independently retained the same bound
|
||||||
|
# status. No canonical STOP was requested or attempted.
|
||||||
|
#
|
||||||
|
# SCAN_OVER proves that this acquisition cannot be resumed. It
|
||||||
|
# does not prove READY and does not authorize a new START; the
|
||||||
|
# durable physical/checkpoint chain remains fenced until a later
|
||||||
|
# explicit read-only reconciliation observes the returned K1.
|
||||||
|
acquisition.transition(
|
||||||
|
"finalizing",
|
||||||
|
message_code="acquisition.recovery.device_standby_observed",
|
||||||
|
)
|
||||||
|
camera_terminal_status = "interrupted"
|
||||||
|
camera_failure_code = "device-reported-scan-over"
|
||||||
|
recovery_standby_after_seal = True
|
||||||
|
if point_frames <= 0:
|
||||||
|
recovery_standby_start_operation_id = (
|
||||||
|
self._acquisition_start_operation_id
|
||||||
|
)
|
||||||
|
completion_message_code = (
|
||||||
|
"acquisition.recovery.device_standby_observed"
|
||||||
|
)
|
||||||
|
completion_result = {
|
||||||
|
"receiver_stopped": True,
|
||||||
|
"device_state": "scan_over",
|
||||||
|
"device_stop": "not-sent",
|
||||||
|
"automatic_command_retry": False,
|
||||||
|
"read_only_recovery": True,
|
||||||
|
"physical_reconciliation_required": True,
|
||||||
|
}
|
||||||
elif prepared_stop_recovery_standby:
|
elif prepared_stop_recovery_standby:
|
||||||
acquisition.transition(
|
acquisition.transition(
|
||||||
"finalizing",
|
"finalizing",
|
||||||
@@ -29815,6 +30203,28 @@ class XgridsK1PluginFacade:
|
|||||||
|
|
||||||
def __init__(self, service: XgridsK1ServicePort) -> None:
|
def __init__(self, service: XgridsK1ServicePort) -> None:
|
||||||
self.service = service
|
self.service = service
|
||||||
|
self._state_read_task: asyncio.Task[dict[str, Any]] | None = None
|
||||||
|
|
||||||
|
def _retire_state_read_task(self, task: asyncio.Task[dict[str, Any]]) -> None:
|
||||||
|
if self._state_read_task is task:
|
||||||
|
self._state_read_task = None
|
||||||
|
with suppress(asyncio.CancelledError):
|
||||||
|
task.exception()
|
||||||
|
|
||||||
|
async def _read_state_single_flight(self) -> dict[str, Any]:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
task = self._state_read_task
|
||||||
|
if task is None or task.done() or task.get_loop() is not loop:
|
||||||
|
task = loop.create_task(
|
||||||
|
asyncio.to_thread(self.service.state),
|
||||||
|
name="xgrids-k1-state-read",
|
||||||
|
)
|
||||||
|
self._state_read_task = task
|
||||||
|
task.add_done_callback(self._retire_state_read_task)
|
||||||
|
# An HTTP disconnect cancels only that request waiter. The shared
|
||||||
|
# read keeps running so a later poll joins it instead of stacking a
|
||||||
|
# second service.state thread behind the lifecycle gate.
|
||||||
|
return await asyncio.shield(task)
|
||||||
|
|
||||||
async def invoke(self, invocation: RuntimeActionInvocation) -> dict[str, Any]:
|
async def invoke(self, invocation: RuntimeActionInvocation) -> dict[str, Any]:
|
||||||
if invocation.plugin_id != self.plugin_id:
|
if invocation.plugin_id != self.plugin_id:
|
||||||
@@ -29887,7 +30297,7 @@ class XgridsK1PluginFacade:
|
|||||||
self.service.bind_runtime_event_loop(asyncio.get_running_loop())
|
self.service.bind_runtime_event_loop(asyncio.get_running_loop())
|
||||||
if action_id == ACTION_STATE_READ:
|
if action_id == ACTION_STATE_READ:
|
||||||
EmptyRequest.model_validate(payload)
|
EmptyRequest.model_validate(payload)
|
||||||
return await asyncio.to_thread(self.service.state)
|
return await self._read_state_single_flight()
|
||||||
if action_id == ACTION_DISCOVERY_SCAN:
|
if action_id == ACTION_DISCOVERY_SCAN:
|
||||||
scan_request = BleScanRequest.model_validate(payload)
|
scan_request = BleScanRequest.model_validate(payload)
|
||||||
return await self.service.scan_ble(scan_request)
|
return await self.service.scan_ble(scan_request)
|
||||||
@@ -29899,7 +30309,7 @@ class XgridsK1PluginFacade:
|
|||||||
EmptyRequest.model_validate(payload)
|
EmptyRequest.model_validate(payload)
|
||||||
if action_id == ACTION_DEVICE_INSPECT:
|
if action_id == ACTION_DEVICE_INSPECT:
|
||||||
return await asyncio.to_thread(self.service.inspect_device)
|
return await asyncio.to_thread(self.service.inspect_device)
|
||||||
return await asyncio.to_thread(self.service.state)
|
return await self._read_state_single_flight()
|
||||||
if action_id == ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ:
|
if action_id == ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ:
|
||||||
EmptyRequest.model_validate(payload)
|
EmptyRequest.model_validate(payload)
|
||||||
return await asyncio.to_thread(self.service.read_device_calibration_snapshot)
|
return await asyncio.to_thread(self.service.read_device_calibration_snapshot)
|
||||||
@@ -29923,7 +30333,7 @@ class XgridsK1PluginFacade:
|
|||||||
return await self.service.probe_configured_endpoint(probe_request)
|
return await self.service.probe_configured_endpoint(probe_request)
|
||||||
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_STATE:
|
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_STATE:
|
||||||
EmptyRequest.model_validate(payload)
|
EmptyRequest.model_validate(payload)
|
||||||
return await asyncio.to_thread(self.service.state)
|
return await self._read_state_single_flight()
|
||||||
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_ARM:
|
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_ARM:
|
||||||
arm_request = ShadowApplicationControlArmRequest.model_validate(payload)
|
arm_request = ShadowApplicationControlArmRequest.model_validate(payload)
|
||||||
return await asyncio.to_thread(
|
return await asyncio.to_thread(
|
||||||
|
|||||||
@@ -696,6 +696,7 @@ class VisualizationRuntime:
|
|||||||
|
|
||||||
def publish_stream(bridge: RerunBridge) -> None:
|
def publish_stream(bridge: RerunBridge) -> None:
|
||||||
serve_perception_next = False
|
serve_perception_next = False
|
||||||
|
serve_non_point_cloud_next = False
|
||||||
while (
|
while (
|
||||||
not source_done.is_set()
|
not source_done.is_set()
|
||||||
or not preview_empty()
|
or not preview_empty()
|
||||||
@@ -708,14 +709,15 @@ class VisualizationRuntime:
|
|||||||
and preview_empty()
|
and preview_empty()
|
||||||
):
|
):
|
||||||
return
|
return
|
||||||
# A dedicated latest-PCL slot plus this fixed service order
|
# The latest-PCL slot preserves visible-scene recovery, but a
|
||||||
# guarantees visible-scene progress even under an unbounded
|
# continuously full slot must not starve pose. Alternate one
|
||||||
# stream of perception results. Other MQTT preview traffic is
|
# live PCL with one available non-PCL item; when no other item
|
||||||
# then served before one perception result, so neither class can
|
# exists, the fallback below keeps PCL flowing without delay.
|
||||||
# monopolize the single publisher thread.
|
if not serve_non_point_cloud_next:
|
||||||
live_point_cloud = pop_latest_live_point_cloud()
|
live_point_cloud = pop_latest_live_point_cloud()
|
||||||
if live_point_cloud is not None:
|
if live_point_cloud is not None:
|
||||||
publish_message(bridge, live_point_cloud)
|
publish_message(bridge, live_point_cloud)
|
||||||
|
serve_non_point_cloud_next = True
|
||||||
continue
|
continue
|
||||||
if serve_perception_next:
|
if serve_perception_next:
|
||||||
try:
|
try:
|
||||||
@@ -728,6 +730,7 @@ class VisualizationRuntime:
|
|||||||
finally:
|
finally:
|
||||||
self._perception_messages.task_done()
|
self._perception_messages.task_done()
|
||||||
serve_perception_next = False
|
serve_perception_next = False
|
||||||
|
serve_non_point_cloud_next = False
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
message = messages.get_nowait()
|
message = messages.get_nowait()
|
||||||
@@ -739,6 +742,7 @@ class VisualizationRuntime:
|
|||||||
finally:
|
finally:
|
||||||
messages.task_done()
|
messages.task_done()
|
||||||
serve_perception_next = True
|
serve_perception_next = True
|
||||||
|
serve_non_point_cloud_next = False
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
perception = self._perception_messages.get_nowait()
|
perception = self._perception_messages.get_nowait()
|
||||||
@@ -750,6 +754,12 @@ class VisualizationRuntime:
|
|||||||
finally:
|
finally:
|
||||||
self._perception_messages.task_done()
|
self._perception_messages.task_done()
|
||||||
serve_perception_next = False
|
serve_perception_next = False
|
||||||
|
serve_non_point_cloud_next = False
|
||||||
|
continue
|
||||||
|
live_point_cloud = pop_latest_live_point_cloud()
|
||||||
|
if live_point_cloud is not None:
|
||||||
|
publish_message(bridge, live_point_cloud)
|
||||||
|
serve_non_point_cloud_next = True
|
||||||
continue
|
continue
|
||||||
live_point_cloud_ready.wait(timeout=0.05)
|
live_point_cloud_ready.wait(timeout=0.05)
|
||||||
|
|
||||||
|
|||||||
+49
-1
@@ -355,12 +355,50 @@ def newly_finalized_recording_ids(
|
|||||||
return tuple(sorted(set(current_finalized) - known_finalized))
|
return tuple(sorted(set(current_finalized) - known_finalized))
|
||||||
|
|
||||||
|
|
||||||
|
def observation_archive_revision(roots: Iterable[Path]) -> tuple[tuple[object, ...], ...]:
|
||||||
|
"""Return a cheap change fence for direct-child observation archives.
|
||||||
|
|
||||||
|
The K1 writer creates and retires ``.current_session`` in the archive root,
|
||||||
|
and every session directory is a direct child of that same root. Those
|
||||||
|
operations advance the directory metadata, while growing capture/media
|
||||||
|
files do not. This lets the reconciler notice session start, completion and
|
||||||
|
removal without recursively reopening tens of thousands of immutable
|
||||||
|
evidence files every two seconds.
|
||||||
|
"""
|
||||||
|
|
||||||
|
revisions: list[tuple[object, ...]] = []
|
||||||
|
for configured_root in roots:
|
||||||
|
root = configured_root.expanduser().resolve(strict=False)
|
||||||
|
try:
|
||||||
|
metadata = root.lstat()
|
||||||
|
except OSError:
|
||||||
|
revisions.append((str(root), None))
|
||||||
|
continue
|
||||||
|
revisions.append(
|
||||||
|
(
|
||||||
|
str(root),
|
||||||
|
metadata.st_dev,
|
||||||
|
metadata.st_ino,
|
||||||
|
metadata.st_mtime_ns,
|
||||||
|
metadata.st_ctime_ns,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
revisions.sort(key=lambda item: str(item[0]))
|
||||||
|
return tuple(revisions)
|
||||||
|
|
||||||
|
|
||||||
async def _recording_preparation_reconciler() -> None:
|
async def _recording_preparation_reconciler() -> None:
|
||||||
"""Prepare sessions finalized during this process, never historical rows."""
|
"""Prepare sessions finalized during this process, never historical rows."""
|
||||||
|
|
||||||
known_finalized: set[str] | None = None
|
known_finalized: set[str] | None = None
|
||||||
|
reconciled_archive_revision: tuple[tuple[object, ...], ...] | None = None
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
|
archive_revision = await asyncio.to_thread(
|
||||||
|
observation_archive_revision,
|
||||||
|
(archive.root for archive in plugin_environment.observation_archives),
|
||||||
|
)
|
||||||
|
if archive_revision != reconciled_archive_revision:
|
||||||
await asyncio.to_thread(refresh_observation_catalog)
|
await asyncio.to_thread(refresh_observation_catalog)
|
||||||
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
||||||
newly_finalized = newly_finalized_recording_ids(
|
newly_finalized = newly_finalized_recording_ids(
|
||||||
@@ -373,6 +411,10 @@ async def _recording_preparation_reconciler() -> None:
|
|||||||
newly_finalized,
|
newly_finalized,
|
||||||
)
|
)
|
||||||
known_finalized = finalized
|
known_finalized = finalized
|
||||||
|
# Keep the pre-scan revision. If a writer changed the archive
|
||||||
|
# during the expensive discovery, the next two-second check
|
||||||
|
# observes the newer revision and performs one follow-up pass.
|
||||||
|
reconciled_archive_revision = archive_revision
|
||||||
recording_reconciler_readiness.record_success()
|
recording_reconciler_readiness.record_success()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
# A transient filesystem/catalog failure must not permanently
|
# A transient filesystem/catalog failure must not permanently
|
||||||
@@ -496,6 +538,12 @@ _CLOSED_WEBSOCKET_SEND_ERRORS = frozenset(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# This stream carries the comparatively heavy control-state snapshot, not
|
||||||
|
# camera or point-cloud media. Action requests return their own result, and the
|
||||||
|
# frontend also keeps a four-second REST fallback, so a two-second passive
|
||||||
|
# cadence preserves status recovery without starving the live media sockets.
|
||||||
|
DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS = 2.0
|
||||||
|
|
||||||
|
|
||||||
def _is_closed_websocket_send_error(exc: RuntimeError) -> bool:
|
def _is_closed_websocket_send_error(exc: RuntimeError) -> bool:
|
||||||
message = " ".join(str(exc).split())
|
message = " ".join(str(exc).split())
|
||||||
@@ -528,7 +576,7 @@ async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
|
|||||||
if _is_closed_websocket_send_error(exc):
|
if _is_closed_websocket_send_error(exc):
|
||||||
return
|
return
|
||||||
raise
|
raise
|
||||||
await asyncio.sleep(0.5)
|
await asyncio.sleep(DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS)
|
||||||
except (PluginNotFoundError, PluginActionNotFoundError):
|
except (PluginNotFoundError, PluginActionNotFoundError):
|
||||||
await websocket.close(code=1008, reason="Device plugin is not available")
|
await websocket.close(code=1008, reason="Device plugin is not available")
|
||||||
except (PluginExecutionError, PluginRuntimeUnavailableError):
|
except (PluginExecutionError, PluginRuntimeUnavailableError):
|
||||||
|
|||||||
@@ -326,6 +326,7 @@ def test_live_rerun_process_and_close_failure_preserve_capture_then_recover(
|
|||||||
allow_second_bridge = threading.Event()
|
allow_second_bridge = threading.Event()
|
||||||
second_bridge_ready = threading.Event()
|
second_bridge_ready = threading.Event()
|
||||||
recovery_confirmed = threading.Event()
|
recovery_confirmed = threading.Event()
|
||||||
|
raw_capture_completed = threading.Event()
|
||||||
raw_sequences: list[int] = []
|
raw_sequences: list[int] = []
|
||||||
should_stop_before_explicit_stop: list[bool] = []
|
should_stop_before_explicit_stop: list[bool] = []
|
||||||
confirmed_attempts: list[int] = []
|
confirmed_attempts: list[int] = []
|
||||||
@@ -389,6 +390,7 @@ def test_live_rerun_process_and_close_failure_preserve_capture_then_recover(
|
|||||||
assert second_bridge_ready.wait(timeout=2.0)
|
assert second_bridge_ready.wait(timeout=2.0)
|
||||||
raw_sequences.append(10)
|
raw_sequences.append(10)
|
||||||
enqueue(_captured_live_point_cloud(10)) # type: ignore[operator]
|
enqueue(_captured_live_point_cloud(10)) # type: ignore[operator]
|
||||||
|
raw_capture_completed.set()
|
||||||
assert recovery_confirmed.wait(timeout=2.0)
|
assert recovery_confirmed.wait(timeout=2.0)
|
||||||
|
|
||||||
while not should_stop(): # type: ignore[operator]
|
while not should_stop(): # type: ignore[operator]
|
||||||
@@ -409,6 +411,7 @@ def test_live_rerun_process_and_close_failure_preserve_capture_then_recover(
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert recovery_confirmed.wait(timeout=3.0)
|
assert recovery_confirmed.wait(timeout=3.0)
|
||||||
|
assert raw_capture_completed.wait(timeout=3.0)
|
||||||
snapshot = runtime.snapshot()
|
snapshot = runtime.snapshot()
|
||||||
assert snapshot["phase"] == "live"
|
assert snapshot["phase"] == "live"
|
||||||
assert snapshot["source_ready"] is True
|
assert snapshot["source_ready"] is True
|
||||||
@@ -510,6 +513,66 @@ def test_live_latest_point_cloud_survives_pose_and_perception_pressure(
|
|||||||
runtime.close()
|
runtime.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_pose_is_published_during_continuous_point_cloud_pressure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
producer_started = threading.Event()
|
||||||
|
producer_finished = threading.Event()
|
||||||
|
pose_published_during_pressure = threading.Event()
|
||||||
|
|
||||||
|
class SlowPointCloudBridge(MetricRuntimeBridgeStub):
|
||||||
|
def process(self, envelope: object) -> None:
|
||||||
|
if isinstance(envelope, DecodedPointCloudView):
|
||||||
|
# Keep another PCL waiting in the latest-wins slot. The
|
||||||
|
# scheduler must still admit pose while pressure continues.
|
||||||
|
time.sleep(0.01)
|
||||||
|
elif (
|
||||||
|
isinstance(envelope, DecodedPoseView)
|
||||||
|
and not producer_finished.is_set()
|
||||||
|
):
|
||||||
|
pose_published_during_pressure.set()
|
||||||
|
super().process(envelope)
|
||||||
|
|
||||||
|
def fake_capture_mqtt(
|
||||||
|
_host: str,
|
||||||
|
_out_dir: Path,
|
||||||
|
**callbacks: object,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
callbacks["on_clock_established"]() # type: ignore[operator]
|
||||||
|
callbacks["on_ready"]() # type: ignore[operator]
|
||||||
|
enqueue = callbacks["on_message_recorded"]
|
||||||
|
producer_started.set()
|
||||||
|
for index in range(300):
|
||||||
|
enqueue(_captured_live_point_cloud(index * 2 + 1)) # type: ignore[operator]
|
||||||
|
enqueue(_captured_live_pose(index * 2 + 2)) # type: ignore[operator]
|
||||||
|
time.sleep(0.001)
|
||||||
|
producer_finished.set()
|
||||||
|
|
||||||
|
should_stop = callbacks["should_stop"]
|
||||||
|
while not should_stop(): # type: ignore[operator]
|
||||||
|
time.sleep(0.005)
|
||||||
|
return {"message_count": 600}
|
||||||
|
|
||||||
|
monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt)
|
||||||
|
runtime = VisualizationRuntime(
|
||||||
|
bridge_factory=lambda **kwargs: SlowPointCloudBridge(kwargs["metrics"]), # type: ignore[arg-type]
|
||||||
|
normalizer=normalize_k1_message,
|
||||||
|
)
|
||||||
|
runtime.start_live(
|
||||||
|
"192.168.1.50",
|
||||||
|
tmp_path / "pose-fairness",
|
||||||
|
duration_seconds=None,
|
||||||
|
project_name="POSEFAIRNESS001",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert producer_started.wait(timeout=2.0)
|
||||||
|
assert producer_finished.wait(timeout=2.0)
|
||||||
|
runtime.stop(wait_seconds=2.0)
|
||||||
|
runtime.close()
|
||||||
|
assert pose_published_during_pressure.is_set()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("attempt", [5, 1025, 10**100])
|
@pytest.mark.parametrize("attempt", [5, 1025, 10**100])
|
||||||
def test_live_rerun_recovery_backoff_saturates(attempt: int) -> None:
|
def test_live_rerun_recovery_backoff_saturates(attempt: int) -> None:
|
||||||
assert runtime_module._rerun_recovery_backoff_seconds(attempt) == 5.0 # noqa: SLF001
|
assert runtime_module._rerun_recovery_backoff_seconds(attempt) == 5.0 # noqa: SLF001
|
||||||
|
|||||||
@@ -1817,7 +1817,7 @@ def test_same_path_positive_reducer_observation_resets_technical_failure_streak(
|
|||||||
asyncio.run(scenario())
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
|
||||||
def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak() -> None:
|
def test_unrelated_supervisor_revisions_preserve_exact_configured_route() -> None:
|
||||||
async def scenario() -> None:
|
async def scenario() -> None:
|
||||||
supervisor, epoch = _configured_unverified_supervisor()
|
supervisor, epoch = _configured_unverified_supervisor()
|
||||||
path = _available_path()
|
path = _available_path()
|
||||||
@@ -1836,8 +1836,12 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
|||||||
async def host_probe(_target: EndpointTarget) -> HostPathProbeResult:
|
async def host_probe(_target: EndpointTarget) -> HostPathProbeResult:
|
||||||
return technical_timeout
|
return technical_timeout
|
||||||
|
|
||||||
|
tcp_calls = 0
|
||||||
|
|
||||||
async def tcp_probe(_target: EndpointTarget) -> bool:
|
async def tcp_probe(_target: EndpointTarget) -> bool:
|
||||||
raise AssertionError("technical failure must skip TCP")
|
nonlocal tcp_calls
|
||||||
|
tcp_calls += 1
|
||||||
|
return True
|
||||||
|
|
||||||
monitor = ReadOnlyConnectionMonitor(
|
monitor = ReadOnlyConnectionMonitor(
|
||||||
supervisor,
|
supervisor,
|
||||||
@@ -1847,8 +1851,10 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
|||||||
)
|
)
|
||||||
|
|
||||||
first = await monitor.poll_once()
|
first = await monitor.poll_once()
|
||||||
assert first.revision == initial.revision
|
assert first.host_path.available is True
|
||||||
assert first.host_path is initial.host_path
|
assert first.host_path.epoch == initial.host_path.epoch
|
||||||
|
assert first.endpoint.tcp_state == "reachable"
|
||||||
|
assert first.authority.control_allowed is False
|
||||||
assert supervisor.observe_endpoint(
|
assert supervisor.observe_endpoint(
|
||||||
target=TARGET,
|
target=TARGET,
|
||||||
intent_id="bridge-1",
|
intent_id="bridge-1",
|
||||||
@@ -1857,11 +1863,14 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
|||||||
)
|
)
|
||||||
endpoint_refresh = supervisor.snapshot()
|
endpoint_refresh = supervisor.snapshot()
|
||||||
assert endpoint_refresh.revision > first.revision
|
assert endpoint_refresh.revision > first.revision
|
||||||
assert endpoint_refresh.host_path is initial.host_path
|
assert endpoint_refresh.host_path.epoch == initial.host_path.epoch
|
||||||
|
assert endpoint_refresh.host_path.fingerprint == initial.host_path.fingerprint
|
||||||
|
|
||||||
second = await monitor.poll_once()
|
second = await monitor.poll_once()
|
||||||
assert second.revision == endpoint_refresh.revision
|
assert second.host_path.available is True
|
||||||
assert second.host_path is initial.host_path
|
assert second.host_path.epoch == initial.host_path.epoch
|
||||||
|
assert second.endpoint.tcp_state == "reachable"
|
||||||
|
assert second.authority.control_allowed is False
|
||||||
assert supervisor.observe_endpoint(
|
assert supervisor.observe_endpoint(
|
||||||
target=TARGET,
|
target=TARGET,
|
||||||
intent_id="bridge-1",
|
intent_id="bridge-1",
|
||||||
@@ -1870,15 +1879,17 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
|||||||
)
|
)
|
||||||
|
|
||||||
confirmed = await monitor.poll_once()
|
confirmed = await monitor.poll_once()
|
||||||
assert confirmed.host_path.available is False
|
assert confirmed.host_path.available is True
|
||||||
assert confirmed.host_path.reason_code == "host-wifi-operation-timeout"
|
assert confirmed.host_path.epoch == initial.host_path.epoch
|
||||||
|
assert confirmed.endpoint.tcp_state == "reachable"
|
||||||
assert confirmed.authority.control_allowed is False
|
assert confirmed.authority.control_allowed is False
|
||||||
|
assert tcp_calls == 3
|
||||||
await monitor.close()
|
await monitor.close()
|
||||||
|
|
||||||
asyncio.run(scenario())
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
|
||||||
def test_inflight_third_timeout_cannot_overwrite_a_concurrent_positive_observation() -> None:
|
def test_inflight_timeout_cannot_overwrite_a_concurrent_positive_observation() -> None:
|
||||||
async def scenario() -> None:
|
async def scenario() -> None:
|
||||||
supervisor, initial_epoch = _configured_unverified_supervisor()
|
supervisor, initial_epoch = _configured_unverified_supervisor()
|
||||||
path = _available_path()
|
path = _available_path()
|
||||||
@@ -1892,20 +1903,20 @@ def test_inflight_third_timeout_cannot_overwrite_a_concurrent_positive_observati
|
|||||||
observation_failure_class="association-observer",
|
observation_failure_class="association-observer",
|
||||||
kernel_route_fingerprint=path.kernel_route_fingerprint,
|
kernel_route_fingerprint=path.kernel_route_fingerprint,
|
||||||
)
|
)
|
||||||
third_probe_entered = asyncio.Event()
|
timeout_probe_entered = asyncio.Event()
|
||||||
release_third_probe = asyncio.Event()
|
release_timeout_probe = asyncio.Event()
|
||||||
host_calls = 0
|
host_calls = 0
|
||||||
|
|
||||||
async def host_probe(_target: EndpointTarget) -> HostPathProbeResult:
|
async def host_probe(_target: EndpointTarget) -> HostPathProbeResult:
|
||||||
nonlocal host_calls
|
nonlocal host_calls
|
||||||
host_calls += 1
|
host_calls += 1
|
||||||
if host_calls == 3:
|
if host_calls == 1:
|
||||||
third_probe_entered.set()
|
timeout_probe_entered.set()
|
||||||
await release_third_probe.wait()
|
await release_timeout_probe.wait()
|
||||||
return technical_timeout
|
return technical_timeout
|
||||||
|
|
||||||
async def tcp_probe(_target: EndpointTarget) -> bool:
|
async def tcp_probe(_target: EndpointTarget) -> bool:
|
||||||
raise AssertionError("technical failure must skip TCP")
|
return True
|
||||||
|
|
||||||
monitor = ReadOnlyConnectionMonitor(
|
monitor = ReadOnlyConnectionMonitor(
|
||||||
supervisor,
|
supervisor,
|
||||||
@@ -1913,32 +1924,20 @@ def test_inflight_third_timeout_cannot_overwrite_a_concurrent_positive_observati
|
|||||||
tcp_probe=tcp_probe,
|
tcp_probe=tcp_probe,
|
||||||
target_provider=lambda: TARGET,
|
target_provider=lambda: TARGET,
|
||||||
)
|
)
|
||||||
first = await monitor.poll_once()
|
inflight_timeout = asyncio.create_task(monitor.poll_once())
|
||||||
second = await monitor.poll_once()
|
await timeout_probe_entered.wait()
|
||||||
assert first.authority.control_allowed is False
|
|
||||||
assert second.authority.control_allowed is False
|
|
||||||
|
|
||||||
inflight_third = asyncio.create_task(monitor.poll_once())
|
|
||||||
await third_probe_entered.wait()
|
|
||||||
refreshed_epoch = supervisor.observe_host_path(path)
|
refreshed_epoch = supervisor.observe_host_path(path)
|
||||||
assert refreshed_epoch == initial_epoch
|
assert refreshed_epoch == initial_epoch
|
||||||
external_positive = supervisor.snapshot()
|
external_positive = supervisor.snapshot()
|
||||||
release_third_probe.set()
|
release_timeout_probe.set()
|
||||||
raced_timeout = await inflight_third
|
raced_timeout = await inflight_timeout
|
||||||
|
|
||||||
assert raced_timeout.revision == external_positive.revision
|
assert raced_timeout.revision >= external_positive.revision
|
||||||
assert raced_timeout.host_path is external_positive.host_path
|
assert raced_timeout.host_path.available is True
|
||||||
|
assert raced_timeout.host_path.epoch == external_positive.host_path.epoch
|
||||||
|
assert raced_timeout.host_path.fingerprint == external_positive.host_path.fingerprint
|
||||||
|
assert raced_timeout.endpoint.tcp_state == "reachable"
|
||||||
assert raced_timeout.authority.control_allowed is False
|
assert raced_timeout.authority.control_allowed is False
|
||||||
|
|
||||||
second_after_positive = await monitor.poll_once()
|
|
||||||
assert second_after_positive.revision == external_positive.revision
|
|
||||||
assert second_after_positive.host_path is external_positive.host_path
|
|
||||||
assert second_after_positive.authority.control_allowed is False
|
|
||||||
|
|
||||||
confirmed = await monitor.poll_once()
|
|
||||||
assert confirmed.host_path.available is False
|
|
||||||
assert confirmed.host_path.reason_code == "host-wifi-operation-timeout"
|
|
||||||
assert confirmed.authority.control_allowed is False
|
|
||||||
await monitor.close()
|
await monitor.close()
|
||||||
|
|
||||||
asyncio.run(scenario())
|
asyncio.run(scenario())
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ from k1link.device_plugins.xgrids_k1.facade import (
|
|||||||
ACTION_PHYSICAL_COMMAND_RECONCILE,
|
ACTION_PHYSICAL_COMMAND_RECONCILE,
|
||||||
ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION,
|
ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION,
|
||||||
ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE,
|
ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE,
|
||||||
|
ACTION_STATE_READ,
|
||||||
ACTION_STREAM_START_LIVE,
|
ACTION_STREAM_START_LIVE,
|
||||||
ACTION_STREAM_START_REPLAY,
|
ACTION_STREAM_START_REPLAY,
|
||||||
ACTION_STREAM_STOP,
|
ACTION_STREAM_STOP,
|
||||||
@@ -1285,6 +1286,57 @@ def test_sync_runtime_actions_run_outside_the_api_event_loop() -> None:
|
|||||||
assert service.thread_id != event_loop_thread
|
assert service.thread_id != event_loop_thread
|
||||||
|
|
||||||
|
|
||||||
|
def test_state_reads_are_single_flight_and_survive_one_cancelled_waiter() -> None:
|
||||||
|
class BlockingStateService(FakeXgridsService):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.state_calls = 0
|
||||||
|
self.started = threading.Event()
|
||||||
|
self.release = threading.Event()
|
||||||
|
|
||||||
|
def state(self) -> dict[str, Any]:
|
||||||
|
self.state_calls += 1
|
||||||
|
self.started.set()
|
||||||
|
if not self.release.wait(timeout=2):
|
||||||
|
raise AssertionError("state read was not released by the test")
|
||||||
|
return {"phase": "idle", "revision": self.state_calls}
|
||||||
|
|
||||||
|
service = BlockingStateService()
|
||||||
|
adapter = XgridsK1PluginFacade(service)
|
||||||
|
|
||||||
|
def invocation(invocation_id: str) -> RuntimeActionInvocation:
|
||||||
|
return RuntimeActionInvocation(
|
||||||
|
invocation_id=invocation_id,
|
||||||
|
plugin_id=adapter.plugin_id,
|
||||||
|
action_id=ACTION_STATE_READ,
|
||||||
|
requested_at=datetime.now(UTC),
|
||||||
|
parameters={},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def exercise() -> None:
|
||||||
|
first = asyncio.create_task(adapter.invoke(invocation("state-read-first")))
|
||||||
|
while not service.started.is_set():
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
second = asyncio.create_task(adapter.invoke(invocation("state-read-second")))
|
||||||
|
await asyncio.sleep(0.02)
|
||||||
|
assert service.state_calls == 1
|
||||||
|
|
||||||
|
first.cancel()
|
||||||
|
with pytest.raises(asyncio.CancelledError):
|
||||||
|
await first
|
||||||
|
assert service.state_calls == 1
|
||||||
|
|
||||||
|
service.release.set()
|
||||||
|
assert await second == {"phase": "idle", "revision": 1}
|
||||||
|
assert await adapter.invoke(invocation("state-read-fresh")) == {
|
||||||
|
"phase": "idle",
|
||||||
|
"revision": 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
asyncio.run(exercise())
|
||||||
|
assert service.state_calls == 2
|
||||||
|
|
||||||
|
|
||||||
def test_runtime_requires_successful_handshake_before_dispatch() -> None:
|
def test_runtime_requires_successful_handshake_before_dispatch() -> None:
|
||||||
adapter = XgridsK1PluginFacade(FakeXgridsService())
|
adapter = XgridsK1PluginFacade(FakeXgridsService())
|
||||||
runtime = _in_process_runtime(adapter, activate=False)
|
runtime = _in_process_runtime(adapter, activate=False)
|
||||||
|
|||||||
@@ -200,6 +200,33 @@ def test_startup_scan_baselines_historical_sessions_without_enqueuing(
|
|||||||
manager.close()
|
manager.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_archive_revision_tracks_session_lifecycle_without_capture_churn(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
sessions = tmp_path / "sessions"
|
||||||
|
sessions.mkdir()
|
||||||
|
initial = app_module.observation_archive_revision((sessions,))
|
||||||
|
|
||||||
|
session = sessions / "20260716T205632Z_viewer_live"
|
||||||
|
session.mkdir()
|
||||||
|
started = app_module.observation_archive_revision((sessions,))
|
||||||
|
assert started != initial
|
||||||
|
|
||||||
|
capture = session / "captures" / "mqtt_live"
|
||||||
|
capture.mkdir(parents=True)
|
||||||
|
(capture / "mqtt.raw.k1mqtt").write_bytes(b"growing-capture")
|
||||||
|
assert app_module.observation_archive_revision((sessions,)) == started
|
||||||
|
|
||||||
|
marker = sessions / ".current_session"
|
||||||
|
marker.write_text(f"{session.name}\n", encoding="utf-8")
|
||||||
|
active = app_module.observation_archive_revision((sessions,))
|
||||||
|
assert active != started
|
||||||
|
|
||||||
|
marker.unlink()
|
||||||
|
finalized = app_module.observation_archive_revision((sessions,))
|
||||||
|
assert finalized != active
|
||||||
|
|
||||||
|
|
||||||
def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session(
|
def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|||||||
@@ -40,6 +40,24 @@ class _FailingSendWebSocket:
|
|||||||
del code, reason
|
del code, reason
|
||||||
|
|
||||||
|
|
||||||
|
class _DisconnectAfterSecondSendWebSocket:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.accepted = False
|
||||||
|
self.send_count = 0
|
||||||
|
|
||||||
|
async def accept(self) -> None:
|
||||||
|
self.accepted = True
|
||||||
|
|
||||||
|
async def send_json(self, payload: dict[str, Any]) -> None:
|
||||||
|
del payload
|
||||||
|
self.send_count += 1
|
||||||
|
if self.send_count == 2:
|
||||||
|
raise WebSocketDisconnect(code=1001)
|
||||||
|
|
||||||
|
async def close(self, *, code: int, reason: str) -> None:
|
||||||
|
del code, reason
|
||||||
|
|
||||||
|
|
||||||
async def _post_json(path: str, payload: dict[str, Any]) -> tuple[int, str]:
|
async def _post_json(path: str, payload: dict[str, Any]) -> tuple[int, str]:
|
||||||
body = json.dumps(payload).encode()
|
body = json.dumps(payload).encode()
|
||||||
request_sent = False
|
request_sent = False
|
||||||
@@ -198,6 +216,25 @@ def test_device_plugin_events_treats_proven_transport_disconnect_as_completion(
|
|||||||
assert websocket.accepted is True
|
assert websocket.accepted is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_device_plugin_events_keeps_heavy_state_poll_off_live_media_cadence(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
websocket = _DisconnectAfterSecondSendWebSocket()
|
||||||
|
sleeps: list[float] = []
|
||||||
|
|
||||||
|
async def record_sleep(delay: float) -> None:
|
||||||
|
sleeps.append(delay)
|
||||||
|
|
||||||
|
monkeypatch.setattr(app_module, "plugin_dispatcher", _StateDispatcher())
|
||||||
|
monkeypatch.setattr(app_module.asyncio, "sleep", record_sleep)
|
||||||
|
|
||||||
|
asyncio.run(app_module.device_plugin_events(websocket, "test.plugin"))
|
||||||
|
|
||||||
|
assert websocket.accepted is True
|
||||||
|
assert sleeps == [app_module.DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS]
|
||||||
|
assert sleeps[0] == 2.0
|
||||||
|
|
||||||
|
|
||||||
def test_device_plugin_events_propagates_arbitrary_send_runtime_error(
|
def test_device_plugin_events_propagates_arbitrary_send_runtime_error(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -5662,7 +5662,7 @@ def test_read_only_verify_scan_is_rejected_after_atomic_fence_before_admission(
|
|||||||
original_retire = service._retire_application_control_for_network_change # noqa: SLF001
|
original_retire = service._retire_application_control_for_network_change # noqa: SLF001
|
||||||
original_apply = service._apply_read_only_device_topology # noqa: SLF001
|
original_apply = service._apply_read_only_device_topology # noqa: SLF001
|
||||||
|
|
||||||
def retire_with_competing_scan() -> None:
|
def retire_with_competing_scan(*, allow_terminal_failure: bool = False) -> None:
|
||||||
assert service._provisioning_active is True # noqa: SLF001
|
assert service._provisioning_active is True # noqa: SLF001
|
||||||
scan_outcomes.append(
|
scan_outcomes.append(
|
||||||
_attempt_competing_scan_from_sync_boundary(
|
_attempt_competing_scan_from_sync_boundary(
|
||||||
@@ -5670,7 +5670,7 @@ def test_read_only_verify_scan_is_rejected_after_atomic_fence_before_admission(
|
|||||||
operation_id="op-00000000-0000-4000-8000-000000000102",
|
operation_id="op-00000000-0000-4000-8000-000000000102",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
original_retire()
|
original_retire(allow_terminal_failure=allow_terminal_failure)
|
||||||
|
|
||||||
def apply_with_fence_assertion(**kwargs: Any) -> str:
|
def apply_with_fence_assertion(**kwargs: Any) -> str:
|
||||||
nonlocal admission_calls
|
nonlocal admission_calls
|
||||||
@@ -7692,6 +7692,58 @@ def test_terminal_prepared_start_without_publish_settles_and_stops_local_capture
|
|||||||
assert service._acquisition_session_lease is None # noqa: SLF001
|
assert service._acquisition_session_lease is None # noqa: SLF001
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_prepared_start_reduction_does_not_touch_publish_fence(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
acquisition_id = "acq-live-prepared"
|
||||||
|
operation_id = "op-live-prepared"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service._physical_command_coordinator, # noqa: SLF001
|
||||||
|
"snapshot",
|
||||||
|
lambda: {
|
||||||
|
"status": "unresolved",
|
||||||
|
"record": {
|
||||||
|
"operation_id": operation_id,
|
||||||
|
"acquisition_id": acquisition_id,
|
||||||
|
"action": "start",
|
||||||
|
"stage": "prepared",
|
||||||
|
"resolution": None,
|
||||||
|
"publish_call_returned": None,
|
||||||
|
"packet_id": None,
|
||||||
|
"qos2_completed": False,
|
||||||
|
"application_response": None,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service._application_control_session, # noqa: SLF001
|
||||||
|
"snapshot",
|
||||||
|
lambda: {"state": "initializing", "failure": None},
|
||||||
|
)
|
||||||
|
|
||||||
|
class PublishFenceMustStayFree:
|
||||||
|
def acquire(self, *args: object, **kwargs: object) -> bool:
|
||||||
|
raise AssertionError("live START reduction touched the publish fence")
|
||||||
|
|
||||||
|
def release(self) -> None:
|
||||||
|
raise AssertionError("live START reduction released an unowned fence")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service,
|
||||||
|
"_k1_command_dispatch_gate",
|
||||||
|
PublishFenceMustStayFree(),
|
||||||
|
)
|
||||||
|
|
||||||
|
outcome = service._settle_prepared_start_worker_failure( # noqa: SLF001
|
||||||
|
acquisition_id=acquisition_id,
|
||||||
|
start_operation_id=operation_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert outcome == "not-applicable"
|
||||||
|
|
||||||
|
|
||||||
def _activate_real_checkpoint_for_prepared_stop_fixture(
|
def _activate_real_checkpoint_for_prepared_stop_fixture(
|
||||||
service: XgridsK1CompatibilityService,
|
service: XgridsK1CompatibilityService,
|
||||||
*,
|
*,
|
||||||
@@ -12273,6 +12325,7 @@ def _stop_response_without_terminal_status_fixture(
|
|||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
*,
|
*,
|
||||||
qos2_completed: bool = True,
|
qos2_completed: bool = True,
|
||||||
|
application_response: bool = True,
|
||||||
) -> SimpleNamespace:
|
) -> SimpleNamespace:
|
||||||
clock_value = [datetime(2026, 8, 10, 8, 10, tzinfo=UTC)]
|
clock_value = [datetime(2026, 8, 10, 8, 10, tzinfo=UTC)]
|
||||||
service, runtime = service_with_fake_runtime(tmp_path)
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
@@ -12424,6 +12477,7 @@ def _stop_response_without_terminal_status_fixture(
|
|||||||
packet_id=82,
|
packet_id=82,
|
||||||
)
|
)
|
||||||
ledger.mark_qos2_completed(stop_operation_id, packet_id=82)
|
ledger.mark_qos2_completed(stop_operation_id, packet_id=82)
|
||||||
|
if application_response:
|
||||||
ledger.record_application_response(
|
ledger.record_application_response(
|
||||||
stop_operation_id,
|
stop_operation_id,
|
||||||
PhysicalCommandApplicationResponse(
|
PhysicalCommandApplicationResponse(
|
||||||
@@ -12450,6 +12504,60 @@ def _stop_response_without_terminal_status_fixture(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_stop_retires_only_terminal_control_and_keeps_live_capture(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
fixture = _stop_response_without_terminal_status_fixture(
|
||||||
|
tmp_path,
|
||||||
|
qos2_completed=False,
|
||||||
|
application_response=False,
|
||||||
|
)
|
||||||
|
service = fixture.service
|
||||||
|
control = fixture.control
|
||||||
|
close_calls = 0
|
||||||
|
worker_retired = False
|
||||||
|
original_snapshot = control.snapshot
|
||||||
|
original_close = control.close
|
||||||
|
|
||||||
|
def terminal_snapshot() -> dict[str, object]:
|
||||||
|
snapshot = original_snapshot()
|
||||||
|
if control.state == "failed" and not worker_retired:
|
||||||
|
snapshot["can_open"] = False
|
||||||
|
return snapshot
|
||||||
|
|
||||||
|
def retire_failed_worker() -> None:
|
||||||
|
nonlocal close_calls, worker_retired
|
||||||
|
close_calls += 1
|
||||||
|
worker_retired = True
|
||||||
|
original_close()
|
||||||
|
|
||||||
|
monkeypatch.setattr(control, "snapshot", terminal_snapshot)
|
||||||
|
monkeypatch.setattr(control, "close", retire_failed_worker)
|
||||||
|
control.state = "failed"
|
||||||
|
control.outcome_unknown = True
|
||||||
|
control.failure = {
|
||||||
|
"code": "ApplicationCommandOutcomeUnknown",
|
||||||
|
"reason_code": "mqtt_response_timeout",
|
||||||
|
"stop_command_attempted": True,
|
||||||
|
"stop_publish_attempts": 1,
|
||||||
|
"safe_to_retry": False,
|
||||||
|
}
|
||||||
|
service._acquire_application_control_process_lease() # noqa: SLF001
|
||||||
|
|
||||||
|
recovered = service.state()
|
||||||
|
|
||||||
|
assert close_calls == 1
|
||||||
|
assert recovered["application_control_session"]["state"] == "idle"
|
||||||
|
assert recovered["acquisition"]["state"] == "awaiting_external_stop"
|
||||||
|
assert recovered["source_mode"] == "live"
|
||||||
|
assert recovered["physical_command"]["status"] == "unresolved"
|
||||||
|
assert recovered["physical_command"]["requires_reconciliation"] is True
|
||||||
|
assert fixture.runtime.stop_calls == 0
|
||||||
|
assert control.stop_calls == 1
|
||||||
|
assert service._application_control_process_lease_holders == set() # noqa: SLF001
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("qos2_completed", [True, False])
|
@pytest.mark.parametrize("qos2_completed", [True, False])
|
||||||
def test_stop_success_without_terminal_status_times_out_into_local_only_recovery(
|
def test_stop_success_without_terminal_status_times_out_into_local_only_recovery(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -13729,7 +13837,18 @@ def test_confirmed_scanning_waits_for_first_authoritative_pcl_before_camera(
|
|||||||
binding=binding,
|
binding=binding,
|
||||||
resolved=True,
|
resolved=True,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(service._physical_command_coordinator, "snapshot", lambda: physical) # noqa: SLF001
|
physical_snapshot_calls = 0
|
||||||
|
|
||||||
|
def snapshot_physical() -> dict[str, object]:
|
||||||
|
nonlocal physical_snapshot_calls
|
||||||
|
physical_snapshot_calls += 1
|
||||||
|
return physical
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service._physical_command_coordinator, # noqa: SLF001
|
||||||
|
"snapshot",
|
||||||
|
snapshot_physical,
|
||||||
|
)
|
||||||
runtime.pcl_frames = 1
|
runtime.pcl_frames = 1
|
||||||
frame = DecodedPointCloudView(
|
frame = DecodedPointCloudView(
|
||||||
context=ConsumerFrameContext(
|
context=ConsumerFrameContext(
|
||||||
@@ -13818,9 +13937,11 @@ def test_confirmed_scanning_waits_for_first_authoritative_pcl_before_camera(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Every later authoritative PCL for the same lineage is idempotent.
|
# Every later authoritative PCL for the same lineage is idempotent.
|
||||||
|
physical_snapshot_calls_before = physical_snapshot_calls
|
||||||
service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001
|
service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001
|
||||||
time.sleep(0.05)
|
time.sleep(0.05)
|
||||||
assert len(events) == 2
|
assert len(events) == 2
|
||||||
|
assert physical_snapshot_calls == physical_snapshot_calls_before
|
||||||
|
|
||||||
|
|
||||||
def test_stale_post_publish_pcl_cannot_activate_camera(
|
def test_stale_post_publish_pcl_cannot_activate_camera(
|
||||||
@@ -15316,6 +15437,85 @@ def test_active_stream_recovery_scanning_resumes_same_physical_lineage_without_c
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_stream_recovery_privacy_bridge_retains_exact_route_for_read_only_rebind(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
|
control, physical = _install_composite_active_recovery_fixture(
|
||||||
|
service,
|
||||||
|
runtime,
|
||||||
|
monkeypatch,
|
||||||
|
)
|
||||||
|
service.state()
|
||||||
|
lineage = service._active_stream_recovery_lineage # noqa: SLF001
|
||||||
|
assert lineage is not None
|
||||||
|
supervisor = service._connection_supervisor # noqa: SLF001
|
||||||
|
baseline = supervisor.snapshot()
|
||||||
|
candidate = replace(
|
||||||
|
baseline,
|
||||||
|
device_identity=replace(baseline.device_identity, state="unverified"),
|
||||||
|
control_plane=replace(
|
||||||
|
baseline.control_plane,
|
||||||
|
state="lost",
|
||||||
|
session_id=None,
|
||||||
|
),
|
||||||
|
lease=replace(baseline.lease, state="configured-unverified"),
|
||||||
|
authority=replace(
|
||||||
|
baseline.authority,
|
||||||
|
control_allowed=False,
|
||||||
|
acquisition_start_allowed=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
start_projects_before = list(control.start_projects)
|
||||||
|
stop_calls_before = control.stop_calls
|
||||||
|
operation_journal_before = service._operations.snapshot() # noqa: SLF001
|
||||||
|
physical_ledger_before = service._physical_command_ledger.snapshot() # noqa: SLF001
|
||||||
|
network_ledger_before = service._network_mutation_ledger.snapshot() # noqa: SLF001
|
||||||
|
monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service._host_wifi_association_probe, # noqa: SLF001
|
||||||
|
"observe",
|
||||||
|
lambda *_args, **_kwargs: {
|
||||||
|
"schema_version": 1,
|
||||||
|
"adapter": "CoreWLAN",
|
||||||
|
"wifi_interface": True,
|
||||||
|
"association_state": "unavailable",
|
||||||
|
"evidence_quality": "unavailable",
|
||||||
|
"continuity_proven": False,
|
||||||
|
"continuity_token": "d" * 64,
|
||||||
|
"reason_code": "association-identity-unavailable",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
supervisor,
|
||||||
|
"association_timeout_retention_candidate",
|
||||||
|
lambda *, expected_target: (
|
||||||
|
candidate
|
||||||
|
if expected_target
|
||||||
|
== EndpointTarget(lineage.target_ipv4, lineage.target_port)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
sampled = service._sample_host_path(lineage.target_ipv4) # noqa: SLF001
|
||||||
|
|
||||||
|
assert sampled.available is True
|
||||||
|
assert sampled.reason_code is None
|
||||||
|
assert sampled.fingerprint == baseline.host_path.fingerprint
|
||||||
|
assert sampled.kernel_route_fingerprint == (
|
||||||
|
baseline.host_path.kernel_route_fingerprint
|
||||||
|
)
|
||||||
|
assert control.start_projects == start_projects_before
|
||||||
|
assert control.stop_calls == stop_calls_before
|
||||||
|
assert service._operations.snapshot() == operation_journal_before # noqa: SLF001
|
||||||
|
assert ( # noqa: SLF001
|
||||||
|
service._physical_command_ledger.snapshot() == physical_ledger_before
|
||||||
|
)
|
||||||
|
assert service._network_mutation_ledger.snapshot() == network_ledger_before # noqa: SLF001
|
||||||
|
assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001
|
||||||
|
|
||||||
|
|
||||||
def test_active_stream_control_adoption_gets_fresh_budget_after_slow_proof(
|
def test_active_stream_control_adoption_gets_fresh_budget_after_slow_proof(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
@@ -15638,7 +15838,7 @@ def test_active_stream_recovery_wrong_identity_blocks_without_retry_or_camera_re
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
service,
|
service,
|
||||||
"_probe_control_endpoint",
|
"_probe_control_endpoint",
|
||||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||||
path=_direct_host_path(lineage.target_ipv4),
|
path=_direct_host_path(lineage.target_ipv4),
|
||||||
reachable=True,
|
reachable=True,
|
||||||
reason_code=None,
|
reason_code=None,
|
||||||
@@ -15699,8 +15899,14 @@ def test_active_stream_recovery_owned_path_is_read_only_and_resumes_exact_lineag
|
|||||||
events.append(("monitor", True))
|
events.append(("monitor", True))
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def probe_exact_target(target_ipv4: str) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001
|
def probe_exact_target( # noqa: SLF001
|
||||||
events.append(("probe", target_ipv4))
|
target_ipv4: str,
|
||||||
|
*,
|
||||||
|
association_timeout_seconds: float,
|
||||||
|
) -> facade_module._CorrelatedEndpointObservation:
|
||||||
|
events.append(
|
||||||
|
("probe", (target_ipv4, association_timeout_seconds))
|
||||||
|
)
|
||||||
return facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
return facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||||
path=_direct_host_path(target_ipv4),
|
path=_direct_host_path(target_ipv4),
|
||||||
reachable=True,
|
reachable=True,
|
||||||
@@ -15770,7 +15976,13 @@ def test_active_stream_recovery_owned_path_is_read_only_and_resumes_exact_lineag
|
|||||||
assert events == [
|
assert events == [
|
||||||
("monitor", True),
|
("monitor", True),
|
||||||
("lease-acquire", "network"),
|
("lease-acquire", "network"),
|
||||||
("probe", lineage.target_ipv4),
|
(
|
||||||
|
"probe",
|
||||||
|
(
|
||||||
|
lineage.target_ipv4,
|
||||||
|
facade_module.ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS,
|
||||||
|
),
|
||||||
|
),
|
||||||
(
|
(
|
||||||
"device-info-status",
|
"device-info-status",
|
||||||
(
|
(
|
||||||
@@ -15910,6 +16122,7 @@ def test_incident_recovery_retires_epoch_one_control_and_retries_fresh_inspectio
|
|||||||
|
|
||||||
def probe_current_epoch(
|
def probe_current_epoch(
|
||||||
target_ipv4: str,
|
target_ipv4: str,
|
||||||
|
**_kwargs: object,
|
||||||
) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001
|
) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001
|
||||||
path = next(probe_paths)
|
path = next(probe_paths)
|
||||||
service._observe_connection_transport( # noqa: SLF001
|
service._observe_connection_transport( # noqa: SLF001
|
||||||
@@ -16065,7 +16278,7 @@ def test_active_stream_recovery_waits_for_terminal_control_worker_retirement(
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
service,
|
service,
|
||||||
"_probe_control_endpoint",
|
"_probe_control_endpoint",
|
||||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||||
path=_direct_host_path(lineage.target_ipv4),
|
path=_direct_host_path(lineage.target_ipv4),
|
||||||
reachable=True,
|
reachable=True,
|
||||||
reason_code=None,
|
reason_code=None,
|
||||||
@@ -16140,7 +16353,7 @@ def test_active_stream_recovery_exact_device_system_fault_is_terminal_without_co
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
service,
|
service,
|
||||||
"_probe_control_endpoint",
|
"_probe_control_endpoint",
|
||||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||||
path=_direct_host_path(lineage.target_ipv4),
|
path=_direct_host_path(lineage.target_ipv4),
|
||||||
reachable=True,
|
reachable=True,
|
||||||
reason_code=None,
|
reason_code=None,
|
||||||
@@ -16223,7 +16436,7 @@ def test_active_stream_recovery_bootstrap_system_error_is_normalized_terminal_fa
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
service,
|
service,
|
||||||
"_probe_control_endpoint",
|
"_probe_control_endpoint",
|
||||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||||
path=_direct_host_path(lineage.target_ipv4),
|
path=_direct_host_path(lineage.target_ipv4),
|
||||||
reachable=True,
|
reachable=True,
|
||||||
reason_code=None,
|
reason_code=None,
|
||||||
@@ -16476,6 +16689,111 @@ def test_active_stream_recovery_never_retries_invalid_fmp4_after_media_commit(
|
|||||||
assert control.stop_calls == 0
|
assert control.stop_calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_bound_scan_over_without_stop_dominates_late_empty_camera_epoch_failure(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
service, runtime = service_with_fake_runtime(tmp_path)
|
||||||
|
control, physical = _install_composite_active_recovery_fixture(
|
||||||
|
service,
|
||||||
|
runtime,
|
||||||
|
monkeypatch,
|
||||||
|
runtime_phase="live",
|
||||||
|
camera_phase="streaming",
|
||||||
|
)
|
||||||
|
camera = service.camera_preview.snapshot()
|
||||||
|
recording = camera["recording"]
|
||||||
|
assert isinstance(recording, dict)
|
||||||
|
recording.update(
|
||||||
|
{
|
||||||
|
"active": True,
|
||||||
|
"active_epoch": 2,
|
||||||
|
"committed_media_segment_count": 4_833,
|
||||||
|
"producer_alive": False,
|
||||||
|
"completed_epochs": 2,
|
||||||
|
"last_summary": {
|
||||||
|
"codec_epoch": 2,
|
||||||
|
"status": "failed",
|
||||||
|
"media_segment_count": 0,
|
||||||
|
"failure_code": "invalid-fmp4",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
camera.update(
|
||||||
|
{
|
||||||
|
"phase": "error",
|
||||||
|
"error": {
|
||||||
|
"code": "invalid-fmp4",
|
||||||
|
"message": "synthetic empty post-power-loss epoch",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
physical.update(
|
||||||
|
{
|
||||||
|
"runtime_bound": True,
|
||||||
|
"reconciliation_ready": True,
|
||||||
|
"observed_session_state": "scan_over",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
control.state = "failed"
|
||||||
|
control.state_revision += 1
|
||||||
|
control.failure = {
|
||||||
|
"reason_code": "application_acceptance_failed",
|
||||||
|
"failed_phase": "scanning",
|
||||||
|
"modeling_command_attempted": True,
|
||||||
|
"stop_command_attempted": False,
|
||||||
|
"diagnostic_snapshot_unavailable": [],
|
||||||
|
"diagnostic_evidence_unavailable": [],
|
||||||
|
"safe_to_retry": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
def scan_over_control_snapshot() -> dict[str, object]:
|
||||||
|
snapshot = FakeInteractiveControlSession.snapshot(control)
|
||||||
|
snapshot["transport"] = {
|
||||||
|
"state": "failed",
|
||||||
|
"publish_attempts": 7,
|
||||||
|
"device_status_reports": 9,
|
||||||
|
"latest_device_session_state": "scan_over",
|
||||||
|
"latest_device_project_bound": True,
|
||||||
|
"latest_device_init_ready": False,
|
||||||
|
"latest_system_error_code": None,
|
||||||
|
"automatic_retry": False,
|
||||||
|
"automatic_reconnect": False,
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
|
|
||||||
|
monkeypatch.setattr(control, "snapshot", scan_over_control_snapshot)
|
||||||
|
monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None)
|
||||||
|
camera_stop_calls: list[str] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service.camera_preview,
|
||||||
|
"stop_current",
|
||||||
|
lambda: camera_stop_calls.append("stop") or {},
|
||||||
|
)
|
||||||
|
start_projects_before = list(control.start_projects)
|
||||||
|
stop_calls_before = control.stop_calls
|
||||||
|
|
||||||
|
terminal = service.state()
|
||||||
|
|
||||||
|
assert terminal["acquisition"]["state"] == "interrupted"
|
||||||
|
assert terminal["acquisition"]["message_code"] == (
|
||||||
|
"acquisition.recovery.device_standby_observed"
|
||||||
|
)
|
||||||
|
assert terminal["acquisition"]["result"] == {
|
||||||
|
"receiver_stopped": True,
|
||||||
|
"device_state": "scan_over",
|
||||||
|
"device_stop": "not-sent",
|
||||||
|
"automatic_command_retry": False,
|
||||||
|
"read_only_recovery": True,
|
||||||
|
"physical_reconciliation_required": True,
|
||||||
|
}
|
||||||
|
assert terminal["acquisition"]["cleanup_pending"] is False
|
||||||
|
assert camera_stop_calls == ["stop"]
|
||||||
|
assert runtime.stop_calls == 1
|
||||||
|
assert control.start_projects == start_projects_before
|
||||||
|
assert control.stop_calls == stop_calls_before == 0
|
||||||
|
|
||||||
|
|
||||||
def test_active_stream_recovery_reopens_exact_camera_epoch_once_and_fences_stale_lineage(
|
def test_active_stream_recovery_reopens_exact_camera_epoch_once_and_fences_stale_lineage(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
@@ -24955,6 +25273,59 @@ def test_configured_unverified_monitor_keeps_same_route_without_promoting_author
|
|||||||
assert "verify-control-device-info" in retained.allowed_actions
|
assert "verify-control-device-info" in retained.allowed_actions
|
||||||
|
|
||||||
|
|
||||||
|
def test_monitor_layer_retains_exact_configured_route_across_association_lock_timeout(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
binding = _seed_supervised_connection(service, with_control=False)
|
||||||
|
baseline = service._connection_supervisor.snapshot() # noqa: SLF001
|
||||||
|
assert baseline.host_path.kernel_route_fingerprint is not None
|
||||||
|
tcp_calls: list[str] = []
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
service,
|
||||||
|
"_sample_host_path",
|
||||||
|
lambda target, **_kwargs: HostPathProbeResult(
|
||||||
|
available=False,
|
||||||
|
fingerprint=None,
|
||||||
|
interface=baseline.host_path.interface,
|
||||||
|
source_ipv4=baseline.host_path.source_ipv4,
|
||||||
|
route_class="unavailable",
|
||||||
|
reason_code="host-wifi-operation-timeout",
|
||||||
|
observation_failure_class="association-observer",
|
||||||
|
kernel_route_fingerprint=baseline.host_path.kernel_route_fingerprint,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
facade_module,
|
||||||
|
"_probe_control_endpoint_socket",
|
||||||
|
lambda target: (
|
||||||
|
tcp_calls.append(target)
|
||||||
|
or facade_module.TcpReachabilityProbeResult(reachable=True)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
async def poll_three_times() -> list[facade_module.ConnectionSupervisorSnapshot]:
|
||||||
|
return [
|
||||||
|
await service._connection_monitor.poll_once() # noqa: SLF001
|
||||||
|
for _ in range(3)
|
||||||
|
]
|
||||||
|
|
||||||
|
snapshots = asyncio.run(poll_three_times())
|
||||||
|
|
||||||
|
assert tcp_calls == [binding.target_ipv4] * 3
|
||||||
|
for retained in snapshots:
|
||||||
|
assert retained.host_path.available is True
|
||||||
|
assert retained.host_path.epoch == baseline.host_path.epoch
|
||||||
|
assert retained.host_path.fingerprint == baseline.host_path.fingerprint
|
||||||
|
assert retained.endpoint.tcp_state == "reachable"
|
||||||
|
assert retained.device_identity.state == "unverified"
|
||||||
|
assert retained.authority.control_allowed is False
|
||||||
|
assert retained.authority.acquisition_start_allowed is False
|
||||||
|
assert "verify-control-device-info" in retained.allowed_actions
|
||||||
|
|
||||||
|
|
||||||
def test_association_timeout_cannot_hide_a_real_kernel_route_change(
|
def test_association_timeout_cannot_hide_a_real_kernel_route_change(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
@@ -29825,6 +30196,91 @@ def test_explicit_verify_reconciles_persisted_start_to_ready_without_device_io(
|
|||||||
assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001
|
assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001
|
||||||
|
|
||||||
|
|
||||||
|
def test_long_first_verify_refreshes_aged_transport_before_exposing_prestart_control(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""A long checkpoint reconciliation cannot orphan its new ready socket."""
|
||||||
|
|
||||||
|
service, _ = service_with_fake_runtime(tmp_path)
|
||||||
|
coordinator = _VerifyPhysicalRecoveryCoordinator(
|
||||||
|
observed_session_state="ready",
|
||||||
|
reconciliation_ready=True,
|
||||||
|
)
|
||||||
|
_install_synthetic_verify_recovery(service, coordinator)
|
||||||
|
supervisor = service._connection_supervisor # noqa: SLF001
|
||||||
|
monotonic_now = [100.0]
|
||||||
|
suspend_aware_now = [1_000.0]
|
||||||
|
supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001
|
||||||
|
supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001
|
||||||
|
before = supervisor.snapshot()
|
||||||
|
assert before.intent is not None
|
||||||
|
assert before.device_network.target is not None
|
||||||
|
host_epoch = supervisor.observe_host_path(
|
||||||
|
_association_bound_direct_host_path(before.device_network.target.ipv4)
|
||||||
|
)
|
||||||
|
assert supervisor.observe_endpoint(
|
||||||
|
target=before.device_network.target,
|
||||||
|
intent_id=before.intent.intent_id,
|
||||||
|
host_path_epoch=host_epoch,
|
||||||
|
reachable=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path)
|
||||||
|
tcp_probes: list[str] = []
|
||||||
|
|
||||||
|
def reachable(target: str) -> bool:
|
||||||
|
tcp_probes.append(target)
|
||||||
|
return True
|
||||||
|
|
||||||
|
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", reachable)
|
||||||
|
reconcile = service._reconcile_physical_command_after_verify_owned # noqa: SLF001
|
||||||
|
|
||||||
|
async def reconcile_after_transport_ages(
|
||||||
|
bound_service: XgridsK1CompatibilityService,
|
||||||
|
*,
|
||||||
|
verify_operation_id: str,
|
||||||
|
allow_receiver_rehydrate: bool = True,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
result = await reconcile(
|
||||||
|
verify_operation_id=verify_operation_id,
|
||||||
|
allow_receiver_rehydrate=allow_receiver_rehydrate,
|
||||||
|
)
|
||||||
|
monotonic_now[0] += 20.0
|
||||||
|
suspend_aware_now[0] += 20.0
|
||||||
|
return result
|
||||||
|
|
||||||
|
service._reconcile_physical_command_after_verify_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001
|
||||||
|
reconcile_after_transport_ages,
|
||||||
|
service,
|
||||||
|
)
|
||||||
|
operation_id = "op-00000000-0000-4000-8000-000000001202"
|
||||||
|
|
||||||
|
verified = asyncio.run(
|
||||||
|
service.verify_connection(
|
||||||
|
_retained_physical_recovery_verify_request(operation_id=operation_id)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert tcp_probes == ["192.168.1.20"]
|
||||||
|
assert verified["last_operation"]["status"] == "succeeded"
|
||||||
|
assert verified["active_connection_mode"] == "bridge"
|
||||||
|
assert verified["connection_policy"]["actions"]["start-acquisition"]["allowed"] is True
|
||||||
|
assert service._application_control_session.snapshot()["state"] == "connection-ready" # noqa: SLF001
|
||||||
|
assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001
|
||||||
|
fresh = supervisor.snapshot()
|
||||||
|
assert fresh.intent is not None
|
||||||
|
assert fresh.device_network.target is not None
|
||||||
|
assert supervisor.endpoint_observation_has_remaining_lease(
|
||||||
|
target=fresh.device_network.target,
|
||||||
|
intent_id=fresh.intent.intent_id,
|
||||||
|
host_path_epoch=fresh.host_path.epoch,
|
||||||
|
minimum_remaining_seconds=(
|
||||||
|
facade_module.VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_explicit_verify_reconciliation_does_not_require_normal_command_authority(
|
def test_explicit_verify_reconciliation_does_not_require_normal_command_authority(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -1843,6 +1843,25 @@ def test_chained_stop_facade_cessation_uses_real_ledger_ancestry(
|
|||||||
stale_reconciliation,
|
stale_reconciliation,
|
||||||
final_record,
|
final_record,
|
||||||
)
|
)
|
||||||
|
# Reproduce the append-only field shape: an otherwise valid historical
|
||||||
|
# standby reconciliation is inherited by the newer ordinary STOP. The
|
||||||
|
# checkpoint reducer must use the current STOP's own READY status, not the
|
||||||
|
# last historical reconciliation merely because it is terminal standby.
|
||||||
|
snapshot_with_stale_standby = replace(
|
||||||
|
ledger.snapshot(),
|
||||||
|
record=replace(
|
||||||
|
final_record,
|
||||||
|
reconciliations=(
|
||||||
|
*final_record.reconciliations,
|
||||||
|
stale_reconciliation,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ledger,
|
||||||
|
"snapshot",
|
||||||
|
lambda: snapshot_with_stale_standby,
|
||||||
|
)
|
||||||
|
|
||||||
service._application_control_session.snapshot = lambda: { # type: ignore[method-assign]
|
service._application_control_session.snapshot = lambda: { # type: ignore[method-assign]
|
||||||
"verified_control": _verified_control(first_rebind)
|
"verified_control": _verified_control(first_rebind)
|
||||||
@@ -1862,6 +1881,88 @@ def test_chained_stop_facade_cessation_uses_real_ledger_ancestry(
|
|||||||
assert checkpoint.cessation_physical_proof.ancestor_chain == ancestry
|
assert checkpoint.cessation_physical_proof.ancestor_chain == ancestry
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_ceases_active_checkpoint_after_composite_ordinary_stop(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
service = _service(tmp_path, monkeypatch)
|
||||||
|
connection = _connection(
|
||||||
|
control_session_id="control-before-ordinary-stop-restart",
|
||||||
|
host_path_epoch=1,
|
||||||
|
producer_generation=1,
|
||||||
|
)
|
||||||
|
_prepare_and_activate_checkpoint(service, connection)
|
||||||
|
ledger = service._physical_command_ledger # noqa: SLF001
|
||||||
|
start = ledger.snapshot().record
|
||||||
|
assert start is not None and start.operation_id == START_OPERATION_ID
|
||||||
|
identity = PhysicalCommandIdentity(
|
||||||
|
vendor_device_id_sha256=VENDOR_SHA256,
|
||||||
|
device_serial_sha256=SERIAL_SHA256,
|
||||||
|
)
|
||||||
|
stop_operation_id = "physical-stop-ordinary-restart"
|
||||||
|
ledger.prepare(
|
||||||
|
operation_id=stop_operation_id,
|
||||||
|
parent_operation_id=START_OPERATION_ID,
|
||||||
|
acquisition_id=ACQUISITION_ID,
|
||||||
|
action="stop",
|
||||||
|
identity=identity,
|
||||||
|
connection=connection,
|
||||||
|
compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID,
|
||||||
|
payload_sha256="6" * 64,
|
||||||
|
baseline_status=_status(
|
||||||
|
connection,
|
||||||
|
"scanning",
|
||||||
|
observed_at_utc="2026-08-13T12:02:00.000Z",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
ledger.mark_dispatching(stop_operation_id)
|
||||||
|
ledger.mark_observing(
|
||||||
|
stop_operation_id,
|
||||||
|
publish_call_returned=True,
|
||||||
|
packet_id=43,
|
||||||
|
)
|
||||||
|
ledger.mark_qos2_completed(stop_operation_id, packet_id=43)
|
||||||
|
ledger.record_application_response(
|
||||||
|
stop_operation_id,
|
||||||
|
PhysicalCommandApplicationResponse(
|
||||||
|
operation_id=stop_operation_id,
|
||||||
|
action="stop",
|
||||||
|
control_session_id=connection.control_session_id,
|
||||||
|
host_path_epoch=connection.host_path_epoch,
|
||||||
|
producer_generation=connection.producer_generation,
|
||||||
|
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
|
||||||
|
success=True,
|
||||||
|
payload_sha256="7" * 64,
|
||||||
|
observed_at_utc="2026-08-13T12:02:01.000Z",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
ledger.record_status_observation(
|
||||||
|
stop_operation_id,
|
||||||
|
_status(
|
||||||
|
connection,
|
||||||
|
"ready",
|
||||||
|
observed_at_utc="2026-08-13T12:02:02.000Z",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
stop = ledger.resolve(
|
||||||
|
stop_operation_id,
|
||||||
|
resolution="stop-standby-observed",
|
||||||
|
)
|
||||||
|
before_restart = ActiveAcquisitionRecoveryCheckpointStore(tmp_path).snapshot()
|
||||||
|
assert before_restart.status == "active"
|
||||||
|
|
||||||
|
restarted = XgridsK1CompatibilityService(tmp_path)
|
||||||
|
|
||||||
|
after_restart = ActiveAcquisitionRecoveryCheckpointStore(tmp_path).snapshot()
|
||||||
|
assert after_restart.status == "ceased"
|
||||||
|
assert after_restart.checkpoint is not None
|
||||||
|
assert after_restart.checkpoint.cessation_physical_proof is not None
|
||||||
|
assert after_restart.checkpoint.cessation_physical_proof.operation_id == (
|
||||||
|
stop.operation_id
|
||||||
|
)
|
||||||
|
assert restarted._active_acquisition_checkpoint_trust == "trusted" # noqa: SLF001
|
||||||
|
|
||||||
|
|
||||||
def test_restart_ready_classification_ceases_active_checkpoint_after_undispatched_stop(
|
def test_restart_ready_classification_ceases_active_checkpoint_after_undispatched_stop(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|||||||
@@ -887,7 +887,7 @@ def test_ambiguous_prepared_restart_ready_settles_after_dhcp_target_change(
|
|||||||
mqtt_retained=False,
|
mqtt_retained=False,
|
||||||
observed_at_utc="2026-08-13T12:20:01.000Z",
|
observed_at_utc="2026-08-13T12:20:01.000Z",
|
||||||
)
|
)
|
||||||
record = restarted._physical_command_ledger.reconcile_ambiguous( # noqa: SLF001
|
restarted._physical_command_ledger.reconcile_ambiguous( # noqa: SLF001
|
||||||
restart_support.START_OPERATION_ID,
|
restart_support.START_OPERATION_ID,
|
||||||
reconciliation_id="reconciliation-ready-new-dhcp-address",
|
reconciliation_id="reconciliation-ready-new-dhcp-address",
|
||||||
resolution="physical-standby-observed",
|
resolution="physical-standby-observed",
|
||||||
@@ -1057,7 +1057,7 @@ def test_restart_receiver_link_loss_before_first_pcl_rebinds_read_only_then_prom
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
restarted,
|
restarted,
|
||||||
"_probe_control_endpoint",
|
"_probe_control_endpoint",
|
||||||
lambda target_ipv4: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
lambda target_ipv4, **_kwargs: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||||
path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001
|
path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001
|
||||||
reachable=True,
|
reachable=True,
|
||||||
reason_code=None,
|
reason_code=None,
|
||||||
@@ -1237,7 +1237,7 @@ def test_ambiguous_prepared_restart_second_rebind_promotes_only_after_fresh_pcl(
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
restarted,
|
restarted,
|
||||||
"_probe_control_endpoint",
|
"_probe_control_endpoint",
|
||||||
lambda target_ipv4: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
lambda target_ipv4, **_kwargs: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||||
path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001
|
path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001
|
||||||
reachable=True,
|
reachable=True,
|
||||||
reason_code=None,
|
reason_code=None,
|
||||||
|
|||||||
@@ -394,7 +394,10 @@ def _wait_phase(
|
|||||||
session: InteractiveApplicationControlSession,
|
session: InteractiveApplicationControlSession,
|
||||||
expected: str,
|
expected: str,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
deadline = time.monotonic() + 2.0
|
# FakeExecutor may use its full two-second checkpoint timeout before the
|
||||||
|
# worker publishes the terminal state. Keep the observer deadline strictly
|
||||||
|
# larger so this helper does not race the transition it is asserting.
|
||||||
|
deadline = time.monotonic() + 3.0
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
snapshot = session.snapshot()
|
snapshot = session.snapshot()
|
||||||
if snapshot["state"] == expected:
|
if snapshot["state"] == expected:
|
||||||
|
|||||||
Reference in New Issue
Block a user