fix(k1): stabilize live recovery and media admission
This commit is contained in:
@@ -41,7 +41,9 @@ export interface CameraStartupWatchdog {
|
||||
* Supervise only the disposable browser transport. One fixed first-media
|
||||
* deadline covers both an MSE that never opens and an open-but-silent socket.
|
||||
* The first non-empty fragment then starts a separate first-playable-frame
|
||||
* deadline; later fragments deliberately do not extend it.
|
||||
* deadline. Every subsequent fragment extends that deadline: a browser that is
|
||||
* still receiving the authoritative stream must not destroy its partial MSE
|
||||
* decode solely because the main thread or decoder needed longer to start.
|
||||
*/
|
||||
export function createCameraStartupWatchdog({
|
||||
schedule,
|
||||
@@ -83,7 +85,7 @@ export function createCameraStartupWatchdog({
|
||||
arm("first-media", firstMediaTimeoutMs);
|
||||
},
|
||||
markMediaReceived() {
|
||||
if (mediaReceived || playing) return;
|
||||
if (playing) return;
|
||||
mediaReceived = true;
|
||||
arm("first-playable-frame", firstPlayableFrameTimeoutMs);
|
||||
},
|
||||
@@ -189,6 +191,7 @@ export function MseFmp4WebSocketPlayer({
|
||||
const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id));
|
||||
const activeAuthorityRef = useRef(recoveryAuthorityIdentity);
|
||||
const recoveryPendingAuthorityRef = useRef<string | null>(null);
|
||||
const transportHealthyRef = useRef(false);
|
||||
const transportEpochRef = useRef(0);
|
||||
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
|
||||
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
|
||||
@@ -243,6 +246,7 @@ export function MseFmp4WebSocketPlayer({
|
||||
let receivedMedia = false;
|
||||
let failed = false;
|
||||
let startupWatchdog: CameraStartupWatchdog | null = null;
|
||||
transportHealthyRef.current = false;
|
||||
const recovering = Boolean(
|
||||
activeAuthorityRef.current
|
||||
&& recoveryPendingAuthorityRef.current === activeAuthorityRef.current,
|
||||
@@ -254,6 +258,7 @@ export function MseFmp4WebSocketPlayer({
|
||||
if (!transportIsCurrent() || failed) return;
|
||||
startupWatchdog?.clear();
|
||||
failed = true;
|
||||
transportHealthyRef.current = false;
|
||||
recoveryPendingAuthorityRef.current = null;
|
||||
setStatus("error");
|
||||
setMessage(copy);
|
||||
@@ -280,6 +285,7 @@ export function MseFmp4WebSocketPlayer({
|
||||
}
|
||||
startupWatchdog?.clear();
|
||||
failed = true;
|
||||
transportHealthyRef.current = false;
|
||||
queue.length = 0;
|
||||
queuedBytes = 0;
|
||||
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
|
||||
@@ -320,6 +326,7 @@ export function MseFmp4WebSocketPlayer({
|
||||
const onPlaying = () => {
|
||||
if (!transportIsCurrent() || failed) return;
|
||||
startupWatchdog?.markPlaying();
|
||||
transportHealthyRef.current = true;
|
||||
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
|
||||
recoveryPendingAuthorityRef.current = null;
|
||||
setStatus("playing");
|
||||
@@ -464,6 +471,7 @@ export function MseFmp4WebSocketPlayer({
|
||||
const disposeTransport = () => {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
transportHealthyRef.current = false;
|
||||
startupWatchdog?.clear();
|
||||
if (retryTimer !== undefined) window.clearTimeout(retryTimer);
|
||||
queue.length = 0;
|
||||
@@ -518,6 +526,7 @@ export function MseFmp4WebSocketPlayer({
|
||||
now: Date.now(),
|
||||
documentVisible: document.visibilityState === "visible",
|
||||
networkOnline: navigator.onLine !== false,
|
||||
transportHealthy: transportHealthyRef.current,
|
||||
});
|
||||
recovery = decision.state;
|
||||
if (!decision.reopen) return;
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
initialLiveReceiverRecoveryState,
|
||||
initialLiveReceiverWatchdogState,
|
||||
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
|
||||
LIVE_RECEIVER_OPEN_MAX_AGE_MS,
|
||||
LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS,
|
||||
requestLiveReceiverRecovery,
|
||||
} from "../core/observation/liveReceiverWatchdog";
|
||||
import {
|
||||
@@ -1091,25 +1091,6 @@ export function RerunViewport({
|
||||
};
|
||||
const clearRecordedAdmissionWatchdog = () => recordedOpenWatchdog?.clear();
|
||||
const clearLiveRecordingOpenTimer = diagnosticLifecycle.clearAdmissionTimeout;
|
||||
function refreshOpeningLiveReceiver(openForMs: number) {
|
||||
if (disposed || recordingOpened) return;
|
||||
diagnosticLifecycle.post({
|
||||
eventCode: "live_receiver_restart_requested",
|
||||
failureStage: "recording-open-timeout",
|
||||
streamId: liveStreamIdRef.current,
|
||||
backendActivitySequence: liveActivitySequenceRef.current,
|
||||
viewerRangeMaxNs: latestLiveRangeMaxNs,
|
||||
stalledForMs: Math.round(openForMs),
|
||||
recoveryAttempt: liveRecoveryRef.current.attempts || null,
|
||||
});
|
||||
setStatus("loading");
|
||||
onStatusChange?.(
|
||||
"loading",
|
||||
"Живой визуализатор обновляет приёмник продолжающегося потока.",
|
||||
);
|
||||
disposeViewer?.();
|
||||
setRetryNonce((nonce) => nonce + 1);
|
||||
}
|
||||
const armLiveRecordingOpenTimer = () => {
|
||||
clearLiveRecordingOpenTimer();
|
||||
diagnosticLifecycle.armAdmissionTimeout(() => {
|
||||
@@ -1129,17 +1110,9 @@ export function RerunViewport({
|
||||
armLiveRecordingOpenTimer();
|
||||
return;
|
||||
}
|
||||
if (observed.signal === "refresh-receiver") {
|
||||
// Publication is healthy, so this is presentation-only maintenance:
|
||||
// refresh the aged native receiver without spending (or clearing)
|
||||
// recovery debt.
|
||||
recordingOpenTimedOut = true;
|
||||
refreshOpeningLiveReceiver(observed.openForMs);
|
||||
return;
|
||||
}
|
||||
recordingOpenTimedOut = true;
|
||||
requestLiveRecovery("recording-open-timeout");
|
||||
}, LIVE_RECEIVER_OPEN_MAX_AGE_MS);
|
||||
}, LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS);
|
||||
};
|
||||
const clearLiveRecordingDiscoveryTimer = diagnosticLifecycle.clearAdmissionInterval;
|
||||
const clearLiveRecoveryRetryTimer = () => {
|
||||
|
||||
@@ -26,6 +26,7 @@ export interface CameraPlaybackRecoveryContext {
|
||||
now: number;
|
||||
documentVisible: boolean;
|
||||
networkOnline: boolean;
|
||||
transportHealthy?: boolean;
|
||||
}
|
||||
|
||||
export interface CameraPlaybackRecoveryDecision {
|
||||
@@ -186,7 +187,8 @@ export function reduceCameraPlaybackRecovery(
|
||||
} else if (event.type === "page-restore") {
|
||||
candidate = event.persisted;
|
||||
} else if (event.type === "heartbeat") {
|
||||
candidate = now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
|
||||
candidate = context.transportHealthy !== true
|
||||
&& now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
|
||||
}
|
||||
|
||||
const outsideCooldown = current.lastReopenAt === null
|
||||
|
||||
@@ -31,7 +31,6 @@ export interface LiveReceiverOpenWatchdogState {
|
||||
|
||||
export type LiveReceiverOpenWatchdogSignal =
|
||||
| "wait-for-store"
|
||||
| "refresh-receiver"
|
||||
| "restart-receiver";
|
||||
|
||||
export interface LiveReceiverOpenWatchdogResult {
|
||||
@@ -60,10 +59,11 @@ export interface LiveReceiverRecoveryRequest {
|
||||
export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
|
||||
export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
|
||||
// The bridge publishes its URL only after StoreInfo, blueprint and static
|
||||
// scene data have been flushed. A receiver that still has not admitted that
|
||||
// store after one operator-visible four-second window is wedged, not merely
|
||||
// slow; keeping it for 48 seconds made a healthy live scan look blank.
|
||||
export const LIVE_RECEIVER_OPEN_MAX_AGE_MS = 4_000;
|
||||
// scene data have been flushed. Poll at the original four-second boundary,
|
||||
// but never discard a receiver while this exact acquisition is still proving
|
||||
// fresh publication progress: doing so throws away its partial store replay and
|
||||
// can keep a healthy long-running stream blank indefinitely.
|
||||
export const LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS = 4_000;
|
||||
const LIVE_RECEIVER_RECOVERY_DELAYS_MS = [400, 1_000, 2_000, 5_000] as const;
|
||||
|
||||
export function liveReceiverRecoveryRetryDelay(attempt: number): number {
|
||||
@@ -216,11 +216,10 @@ export function initialLiveReceiverOpenWatchdogState(
|
||||
/**
|
||||
* Keep one still-opening Rerun receiver alive while the backend is proving
|
||||
* fresh publication progress. Recreating the WASM receiver on a fixed timer
|
||||
* can repeatedly discard an otherwise healthy late StoreInfo replay. Rolling
|
||||
* patience is nevertheless bounded: a receiver that has not admitted a store
|
||||
* by the absolute open-age limit is refreshed without consuming the recovery
|
||||
* budget. A true lack of backend progress delegates to the bounded restart
|
||||
* policy. Recovery debt is cleared only after viewer admission, never merely
|
||||
* can repeatedly discard an otherwise healthy late StoreInfo replay. Preserve
|
||||
* that receiver for as long as the backend proves fresh publication. A true
|
||||
* lack of backend progress delegates to the bounded restart policy on the next
|
||||
* check. Recovery debt is cleared only after viewer admission, never merely
|
||||
* because the backend counter advanced.
|
||||
*/
|
||||
export function advanceLiveReceiverOpenWatchdog(
|
||||
@@ -228,7 +227,6 @@ export function advanceLiveReceiverOpenWatchdog(
|
||||
recoveryState: LiveReceiverRecoveryState,
|
||||
backendActivitySequence: number | null,
|
||||
nowMs = Date.now(),
|
||||
maxOpenAgeMs = LIVE_RECEIVER_OPEN_MAX_AGE_MS,
|
||||
): LiveReceiverOpenWatchdogResult {
|
||||
const sequence = validBackendActivitySequence(backendActivitySequence);
|
||||
const previous = current.lastBackendActivitySequence;
|
||||
@@ -247,9 +245,7 @@ export function advanceLiveReceiverOpenWatchdog(
|
||||
return {
|
||||
state,
|
||||
recoveryState,
|
||||
signal: openForMs >= maxOpenAgeMs
|
||||
? "refresh-receiver"
|
||||
: "wait-for-store",
|
||||
signal: "wait-for-store",
|
||||
openForMs,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -129,6 +129,26 @@ export function admitLiveDefaultPresentations(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A restored workspace may hide cameras that the device plugin has not selected.
|
||||
* It must still admit the exact selected delivery in a fresh browser document,
|
||||
* or a replacement delivery for a camera already presented in this acquisition.
|
||||
* An explicit close remains a separate acquisition-scoped fence in
|
||||
* `admitLiveDefaultPresentations` and de-selects the source in the plugin first.
|
||||
*/
|
||||
export function restoredLayoutMayAdmitLiveDefault(
|
||||
sources: readonly ObservationSourceDescriptor[],
|
||||
presentedAcquisitionSources: ReadonlySet<string>,
|
||||
): boolean {
|
||||
const selectedSources = sources.filter(automaticLivePresentationIdentity);
|
||||
if (selectedSources.length === 0) return false;
|
||||
if (presentedAcquisitionSources.size === 0) return true;
|
||||
return selectedSources.some((source) => {
|
||||
const lineage = livePresentationCloseFence(source);
|
||||
return Boolean(lineage && presentedAcquisitionSources.has(lineage));
|
||||
});
|
||||
}
|
||||
|
||||
function catalogIdentity(sources: readonly ObservationSourceDescriptor[]): string {
|
||||
return sources
|
||||
.map((source) => [
|
||||
@@ -170,6 +190,7 @@ export function useObservationLayout(
|
||||
const restoredLayoutAuthorityRef = useRef(false);
|
||||
const admittedLivePresentationIdentitiesRef = useRef(new Set<string>());
|
||||
const closedLiveAcquisitionSourcesRef = useRef(new Set<string>());
|
||||
const presentedLiveAcquisitionSourcesRef = useRef(new Set<string>());
|
||||
const initializedCatalog = useRef<string | null>(null);
|
||||
const sourceIdList = sources.map((source) => source.id).sort();
|
||||
const sourceIdsIdentity = sourceIdList.join("\u0000");
|
||||
@@ -182,6 +203,12 @@ export function useObservationLayout(
|
||||
|
||||
const commitVisibleIds = useCallback((next: readonly string[]) => {
|
||||
const unique = [...new Set(next)];
|
||||
const visible = new Set(unique);
|
||||
for (const source of sourcesRef.current) {
|
||||
if (!visible.has(source.id) || !automaticLivePresentationIdentity(source)) continue;
|
||||
const lineage = livePresentationCloseFence(source);
|
||||
if (lineage) presentedLiveAcquisitionSourcesRef.current.add(lineage);
|
||||
}
|
||||
visibleIdsRef.current = unique;
|
||||
setVisibleIds(unique);
|
||||
}, []);
|
||||
@@ -352,11 +379,18 @@ export function useObservationLayout(
|
||||
.filter((candidate): candidate is string => candidate !== null)
|
||||
.sort());
|
||||
|
||||
useEffect(() => {
|
||||
if (restoredLayoutAuthorityRef.current) return;
|
||||
const admitAutomaticLivePresentations = useCallback(() => {
|
||||
const currentSources = sourcesRef.current;
|
||||
if (
|
||||
restoredLayoutAuthorityRef.current
|
||||
&& !restoredLayoutMayAdmitLiveDefault(
|
||||
currentSources,
|
||||
presentedLiveAcquisitionSourcesRef.current,
|
||||
)
|
||||
) return;
|
||||
const admission = admitLiveDefaultPresentations(
|
||||
visibleIdsRef.current,
|
||||
sources,
|
||||
currentSources,
|
||||
admittedLivePresentationIdentitiesRef.current,
|
||||
closedLiveAcquisitionSourcesRef.current,
|
||||
);
|
||||
@@ -367,7 +401,11 @@ export function useObservationLayout(
|
||||
commitVisibleIds(admission.visibleIds);
|
||||
clearPresentation(admission.removedIds, false);
|
||||
persistLiveLayout();
|
||||
}, [clearPresentation, commitVisibleIds, persistLiveLayout, selectedDeliveryIdentity]);
|
||||
}, [clearPresentation, commitVisibleIds, persistLiveLayout]);
|
||||
|
||||
useEffect(() => {
|
||||
admitAutomaticLivePresentations();
|
||||
}, [admitAutomaticLivePresentations, selectedDeliveryIdentity]);
|
||||
|
||||
const markPending = useCallback((source: ObservationSourceDescriptor, pending: boolean) => {
|
||||
const groupId = source.activation?.groupId;
|
||||
@@ -545,7 +583,10 @@ export function useObservationLayout(
|
||||
});
|
||||
restoredLayoutAuthorityRef.current = true;
|
||||
applyDesiredSnapshot(desiredSnapshotRef.current, "reset");
|
||||
}, [applyDesiredSnapshot]);
|
||||
// The selected delivery may already have arrived before the saved layout.
|
||||
// Re-run admission here so restore ordering cannot strand its camera window.
|
||||
admitAutomaticLivePresentations();
|
||||
}, [admitAutomaticLivePresentations, applyDesiredSnapshot]);
|
||||
|
||||
const visibleSourceIds = useMemo(() => new Set(visibleIds), [visibleIds]);
|
||||
const pendingSourceIds = useMemo(() => new Set(pendingIds), [pendingIds]);
|
||||
|
||||
@@ -235,12 +235,7 @@ function SpatialWorkspace({
|
||||
Boolean(observationLayout.maximizedFloatingSourceId);
|
||||
const timeline = state?.observationTimeline;
|
||||
const viewportRef = useRef<HTMLDivElement>(null);
|
||||
const intentionalSourceEnd = !recordedSource && [
|
||||
"awaiting_external_stop",
|
||||
"stopping",
|
||||
"finalizing",
|
||||
"completed",
|
||||
].includes(state?.acquisition?.state ?? "");
|
||||
const intentionalSourceEnd = !recordedSource && state?.sourceMode === "idle" && ["awaiting_external_stop", "stopping", "finalizing", "completed",].includes(state?.acquisition?.state ?? "");
|
||||
const presentedViewerStatus = intentionalSourceEnd
|
||||
? "idle"
|
||||
: rerunPresentationStatus(
|
||||
|
||||
@@ -620,6 +620,53 @@ test("canonical K1 preparation stops before START and guards every async stage",
|
||||
);
|
||||
});
|
||||
|
||||
test("K1 control-phase polling fails closed instead of stacking timed-out state reads", async () => {
|
||||
const hookSource = await readFile(
|
||||
new URL(
|
||||
"../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts",
|
||||
import.meta.url,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
const controlPollingLoop = hookSource.slice(
|
||||
hookSource.indexOf("async function waitForControlPhase"),
|
||||
hookSource.indexOf("async function waitForPhysicalReconciliationProof"),
|
||||
);
|
||||
|
||||
assert.match(hookSource, /const CONTROL_PHASE_WAIT_TIMEOUT_MS = 120_000/);
|
||||
assert.match(controlPollingLoop, /xgridsK1Api\.getState\(\)/);
|
||||
assert.doesNotMatch(controlPollingLoop, /ApiRequestTimeoutError/);
|
||||
assert.doesNotMatch(controlPollingLoop, /catch \(error\)/);
|
||||
assert.doesNotMatch(
|
||||
controlPollingLoop,
|
||||
/(?:openApplicationControlSession|enterApplicationWorkspace|prepareAcquisition|startAcquisition|stopAcquisition)\(/,
|
||||
);
|
||||
});
|
||||
|
||||
test("K1 frontend coalesces concurrent read-only state requests without caching them", async () => {
|
||||
const apiSource = await readFile(
|
||||
new URL(
|
||||
"../../../plugins/xgrids-k1/frontend/src/api.ts",
|
||||
import.meta.url,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
const singleFlight = apiSource.slice(
|
||||
apiSource.indexOf("let stateReadInFlight"),
|
||||
apiSource.indexOf("export const xgridsK1Api"),
|
||||
);
|
||||
const getState = apiSource.slice(
|
||||
apiSource.indexOf("async getState"),
|
||||
apiSource.indexOf("scanBle("),
|
||||
);
|
||||
|
||||
assert.match(singleFlight, /if \(stateReadInFlight\) return stateReadInFlight/);
|
||||
assert.match(singleFlight, /invokeState\(xgridsK1Actions\.stateRead\)/);
|
||||
assert.match(singleFlight, /stateReadInFlight === request[\s\S]*?stateReadInFlight = null/);
|
||||
assert.match(getState, /return readStateSingleFlight\(\)/);
|
||||
assert.doesNotMatch(singleFlight, /setTimeout|setInterval|retry/i);
|
||||
});
|
||||
|
||||
test("K1 control-session CAS is exact, integer-only and mapped for each mutation family", () => {
|
||||
const state = {
|
||||
application_control_session: {
|
||||
|
||||
@@ -244,13 +244,16 @@ test("connecting Rerun authority requires an exact recovery generation lease", (
|
||||
);
|
||||
});
|
||||
|
||||
test("opening receiver gets bounded rolling patience while backend publication advances", () => {
|
||||
test("opening receiver preserves partial store replay while backend publication advances", () => {
|
||||
let openState = initialLiveReceiverOpenWatchdogState(0, 0);
|
||||
let recoveryState = initialLiveReceiverRecoveryState();
|
||||
|
||||
const samples = [
|
||||
[134, 3_999, "wait-for-store"],
|
||||
[266, 4_000, "refresh-receiver"],
|
||||
[266, 4_000, "wait-for-store"],
|
||||
[380, 8_000, "wait-for-store"],
|
||||
[486, 12_000, "wait-for-store"],
|
||||
[700, 120_000, "wait-for-store"],
|
||||
];
|
||||
for (const [backendActivitySequence, nowMs, expectedSignal] of samples) {
|
||||
const observed = advanceLiveReceiverOpenWatchdog(
|
||||
@@ -302,17 +305,17 @@ test("fresh backend progress preserves earlier restart debt until viewer admissi
|
||||
});
|
||||
});
|
||||
|
||||
test("aged active receiver refresh does not spend or erase restart debt", () => {
|
||||
test("aged active receiver remains intact and preserves restart debt", () => {
|
||||
const openState = initialLiveReceiverOpenWatchdogState(486, 0);
|
||||
const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
|
||||
const observed = advanceLiveReceiverOpenWatchdog(
|
||||
openState,
|
||||
consumedRestart.state,
|
||||
900,
|
||||
4_000,
|
||||
12_000,
|
||||
);
|
||||
|
||||
assert.equal(observed.signal, "refresh-receiver");
|
||||
assert.equal(observed.signal, "wait-for-store");
|
||||
assert.deepEqual(observed.recoveryState, consumedRestart.state);
|
||||
assert.equal(observed.openForMs, 4_000);
|
||||
assert.equal(observed.openForMs, 12_000);
|
||||
});
|
||||
|
||||
@@ -1199,7 +1199,7 @@ test("camera startup watchdog replaces an open-but-silent MSE or WebSocket", ()
|
||||
);
|
||||
});
|
||||
|
||||
test("first media starts one non-sliding first-playable-frame deadline", () => {
|
||||
test("continuing media extends the first-playable-frame deadline", () => {
|
||||
const scheduled = new Map();
|
||||
const cancelled = [];
|
||||
const timeouts = [];
|
||||
@@ -1228,8 +1228,14 @@ test("first media starts one non-sliding first-playable-frame deadline", () => {
|
||||
assert.equal(scheduled.get(playableHandle).timeoutMs, CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS);
|
||||
|
||||
watchdog.markMediaReceived();
|
||||
assert.equal(nextHandle - 1, playableHandle, "later fragments must not extend the deadline");
|
||||
scheduled.get(playableHandle).callback();
|
||||
const extendedPlayableHandle = nextHandle - 1;
|
||||
assert.notEqual(extendedPlayableHandle, playableHandle);
|
||||
assert.deepEqual(cancelled, [10, playableHandle]);
|
||||
assert.equal(
|
||||
scheduled.get(extendedPlayableHandle).timeoutMs,
|
||||
CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS,
|
||||
);
|
||||
scheduled.get(extendedPlayableHandle).callback();
|
||||
assert.deepEqual(timeouts, ["first-playable-frame"]);
|
||||
assert.match(
|
||||
cameraStartupWatchdogRecoveryMessage("first-playable-frame"),
|
||||
@@ -1306,6 +1312,7 @@ test("a laptop sleep gap reopens only the exact authoritative live camera transp
|
||||
now: 7_000,
|
||||
documentVisible: true,
|
||||
networkOnline: true,
|
||||
transportHealthy: false,
|
||||
});
|
||||
assert.equal(wake.reopen, true);
|
||||
assert.equal(wake.state.lastReopenAt, 7_000);
|
||||
@@ -1369,15 +1376,16 @@ test("visibility, pageshow and online wake burst owns one replacement decoder",
|
||||
assert.equal(secondWake.state.lastReopenAt, 18_000);
|
||||
});
|
||||
|
||||
test("healthy visible camera heartbeats do not churn its WebSocket or decoder", () => {
|
||||
test("healthy camera survives a long main-thread heartbeat gap without decoder churn", () => {
|
||||
const authority = "camera-authority-acquisition-healthy";
|
||||
let state = initialCameraPlaybackRecoveryState(authority, 1_000);
|
||||
for (const now of [2_000, 3_000, 4_000, 5_000]) {
|
||||
for (const now of [2_000, 3_000, 4_000, 12_000, 25_000]) {
|
||||
const heartbeat = reduceCameraPlaybackRecovery(state, { type: "heartbeat" }, {
|
||||
activeAuthorityIdentity: authority,
|
||||
now,
|
||||
documentVisible: true,
|
||||
networkOnline: true,
|
||||
transportHealthy: true,
|
||||
});
|
||||
assert.equal(heartbeat.reopen, false);
|
||||
state = heartbeat.state;
|
||||
|
||||
@@ -295,6 +295,25 @@ test("raw replay exercises the same streaming receiver lifecycle as a live scan"
|
||||
);
|
||||
});
|
||||
|
||||
test("pending K1 STOP keeps the live Rerun source mounted until local capture ends", async () => {
|
||||
const source = await readFile(
|
||||
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/const intentionalSourceEnd = !recordedSource &&\s*state\?\.sourceMode === "idle" && \[/,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/"awaiting_external_stop",\s*"stopping",\s*"finalizing",\s*"completed",/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
source,
|
||||
/const intentionalSourceEnd = !recordedSource && \[\s*"awaiting_external_stop"/,
|
||||
);
|
||||
});
|
||||
|
||||
test("the spatial header reports raw replay as an active source", async () => {
|
||||
const source = await readFile(
|
||||
new URL("../src/App.tsx", import.meta.url),
|
||||
|
||||
@@ -16,6 +16,7 @@ let WorkspaceLayoutContractError;
|
||||
let admitLiveDefaultPresentations;
|
||||
let automaticLivePresentationIdentity;
|
||||
let livePresentationCloseFence;
|
||||
let restoredLayoutMayAdmitLiveDefault;
|
||||
let observationPresentationSourceAfterLayoutApply;
|
||||
let visibleSourceIdsAfterRecordedCatalogActivation;
|
||||
|
||||
@@ -39,6 +40,7 @@ before(async () => {
|
||||
admitLiveDefaultPresentations,
|
||||
automaticLivePresentationIdentity,
|
||||
livePresentationCloseFence,
|
||||
restoredLayoutMayAdmitLiveDefault,
|
||||
observationPresentationSourceAfterLayoutApply,
|
||||
visibleSourceIdsAfterRecordedCatalogActivation,
|
||||
} = await server.ssrLoadModule("/src/core/observation/useObservationLayout.ts"));
|
||||
@@ -308,6 +310,67 @@ test("a sequential live acquisition re-arms the same camera without reopening a
|
||||
]);
|
||||
});
|
||||
|
||||
test("a restored layout admits a selected delivery after reload and only same-acquisition successors", () => {
|
||||
const camera = (acquisitionId, deliveryId) => ({
|
||||
id: "k1:sensor.camera.right",
|
||||
sourceId: "sensor.camera.right",
|
||||
modality: "video",
|
||||
availability: "streaming",
|
||||
transport: "websocket",
|
||||
previewUrl: null,
|
||||
delivery: {
|
||||
id: deliveryId,
|
||||
kind: "mse-fmp4-websocket",
|
||||
url: `/camera-preview/${deliveryId}`,
|
||||
mediaType: 'video/mp4; codecs="avc1.641028"',
|
||||
},
|
||||
activation: {
|
||||
groupId: "k1:device-session-reused:camera.preview.decoder",
|
||||
maxActive: 1,
|
||||
selected: true,
|
||||
controllable: true,
|
||||
},
|
||||
binding: {
|
||||
deviceId: "k1-a",
|
||||
deviceSessionId: "device-session-reused",
|
||||
acquisitionId,
|
||||
},
|
||||
capabilities: { defaultVisible: true, overlay: true },
|
||||
});
|
||||
const original = camera("acquisition-a", "camera-preview-2");
|
||||
const successor = camera("acquisition-a", "camera-preview-3");
|
||||
const unrelated = camera("acquisition-b", "camera-preview-3");
|
||||
const presented = new Set([livePresentationCloseFence(original)]);
|
||||
|
||||
assert.equal(restoredLayoutMayAdmitLiveDefault([original], new Set()), true);
|
||||
assert.equal(restoredLayoutMayAdmitLiveDefault([successor], presented), true);
|
||||
assert.equal(restoredLayoutMayAdmitLiveDefault([unrelated], presented), false);
|
||||
assert.equal(restoredLayoutMayAdmitLiveDefault([{
|
||||
...original,
|
||||
activation: { ...original.activation, selected: false },
|
||||
}], new Set()), false);
|
||||
|
||||
const admitted = admitLiveDefaultPresentations(
|
||||
[],
|
||||
[successor],
|
||||
new Set([automaticLivePresentationIdentity(original)]),
|
||||
new Set(),
|
||||
);
|
||||
assert.deepEqual(admitted.visibleIds, [successor.id]);
|
||||
assert.deepEqual(admitted.admittedIdentities, [
|
||||
automaticLivePresentationIdentity(successor),
|
||||
]);
|
||||
|
||||
const deliberatelyClosed = admitLiveDefaultPresentations(
|
||||
[],
|
||||
[successor],
|
||||
new Set([automaticLivePresentationIdentity(original)]),
|
||||
presented,
|
||||
);
|
||||
assert.deepEqual(deliberatelyClosed.visibleIds, []);
|
||||
assert.deepEqual(deliberatelyClosed.admittedIdentities, []);
|
||||
});
|
||||
|
||||
test("workspace layout API uses the canonical endpoint and optimistic revision", async () => {
|
||||
const calls = [];
|
||||
const current = decodeObservationWorkspaceLayoutProfile(wireProfile());
|
||||
|
||||
@@ -1875,6 +1875,20 @@ function invokeState(actionId: string, input: object = {}): Promise<XgridsK1Stat
|
||||
);
|
||||
}
|
||||
|
||||
let stateReadInFlight: Promise<XgridsK1State> | null = null;
|
||||
|
||||
function readStateSingleFlight(): Promise<XgridsK1State> {
|
||||
if (stateReadInFlight) return stateReadInFlight;
|
||||
|
||||
const request = invokeState(xgridsK1Actions.stateRead);
|
||||
stateReadInFlight = request;
|
||||
const release = () => {
|
||||
if (stateReadInFlight === request) stateReadInFlight = null;
|
||||
};
|
||||
void request.then(release, release);
|
||||
return request;
|
||||
}
|
||||
|
||||
export const xgridsK1Api = {
|
||||
async getHealth(): Promise<HealthResponse> {
|
||||
const payload = await requestJson(
|
||||
@@ -1890,7 +1904,7 @@ export const xgridsK1Api = {
|
||||
},
|
||||
|
||||
async getState(): Promise<XgridsK1State> {
|
||||
return invokeState(xgridsK1Actions.stateRead);
|
||||
return readStateSingleFlight();
|
||||
},
|
||||
|
||||
scanBle(
|
||||
|
||||
@@ -67,10 +67,12 @@ function RecoveryState({
|
||||
{copy.showSpinner ? <ActivityIndicator size="compact" /> : null}
|
||||
<div className="active-stream-recovery__copy">
|
||||
<strong>{copy.title}</strong>
|
||||
<span>{copy.detail}</span>
|
||||
{presentation?.progressLabel ? (
|
||||
<small>{presentation.progressLabel}</small>
|
||||
) : null}
|
||||
<span className="active-stream-recovery__detail">
|
||||
<span>{copy.detail}</span>
|
||||
{presentation?.progressLabel ? (
|
||||
<small>{presentation.progressLabel}</small>
|
||||
) : null}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -599,6 +599,12 @@ box-sizing: border-box;
|
||||
gap: 0.28rem;
|
||||
}
|
||||
|
||||
.active-stream-recovery__detail {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.28rem;
|
||||
}
|
||||
|
||||
.active-stream-recovery__copy strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.72rem;
|
||||
@@ -1023,25 +1029,83 @@ box-sizing: border-box;
|
||||
min-width: 0;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 0.65rem;
|
||||
column-gap: 0.75rem;
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__state {
|
||||
grid-column: 1;
|
||||
grid-row: 1;
|
||||
min-width: 0;
|
||||
align-items: center;
|
||||
gap: 0.65rem;
|
||||
background: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact
|
||||
.active-stream-recovery__state
|
||||
> .nodedc-activity-indicator {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__copy {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
justify-items: center;
|
||||
gap: 0.1rem;
|
||||
overflow: hidden;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__copy strong {
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
font-size: 0.62rem;
|
||||
font-weight: 600;
|
||||
line-height: 1.35;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__detail {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
align-items: baseline;
|
||||
justify-content: center;
|
||||
gap: 0.4rem;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__detail > span {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
font-size: 0.56rem;
|
||||
line-height: 1.35;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__copy small {
|
||||
flex: 0 0 auto;
|
||||
font-size: 0.54rem;
|
||||
line-height: 1.35;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__actions {
|
||||
max-width: 15rem;
|
||||
grid-column: 2;
|
||||
grid-row: 1;
|
||||
align-self: start;
|
||||
max-width: none;
|
||||
grid-template-columns: auto;
|
||||
justify-items: end;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__actions p {
|
||||
font-size: 0.5rem;
|
||||
line-height: 1.35;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.xgrids-k1-spatial-controls__phase {
|
||||
@@ -1155,14 +1219,10 @@ box-sizing: border-box;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__actions {
|
||||
max-width: none;
|
||||
}
|
||||
|
||||
.active-stream-recovery--compact .active-stream-recovery__actions p {
|
||||
display: none;
|
||||
grid-column: 2;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,7 +248,7 @@ export function connectionActionAuthoritySnapshot(
|
||||
}
|
||||
|
||||
const CONTROL_STATE_READ_INTERVAL_MS = 250;
|
||||
const CONTROL_PHASE_WAIT_TIMEOUT_MS = 35_000;
|
||||
const CONTROL_PHASE_WAIT_TIMEOUT_MS = 120_000;
|
||||
|
||||
function controlPhase(state: XgridsK1State): XgridsApplicationControlPhase {
|
||||
return state.application_control_session?.state ?? "idle";
|
||||
|
||||
@@ -25,7 +25,7 @@ WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl"
|
||||
RUNNER_NAME = RUNNER.name
|
||||
PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
|
||||
EXPECTED_BASELINE_SHA256 = "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8"
|
||||
EXPECTED_WHEEL_SHA256 = "9a60efa68eadf2267fffe3dbb89fb58d5e69e672fa3cdf7529474c9f47416acb"
|
||||
EXPECTED_WHEEL_SHA256 = "411e1a040b708a118827dfd6898ef1f9911e42a86fa58b9e476688cfa046fce1"
|
||||
PAYLOAD_FILES = (
|
||||
RUNNER_NAME,
|
||||
WHEEL_NAME,
|
||||
|
||||
@@ -1868,9 +1868,10 @@ class ReadOnlyConnectionMonitor:
|
||||
return None
|
||||
previous_path = baseline.host_path
|
||||
intent = baseline.intent
|
||||
exact_verified_control = bool(
|
||||
intent is not None
|
||||
and previous_path.available
|
||||
if intent is None:
|
||||
return None
|
||||
exact_reachable_route = bool(
|
||||
previous_path.available
|
||||
and previous_path.fingerprint is not None
|
||||
and previous_path.kernel_route_fingerprint is not None
|
||||
and baseline.device_network.state == "applied"
|
||||
@@ -1880,6 +1881,9 @@ class ReadOnlyConnectionMonitor:
|
||||
and baseline.endpoint.intent_id == intent.intent_id
|
||||
and baseline.endpoint.host_path_epoch == previous_path.epoch
|
||||
and baseline.endpoint.tcp_state == "reachable"
|
||||
)
|
||||
exact_verified_control = bool(
|
||||
exact_reachable_route
|
||||
and baseline.device_identity.state == "verified"
|
||||
and baseline.device_identity.intent_id == intent.intent_id
|
||||
and baseline.device_identity.host_path_epoch == previous_path.epoch
|
||||
@@ -1892,7 +1896,11 @@ class ReadOnlyConnectionMonitor:
|
||||
and baseline.lease.target == target
|
||||
and baseline.authority.control_allowed
|
||||
)
|
||||
if not exact_verified_control:
|
||||
technical_timeout_on_exact_route = bool(
|
||||
result.reason_code == "host-wifi-operation-timeout"
|
||||
and exact_reachable_route
|
||||
)
|
||||
if not (exact_verified_control or technical_timeout_on_exact_route):
|
||||
return None
|
||||
if (
|
||||
result.kernel_route_fingerprint is None
|
||||
|
||||
@@ -278,6 +278,13 @@ HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS = 2.0
|
||||
# without weakening the sub-second association cache or skipping the final
|
||||
# route/association continuity recheck.
|
||||
CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS = 3.0
|
||||
# Active-stream recovery has already frozen an exact acquisition/START owner.
|
||||
# Its endpoint probe may therefore use the same short read-only association
|
||||
# budget as the monitor: the exact raw route bridge above preserves continuity
|
||||
# across a technical CoreWLAN timeout, while fresh DeviceInfo still has to
|
||||
# verify the K1 before MQTT publication resumes. Do not inherit the generic
|
||||
# 30-second helper budget on every Wi-Fi return.
|
||||
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS = 3.0
|
||||
# Command checkpoints must not inherit either the monitor's three-second
|
||||
# contact budget or the host helper's generic 30-second budget. One command
|
||||
# may legitimately wait about three seconds for the service-lifetime
|
||||
@@ -303,6 +310,9 @@ CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS = (
|
||||
2 * CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS + CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS
|
||||
)
|
||||
CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS = 8.0
|
||||
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS = (
|
||||
CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS + 1.0
|
||||
)
|
||||
CONNECTION_MONITOR_DEBOUNCEABLE_ASSOCIATION_REASONS = frozenset({"host-wifi-operation-timeout"})
|
||||
LIVE_DATA_PLANE_STALL_SECONDS = 5.0
|
||||
LIVE_DATA_PLANE_LOST_SECONDS = LIVE_DATA_PLANE_STALL_SECONDS * 2.0
|
||||
@@ -2557,8 +2567,11 @@ class XgridsK1CompatibilityService:
|
||||
A privacy-limited observation has the stricter gate below: exact
|
||||
DeviceInfo, healthy control session, reachable lease and command
|
||||
authority must all still match the same intent/mode/target/epoch. The
|
||||
legacy timeout bridge may retain only its exact reachable route but
|
||||
cannot manufacture any missing command authority.
|
||||
sole exception is an already-admitted active-stream recovery lineage:
|
||||
it may retain only this exact reachable route long enough to perform a
|
||||
fresh read-only DeviceInfo/MQTT rebind. That route bridge grants no
|
||||
control authority and cannot send START, STOP or a network mutation.
|
||||
The legacy timeout bridge has the same route-only limitation.
|
||||
|
||||
A proven association change, a different route/interface/source, a
|
||||
stale or unreachable endpoint, or a different intent/target bypasses
|
||||
@@ -2623,7 +2636,13 @@ class XgridsK1CompatibilityService:
|
||||
and snapshot.lease.target == endpoint_target
|
||||
and snapshot.authority.control_allowed
|
||||
)
|
||||
if not exact_verified_control:
|
||||
exact_active_stream_rebind = (
|
||||
self._exact_active_stream_recovery_route_rebind(
|
||||
snapshot=snapshot,
|
||||
endpoint_target=endpoint_target,
|
||||
)
|
||||
)
|
||||
if not (exact_verified_control or exact_active_stream_rebind):
|
||||
return None
|
||||
return HostPathProbeResult(
|
||||
available=True,
|
||||
@@ -2640,6 +2659,68 @@ class XgridsK1CompatibilityService:
|
||||
kernel_route_fingerprint=path.fingerprint,
|
||||
)
|
||||
|
||||
def _exact_active_stream_recovery_route_rebind(
|
||||
self,
|
||||
*,
|
||||
snapshot: ConnectionSupervisorSnapshot,
|
||||
endpoint_target: EndpointTarget,
|
||||
) -> bool:
|
||||
"""Allow one exact route-only bridge for an admitted live recovery.
|
||||
|
||||
Admission of ``_ActiveStreamRecoveryLineage`` already proved the
|
||||
durable original START owner. This guard rechecks its process-local
|
||||
acquisition/runtime coordinates without consulting or refreshing any
|
||||
device authority. The subsequent recovery path must still establish
|
||||
a fresh DeviceInfo identity and MQTT control proof before it can
|
||||
resume publication.
|
||||
"""
|
||||
|
||||
intent = snapshot.intent
|
||||
with self._lock:
|
||||
lineage = self._active_stream_recovery_lineage
|
||||
acquisition = self._acquisition
|
||||
if lineage is None or acquisition is None:
|
||||
return False
|
||||
exact_local_lineage = bool(
|
||||
self._active_stream_recovery_state == "reconnecting"
|
||||
and self._active_stream_recovery_generation
|
||||
== lineage.recovery_generation
|
||||
and self._snapshot_runtime_id == lineage.snapshot_runtime_id
|
||||
and acquisition.acquisition_id == lineage.acquisition_id
|
||||
and acquisition.device_id == lineage.device_id
|
||||
and acquisition.device_session_id == lineage.device_session_id
|
||||
and acquisition.state == "acquiring"
|
||||
and self._acquisition_start_operation_id
|
||||
in {None, lineage.start_operation_id}
|
||||
and self._acquisition_session_lease is not None
|
||||
and self._acquisition_out_dir is not None
|
||||
and self._acquisition_out_dir.name == lineage.evidence_session_id
|
||||
and self._selected_device_id == lineage.transport_ref
|
||||
and self._device_id == lineage.device_id
|
||||
and self._device_session_id == lineage.device_session_id
|
||||
and self._connection_mode == lineage.connection_mode
|
||||
and self._k1_ip == lineage.target_ipv4
|
||||
)
|
||||
if not exact_local_lineage:
|
||||
return False
|
||||
runtime = self.runtime.snapshot()
|
||||
return bool(
|
||||
runtime.get("phase") == "reconnecting"
|
||||
and runtime.get("source_mode") == "live"
|
||||
and runtime.get("producer_generation")
|
||||
== lineage.runtime_producer_generation
|
||||
and intent is not None
|
||||
and intent.intent_id == lineage.intent_id
|
||||
and intent.requested_mode == lineage.connection_mode
|
||||
and snapshot.device_network.state == "applied"
|
||||
and snapshot.device_network.intent_id == lineage.intent_id
|
||||
and snapshot.device_network.transport_ref == lineage.transport_ref
|
||||
and snapshot.device_network.connection_mode == lineage.connection_mode
|
||||
and snapshot.device_network.target == endpoint_target
|
||||
and endpoint_target
|
||||
== EndpointTarget(lineage.target_ipv4, lineage.target_port)
|
||||
)
|
||||
|
||||
async def _monitor_host_path(self, target: EndpointTarget) -> HostPathProbeResult:
|
||||
if not self._connection_monitor_contact_gate.acquire(blocking=False):
|
||||
raise ConnectionMonitorProbeSuperseded(
|
||||
@@ -2715,6 +2796,57 @@ class XgridsK1CompatibilityService:
|
||||
)
|
||||
self._connection_supervisor.record_monitor_failure("connection-monitor-start-failed")
|
||||
|
||||
async def _refresh_aged_verify_transport_owned(
|
||||
self,
|
||||
*,
|
||||
application_control_session: Mapping[str, Any],
|
||||
) -> None:
|
||||
"""Refresh only a nearly expired Verify transport with real I/O.
|
||||
|
||||
The caller owns the Verify lifecycle/network fences, so the background
|
||||
monitor is deliberately superseded. If DeviceInfo or physical
|
||||
reconciliation consumed most of the existing endpoint lease, perform
|
||||
the same route/TCP/route observation before releasing those fences.
|
||||
No cached timestamp, MQTT reconnect, BLE operation, or device command
|
||||
is manufactured here.
|
||||
"""
|
||||
|
||||
verified_binding = application_control_session.get("verified_control")
|
||||
if not isinstance(verified_binding, Mapping):
|
||||
raise ConnectionVerificationError(
|
||||
"Verify lost its exact control proof before transport refresh",
|
||||
reason_code="connection-verify-final-binding-superseded",
|
||||
)
|
||||
target = EndpointTarget(
|
||||
str(verified_binding["target_ipv4"]),
|
||||
int(verified_binding["target_port"]),
|
||||
)
|
||||
intent_id = str(verified_binding["intent_id"])
|
||||
host_path_epoch = int(verified_binding["host_path_epoch"])
|
||||
if self._connection_supervisor.endpoint_observation_has_remaining_lease(
|
||||
target=target,
|
||||
intent_id=intent_id,
|
||||
host_path_epoch=host_path_epoch,
|
||||
minimum_remaining_seconds=(
|
||||
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS
|
||||
),
|
||||
):
|
||||
return
|
||||
observation = await _run_blocking_operation_without_abandonment(
|
||||
self._probe_control_endpoint,
|
||||
target.ipv4,
|
||||
association_timeout_seconds=(CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS),
|
||||
)
|
||||
if not observation.reachable:
|
||||
raise ConnectionVerificationError(
|
||||
"K1 endpoint не подтвердил связь после долгой read-only сверки",
|
||||
reason_code="connection-verify-mqtt-unreachable",
|
||||
)
|
||||
self._reconcile_connection_supervisor(
|
||||
application_control_session,
|
||||
self.runtime.snapshot(),
|
||||
)
|
||||
|
||||
def _observe_connection_transport(
|
||||
self,
|
||||
target: str,
|
||||
@@ -2723,9 +2855,17 @@ class XgridsK1CompatibilityService:
|
||||
reachable: bool | None,
|
||||
reason_code: str | None = None,
|
||||
) -> None:
|
||||
supervisor = self._connection_supervisor.snapshot()
|
||||
intent = supervisor.intent
|
||||
endpoint = EndpointTarget(validate_private_ipv4(target), CONTROL_MQTT_PORT)
|
||||
# A completed route sample is fresher than the silence fallback. Read
|
||||
# the exact applied target without expiring the previous observation
|
||||
# between sampling and publication; the CAS below still rejects an
|
||||
# intent/target change and the sample itself decides route continuity.
|
||||
supervisor = self._connection_supervisor.association_timeout_retention_candidate(
|
||||
expected_target=endpoint,
|
||||
)
|
||||
if supervisor is None:
|
||||
return
|
||||
intent = supervisor.intent
|
||||
if (
|
||||
intent is None
|
||||
or supervisor.device_network.state != "applied"
|
||||
@@ -2734,7 +2874,14 @@ class XgridsK1CompatibilityService:
|
||||
or supervisor.device_network.target != endpoint
|
||||
):
|
||||
return
|
||||
epoch = self._connection_supervisor.observe_host_path(path)
|
||||
epoch = self._connection_supervisor.observe_host_path_if_current(
|
||||
expected_intent_id=intent.intent_id,
|
||||
expected_target=endpoint,
|
||||
expected_host_path_observation=supervisor.host_path,
|
||||
result=path,
|
||||
)
|
||||
if epoch is None:
|
||||
return
|
||||
if path.available and reachable is not None:
|
||||
self._connection_supervisor.observe_endpoint(
|
||||
target=endpoint,
|
||||
@@ -3637,8 +3784,15 @@ class XgridsK1CompatibilityService:
|
||||
pending = self._pending_local_control_retirement
|
||||
if not pending:
|
||||
return True
|
||||
control = self._application_control_session.snapshot()
|
||||
try:
|
||||
self._retire_application_control_for_network_change()
|
||||
self._retire_application_control_for_network_change(
|
||||
# A failed worker may finish retiring after the first bounded
|
||||
# close attempt. Retrying that host-only retirement must not
|
||||
# remain permanently blocked by the terminal phase itself.
|
||||
# The durable physical-command row is intentionally untouched.
|
||||
allow_terminal_failure=control.get("state") == "failed",
|
||||
)
|
||||
except (AttributeError, RuntimeError):
|
||||
return False
|
||||
return True
|
||||
@@ -11616,6 +11770,21 @@ class XgridsK1CompatibilityService:
|
||||
and not reconciliation.observation.init_ready
|
||||
and not reconciliation.observation.mqtt_retained
|
||||
)
|
||||
ordinary_terminal_stop = bool(
|
||||
record.stage == "resolved"
|
||||
and record.resolution == "stop-standby-observed"
|
||||
and record.publish_call_returned is True
|
||||
and record.packet_id is not None
|
||||
and record.qos2_completed
|
||||
and record.application_response is not None
|
||||
and record.application_response.success
|
||||
and record.last_status is not None
|
||||
and record.last_status.source == "live-control-session"
|
||||
and record.last_status.session_state in {"ready", "scan_over"}
|
||||
and not record.last_status.project_bound
|
||||
and not record.last_status.init_ready
|
||||
and not record.last_status.mqtt_retained
|
||||
)
|
||||
if not (
|
||||
persisted is not None
|
||||
and status is not None
|
||||
@@ -11624,6 +11793,7 @@ class XgridsK1CompatibilityService:
|
||||
unresolved_stop
|
||||
or classified_terminal_stop
|
||||
or reconciled_ambiguous_terminal_stop
|
||||
or ordinary_terminal_stop
|
||||
)
|
||||
and record.acquisition_id == checkpoint.acquisition_id
|
||||
and record.identity.vendor_device_id_sha256
|
||||
@@ -13107,6 +13277,75 @@ class XgridsK1CompatibilityService:
|
||||
return False
|
||||
ledger_snapshot = self._physical_command_ledger.snapshot()
|
||||
record = ledger_snapshot.record
|
||||
ordinary_terminal_stop = bool(
|
||||
ledger_snapshot.status == "resolved"
|
||||
and record is not None
|
||||
and record.action == "stop"
|
||||
and record.stage == "resolved"
|
||||
and record.resolution == "stop-standby-observed"
|
||||
and record.publish_call_returned is True
|
||||
and record.packet_id is not None
|
||||
and record.qos2_completed
|
||||
and record.application_response is not None
|
||||
and record.application_response.success
|
||||
and record.last_status is not None
|
||||
and record.last_status.source == "live-control-session"
|
||||
and record.last_status.session_state in {"ready", "scan_over"}
|
||||
and not record.last_status.project_bound
|
||||
and not record.last_status.init_ready
|
||||
and not record.last_status.mqtt_retained
|
||||
and self._checkpoint_binding_matches_physical_connection(
|
||||
token.checkpoint.current_binding,
|
||||
record,
|
||||
)
|
||||
)
|
||||
if ordinary_terminal_stop:
|
||||
assert record is not None and record.last_status is not None
|
||||
store = self._active_acquisition_checkpoint
|
||||
if store is None or not self._active_acquisition_checkpoint_trust_token_is_current(
|
||||
token
|
||||
):
|
||||
return False
|
||||
try:
|
||||
binding = token.checkpoint.current_binding
|
||||
status_proof = self._checkpoint_status_proof(
|
||||
status=record.last_status,
|
||||
binding=binding,
|
||||
evidence_session_id=(
|
||||
token.checkpoint.current_evidence_session_id
|
||||
),
|
||||
)
|
||||
physical_proof = self._checkpoint_physical_proof(
|
||||
record=record,
|
||||
binding=binding,
|
||||
checkpoint=token.checkpoint,
|
||||
)
|
||||
store.cease(
|
||||
transition_id=self._active_acquisition_checkpoint_transition_id(
|
||||
"restart-ordinary-stop-cease",
|
||||
record.operation_id,
|
||||
record.revision,
|
||||
token.checkpoint_revision,
|
||||
),
|
||||
expected_revision=token.checkpoint_revision,
|
||||
expected_acquisition_id=token.acquisition_id,
|
||||
expected_start_operation_id=token.root_start_operation_id,
|
||||
physical_proof=physical_proof,
|
||||
status_proof=status_proof,
|
||||
)
|
||||
self._set_active_acquisition_checkpoint_reason(None)
|
||||
return True
|
||||
except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc:
|
||||
self._set_active_acquisition_checkpoint_reason(
|
||||
str(
|
||||
getattr(
|
||||
exc,
|
||||
"reason_code",
|
||||
"restart-ordinary-stop-checkpoint-settlement-failed",
|
||||
)
|
||||
)
|
||||
)
|
||||
return False
|
||||
reconciliation = (
|
||||
record.reconciliations[-1]
|
||||
if record is not None and record.reconciliations
|
||||
@@ -14314,8 +14553,8 @@ class XgridsK1CompatibilityService:
|
||||
self._last_live_data_suspend_aware = time.time()
|
||||
self._last_live_data_session_id = session_id
|
||||
|
||||
physical = self._physical_command_coordinator.snapshot()
|
||||
control = self._application_control_session.snapshot()
|
||||
now = time.monotonic()
|
||||
runtime_recovery = runtime.get("connection_recovery")
|
||||
with self._lock:
|
||||
acquisition = self._acquisition
|
||||
out_dir = self._acquisition_out_dir
|
||||
@@ -14328,6 +14567,35 @@ class XgridsK1CompatibilityService:
|
||||
return
|
||||
acquisition_id = acquisition.acquisition_id
|
||||
control_mode = acquisition.control_mode
|
||||
lineage = (acquisition_id, out_dir.name, producer_generation)
|
||||
active_recovery_lineage = self._active_stream_recovery_lineage
|
||||
post_recovery_restart_candidate = bool(
|
||||
control_mode == "plugin-commanded"
|
||||
and active_recovery_lineage is not None
|
||||
and active_recovery_lineage.acquisition_id == acquisition_id
|
||||
and active_recovery_lineage.evidence_session_id == out_dir.name
|
||||
and active_recovery_lineage.runtime_producer_generation
|
||||
== producer_generation
|
||||
and isinstance(runtime_recovery, Mapping)
|
||||
and runtime_recovery.get("state") == "recovered"
|
||||
)
|
||||
if not post_recovery_restart_candidate and (
|
||||
self._camera_activation_lineage == lineage
|
||||
or (
|
||||
self._camera_activation_retry_lineage == lineage
|
||||
and now < self._camera_activation_retry_not_before_monotonic
|
||||
)
|
||||
):
|
||||
# The exact lineage already owns its one camera activation (or
|
||||
# its bounded local retry delay). Repeating the durable physical
|
||||
# ledger/checkpoint admission for every 10 Hz PCL blocks this
|
||||
# publisher thread and evicts intervening pose frames. The
|
||||
# data-plane liveness edge above remains per-frame; recovery
|
||||
# deliberately bypasses this ordinary idempotence fast path.
|
||||
return
|
||||
|
||||
physical = self._physical_command_coordinator.snapshot()
|
||||
control = self._application_control_session.snapshot()
|
||||
|
||||
if control_mode == "plugin-commanded":
|
||||
start_operation_id = (
|
||||
@@ -14364,21 +14632,10 @@ class XgridsK1CompatibilityService:
|
||||
# this local admission after the durable SCANNING proof arrives.
|
||||
return
|
||||
|
||||
lineage = (acquisition_id, out_dir.name, producer_generation)
|
||||
now = time.monotonic()
|
||||
camera = self.camera_preview.snapshot()
|
||||
with self._lock:
|
||||
active_recovery_lineage = self._active_stream_recovery_lineage
|
||||
runtime_recovery = runtime.get("connection_recovery")
|
||||
if (
|
||||
control_mode == "plugin-commanded"
|
||||
post_recovery_restart_candidate
|
||||
and active_recovery_lineage is not None
|
||||
and active_recovery_lineage.acquisition_id == acquisition_id
|
||||
and active_recovery_lineage.evidence_session_id == out_dir.name
|
||||
and active_recovery_lineage.runtime_producer_generation
|
||||
== producer_generation
|
||||
and isinstance(runtime_recovery, Mapping)
|
||||
and runtime_recovery.get("state") == "recovered"
|
||||
and self._enqueue_post_recovery_camera_restart(
|
||||
active_recovery_lineage,
|
||||
runtime=runtime,
|
||||
@@ -14888,6 +15145,10 @@ class XgridsK1CompatibilityService:
|
||||
or control_after.get("session_generation")
|
||||
!= control_before.get("session_generation")
|
||||
)
|
||||
control_stage = "post-device-info-transport-refresh"
|
||||
await self._refresh_aged_verify_transport_owned(
|
||||
application_control_session=control_after,
|
||||
)
|
||||
control_stage = "physical-reconciliation"
|
||||
physical_reconciliation = await self._reconcile_physical_command_after_verify_owned(
|
||||
verify_operation_id=verify_operation_id,
|
||||
@@ -14895,6 +15156,12 @@ class XgridsK1CompatibilityService:
|
||||
if provisional_topology is not None:
|
||||
control_stage = "semantic-topology-commit"
|
||||
self._commit_provisional_fresh_bridge_topology(provisional_topology)
|
||||
control_stage = "post-reconciliation-transport-refresh"
|
||||
await self._refresh_aged_verify_transport_owned(
|
||||
application_control_session=dict(
|
||||
self._application_control_session.snapshot()
|
||||
),
|
||||
)
|
||||
control_stage = "final-control-binding"
|
||||
ready = self._connection_supervisor.snapshot()
|
||||
if not (
|
||||
@@ -23267,6 +23534,10 @@ class XgridsK1CompatibilityService:
|
||||
)
|
||||
use_reconciliation = bool(
|
||||
reconciliation is not None
|
||||
and self._physical_reconciliation_belongs_to_record(
|
||||
reconciliation,
|
||||
record,
|
||||
)
|
||||
and reconciliation.kind
|
||||
in {
|
||||
"ambiguous-outcome",
|
||||
@@ -24458,6 +24729,9 @@ class XgridsK1CompatibilityService:
|
||||
observation = await _run_blocking_operation_without_abandonment(
|
||||
self._probe_control_endpoint,
|
||||
lineage.target_ipv4,
|
||||
association_timeout_seconds=(
|
||||
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS
|
||||
),
|
||||
)
|
||||
if not observation.path.available or not observation.reachable:
|
||||
with self._lock:
|
||||
@@ -27846,6 +28120,15 @@ class XgridsK1CompatibilityService:
|
||||
self._physical_command_coordinator.snapshot()
|
||||
) is None:
|
||||
return "not-applicable"
|
||||
# A live START worker cannot be settled. Avoid touching the exact
|
||||
# mark_dispatching -> publish fence on every passive state read while
|
||||
# it is still initializing; the terminal branch below re-samples the
|
||||
# worker under that fence before committing any no-dispatch result.
|
||||
if self._application_control_session.snapshot().get("state") not in {
|
||||
"failed",
|
||||
"closed",
|
||||
}:
|
||||
return "not-applicable"
|
||||
if not self._k1_command_dispatch_gate.acquire(blocking=False):
|
||||
return "defer"
|
||||
try:
|
||||
@@ -28371,6 +28654,59 @@ class XgridsK1CompatibilityService:
|
||||
and durable_start_active_confirmed
|
||||
and durable_checkpoint_active_confirmed
|
||||
)
|
||||
terminal_control_failure = (
|
||||
terminal_control_proof.get("failure")
|
||||
if isinstance(terminal_control_proof, Mapping)
|
||||
else None
|
||||
)
|
||||
terminal_control_transport = (
|
||||
terminal_control_proof.get("transport")
|
||||
if isinstance(terminal_control_proof, Mapping)
|
||||
else None
|
||||
)
|
||||
terminal_verified_control = (
|
||||
terminal_control_proof.get("verified_control")
|
||||
if isinstance(terminal_control_proof, Mapping)
|
||||
else None
|
||||
)
|
||||
device_reported_scan_over_without_stop = bool(
|
||||
current is not None
|
||||
and current.control_mode == "plugin-commanded"
|
||||
and current.state in {"starting", "awaiting_external_start", "acquiring"}
|
||||
and current_stop_operation_id is None
|
||||
and isinstance(terminal_control_proof, Mapping)
|
||||
and terminal_control_proof.get("state") == "failed"
|
||||
and isinstance(terminal_control_failure, Mapping)
|
||||
and terminal_control_failure.get("failed_phase") == "scanning"
|
||||
and terminal_control_failure.get("modeling_command_attempted") is True
|
||||
and terminal_control_failure.get("stop_command_attempted") is False
|
||||
and terminal_control_failure.get("diagnostic_snapshot_unavailable") == []
|
||||
and terminal_control_failure.get("diagnostic_evidence_unavailable") == []
|
||||
and isinstance(terminal_control_transport, Mapping)
|
||||
and isinstance(terminal_control_transport.get("device_status_reports"), int)
|
||||
and not isinstance(
|
||||
terminal_control_transport.get("device_status_reports"),
|
||||
bool,
|
||||
)
|
||||
and int(terminal_control_transport["device_status_reports"]) > 0
|
||||
and terminal_control_transport.get("latest_device_session_state")
|
||||
== "scan_over"
|
||||
and terminal_control_transport.get("latest_system_error_code") is None
|
||||
and isinstance(physical_command_proof, Mapping)
|
||||
and physical_command_proof.get("runtime_bound") is True
|
||||
and physical_command_proof.get("reconciliation_ready") is True
|
||||
and physical_command_proof.get("observed_session_state") == "scan_over"
|
||||
and self._matching_start_active_confirmed(
|
||||
physical_command_proof,
|
||||
acquisition_id=current_acquisition_id,
|
||||
start_operation_id=canonical_start_operation_id,
|
||||
verified_control=(
|
||||
terminal_verified_control
|
||||
if isinstance(terminal_verified_control, Mapping)
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
# Camera-only transport loss is supervised by the backend producer
|
||||
# watchdog and exact camera CAS. Snapshot polling is observational: it
|
||||
# must not move an otherwise live MQTT/PCL runtime to ``reconnecting``.
|
||||
@@ -28459,6 +28795,21 @@ class XgridsK1CompatibilityService:
|
||||
and stop_response_accepted_without_terminal_status
|
||||
and self._operations.deadline_reached(current_stop_operation_id)
|
||||
)
|
||||
terminal_unknown_stop_control = bool(
|
||||
current is not None
|
||||
and current.control_mode == "plugin-commanded"
|
||||
and current.state == "awaiting_external_stop"
|
||||
and application_control_session.get("state") == "failed"
|
||||
and application_control_session.get("outcome_unknown") is True
|
||||
and isinstance(application_control_session.get("failure"), Mapping)
|
||||
and application_control_session["failure"].get("stop_command_attempted") is True
|
||||
and self._matching_unresolved_stop_stage(
|
||||
physical_command_proof,
|
||||
acquisition_id=current_acquisition_id,
|
||||
stop_operation_id=current_stop_operation_id,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
with self._lock:
|
||||
canonical_stop_standby_confirmed = bool(
|
||||
checkpoint_stop_completion_seen and checkpoint_stop_ceased
|
||||
@@ -28504,7 +28855,11 @@ class XgridsK1CompatibilityService:
|
||||
and stop_response_deadline_reached
|
||||
else None
|
||||
)
|
||||
if stop_response_deadline_reached:
|
||||
if terminal_unknown_stop_control or stop_response_deadline_reached:
|
||||
# Retire only the failed host-owned control socket. In particular,
|
||||
# an UNKNOWN physical STOP keeps the capture, viewer and acquisition
|
||||
# alive so an explicit read-only DeviceInfo/DeviceStatus dialogue
|
||||
# can classify SCANNING or READY without a second command.
|
||||
self._retire_local_control_after_stop_timeout()
|
||||
if terminal_unknown_stop_cleanup:
|
||||
try:
|
||||
@@ -28858,6 +29213,39 @@ class XgridsK1CompatibilityService:
|
||||
# read-only classification and seal the old STOP as
|
||||
# non-replayable/none.
|
||||
pass
|
||||
elif device_reported_scan_over_without_stop:
|
||||
# This is not an inference from a quiet camera or point stream.
|
||||
# The continuously DeviceInfo-bound control socket decoded a
|
||||
# fresh, non-retained SCAN_OVER from this exact K1, and the
|
||||
# physical coordinator independently retained the same bound
|
||||
# status. No canonical STOP was requested or attempted.
|
||||
#
|
||||
# SCAN_OVER proves that this acquisition cannot be resumed. It
|
||||
# does not prove READY and does not authorize a new START; the
|
||||
# durable physical/checkpoint chain remains fenced until a later
|
||||
# explicit read-only reconciliation observes the returned K1.
|
||||
acquisition.transition(
|
||||
"finalizing",
|
||||
message_code="acquisition.recovery.device_standby_observed",
|
||||
)
|
||||
camera_terminal_status = "interrupted"
|
||||
camera_failure_code = "device-reported-scan-over"
|
||||
recovery_standby_after_seal = True
|
||||
if point_frames <= 0:
|
||||
recovery_standby_start_operation_id = (
|
||||
self._acquisition_start_operation_id
|
||||
)
|
||||
completion_message_code = (
|
||||
"acquisition.recovery.device_standby_observed"
|
||||
)
|
||||
completion_result = {
|
||||
"receiver_stopped": True,
|
||||
"device_state": "scan_over",
|
||||
"device_stop": "not-sent",
|
||||
"automatic_command_retry": False,
|
||||
"read_only_recovery": True,
|
||||
"physical_reconciliation_required": True,
|
||||
}
|
||||
elif prepared_stop_recovery_standby:
|
||||
acquisition.transition(
|
||||
"finalizing",
|
||||
@@ -29815,6 +30203,28 @@ class XgridsK1PluginFacade:
|
||||
|
||||
def __init__(self, service: XgridsK1ServicePort) -> None:
|
||||
self.service = service
|
||||
self._state_read_task: asyncio.Task[dict[str, Any]] | None = None
|
||||
|
||||
def _retire_state_read_task(self, task: asyncio.Task[dict[str, Any]]) -> None:
|
||||
if self._state_read_task is task:
|
||||
self._state_read_task = None
|
||||
with suppress(asyncio.CancelledError):
|
||||
task.exception()
|
||||
|
||||
async def _read_state_single_flight(self) -> dict[str, Any]:
|
||||
loop = asyncio.get_running_loop()
|
||||
task = self._state_read_task
|
||||
if task is None or task.done() or task.get_loop() is not loop:
|
||||
task = loop.create_task(
|
||||
asyncio.to_thread(self.service.state),
|
||||
name="xgrids-k1-state-read",
|
||||
)
|
||||
self._state_read_task = task
|
||||
task.add_done_callback(self._retire_state_read_task)
|
||||
# An HTTP disconnect cancels only that request waiter. The shared
|
||||
# read keeps running so a later poll joins it instead of stacking a
|
||||
# second service.state thread behind the lifecycle gate.
|
||||
return await asyncio.shield(task)
|
||||
|
||||
async def invoke(self, invocation: RuntimeActionInvocation) -> dict[str, Any]:
|
||||
if invocation.plugin_id != self.plugin_id:
|
||||
@@ -29887,7 +30297,7 @@ class XgridsK1PluginFacade:
|
||||
self.service.bind_runtime_event_loop(asyncio.get_running_loop())
|
||||
if action_id == ACTION_STATE_READ:
|
||||
EmptyRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(self.service.state)
|
||||
return await self._read_state_single_flight()
|
||||
if action_id == ACTION_DISCOVERY_SCAN:
|
||||
scan_request = BleScanRequest.model_validate(payload)
|
||||
return await self.service.scan_ble(scan_request)
|
||||
@@ -29899,7 +30309,7 @@ class XgridsK1PluginFacade:
|
||||
EmptyRequest.model_validate(payload)
|
||||
if action_id == ACTION_DEVICE_INSPECT:
|
||||
return await asyncio.to_thread(self.service.inspect_device)
|
||||
return await asyncio.to_thread(self.service.state)
|
||||
return await self._read_state_single_flight()
|
||||
if action_id == ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ:
|
||||
EmptyRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(self.service.read_device_calibration_snapshot)
|
||||
@@ -29923,7 +30333,7 @@ class XgridsK1PluginFacade:
|
||||
return await self.service.probe_configured_endpoint(probe_request)
|
||||
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_STATE:
|
||||
EmptyRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(self.service.state)
|
||||
return await self._read_state_single_flight()
|
||||
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_ARM:
|
||||
arm_request = ShadowApplicationControlArmRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(
|
||||
|
||||
@@ -696,6 +696,7 @@ class VisualizationRuntime:
|
||||
|
||||
def publish_stream(bridge: RerunBridge) -> None:
|
||||
serve_perception_next = False
|
||||
serve_non_point_cloud_next = False
|
||||
while (
|
||||
not source_done.is_set()
|
||||
or not preview_empty()
|
||||
@@ -708,15 +709,16 @@ class VisualizationRuntime:
|
||||
and preview_empty()
|
||||
):
|
||||
return
|
||||
# A dedicated latest-PCL slot plus this fixed service order
|
||||
# guarantees visible-scene progress even under an unbounded
|
||||
# stream of perception results. Other MQTT preview traffic is
|
||||
# then served before one perception result, so neither class can
|
||||
# monopolize the single publisher thread.
|
||||
live_point_cloud = pop_latest_live_point_cloud()
|
||||
if live_point_cloud is not None:
|
||||
publish_message(bridge, live_point_cloud)
|
||||
continue
|
||||
# The latest-PCL slot preserves visible-scene recovery, but a
|
||||
# continuously full slot must not starve pose. Alternate one
|
||||
# live PCL with one available non-PCL item; when no other item
|
||||
# exists, the fallback below keeps PCL flowing without delay.
|
||||
if not serve_non_point_cloud_next:
|
||||
live_point_cloud = pop_latest_live_point_cloud()
|
||||
if live_point_cloud is not None:
|
||||
publish_message(bridge, live_point_cloud)
|
||||
serve_non_point_cloud_next = True
|
||||
continue
|
||||
if serve_perception_next:
|
||||
try:
|
||||
perception = self._perception_messages.get_nowait()
|
||||
@@ -728,6 +730,7 @@ class VisualizationRuntime:
|
||||
finally:
|
||||
self._perception_messages.task_done()
|
||||
serve_perception_next = False
|
||||
serve_non_point_cloud_next = False
|
||||
continue
|
||||
try:
|
||||
message = messages.get_nowait()
|
||||
@@ -739,6 +742,7 @@ class VisualizationRuntime:
|
||||
finally:
|
||||
messages.task_done()
|
||||
serve_perception_next = True
|
||||
serve_non_point_cloud_next = False
|
||||
continue
|
||||
try:
|
||||
perception = self._perception_messages.get_nowait()
|
||||
@@ -750,6 +754,12 @@ class VisualizationRuntime:
|
||||
finally:
|
||||
self._perception_messages.task_done()
|
||||
serve_perception_next = False
|
||||
serve_non_point_cloud_next = False
|
||||
continue
|
||||
live_point_cloud = pop_latest_live_point_cloud()
|
||||
if live_point_cloud is not None:
|
||||
publish_message(bridge, live_point_cloud)
|
||||
serve_non_point_cloud_next = True
|
||||
continue
|
||||
live_point_cloud_ready.wait(timeout=0.05)
|
||||
|
||||
|
||||
+59
-11
@@ -355,24 +355,66 @@ def newly_finalized_recording_ids(
|
||||
return tuple(sorted(set(current_finalized) - known_finalized))
|
||||
|
||||
|
||||
def observation_archive_revision(roots: Iterable[Path]) -> tuple[tuple[object, ...], ...]:
|
||||
"""Return a cheap change fence for direct-child observation archives.
|
||||
|
||||
The K1 writer creates and retires ``.current_session`` in the archive root,
|
||||
and every session directory is a direct child of that same root. Those
|
||||
operations advance the directory metadata, while growing capture/media
|
||||
files do not. This lets the reconciler notice session start, completion and
|
||||
removal without recursively reopening tens of thousands of immutable
|
||||
evidence files every two seconds.
|
||||
"""
|
||||
|
||||
revisions: list[tuple[object, ...]] = []
|
||||
for configured_root in roots:
|
||||
root = configured_root.expanduser().resolve(strict=False)
|
||||
try:
|
||||
metadata = root.lstat()
|
||||
except OSError:
|
||||
revisions.append((str(root), None))
|
||||
continue
|
||||
revisions.append(
|
||||
(
|
||||
str(root),
|
||||
metadata.st_dev,
|
||||
metadata.st_ino,
|
||||
metadata.st_mtime_ns,
|
||||
metadata.st_ctime_ns,
|
||||
)
|
||||
)
|
||||
revisions.sort(key=lambda item: str(item[0]))
|
||||
return tuple(revisions)
|
||||
|
||||
|
||||
async def _recording_preparation_reconciler() -> None:
|
||||
"""Prepare sessions finalized during this process, never historical rows."""
|
||||
|
||||
known_finalized: set[str] | None = None
|
||||
reconciled_archive_revision: tuple[tuple[object, ...], ...] | None = None
|
||||
while True:
|
||||
try:
|
||||
await asyncio.to_thread(refresh_observation_catalog)
|
||||
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
||||
newly_finalized = newly_finalized_recording_ids(
|
||||
known_finalized,
|
||||
finalized,
|
||||
archive_revision = await asyncio.to_thread(
|
||||
observation_archive_revision,
|
||||
(archive.root for archive in plugin_environment.observation_archives),
|
||||
)
|
||||
if newly_finalized:
|
||||
await asyncio.to_thread(
|
||||
enqueue_replayable_recordings,
|
||||
newly_finalized,
|
||||
if archive_revision != reconciled_archive_revision:
|
||||
await asyncio.to_thread(refresh_observation_catalog)
|
||||
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
||||
newly_finalized = newly_finalized_recording_ids(
|
||||
known_finalized,
|
||||
finalized,
|
||||
)
|
||||
known_finalized = finalized
|
||||
if newly_finalized:
|
||||
await asyncio.to_thread(
|
||||
enqueue_replayable_recordings,
|
||||
newly_finalized,
|
||||
)
|
||||
known_finalized = finalized
|
||||
# Keep the pre-scan revision. If a writer changed the archive
|
||||
# during the expensive discovery, the next two-second check
|
||||
# observes the newer revision and performs one follow-up pass.
|
||||
reconciled_archive_revision = archive_revision
|
||||
recording_reconciler_readiness.record_success()
|
||||
except Exception as exc:
|
||||
# A transient filesystem/catalog failure must not permanently
|
||||
@@ -496,6 +538,12 @@ _CLOSED_WEBSOCKET_SEND_ERRORS = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
# This stream carries the comparatively heavy control-state snapshot, not
|
||||
# camera or point-cloud media. Action requests return their own result, and the
|
||||
# frontend also keeps a four-second REST fallback, so a two-second passive
|
||||
# cadence preserves status recovery without starving the live media sockets.
|
||||
DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS = 2.0
|
||||
|
||||
|
||||
def _is_closed_websocket_send_error(exc: RuntimeError) -> bool:
|
||||
message = " ".join(str(exc).split())
|
||||
@@ -528,7 +576,7 @@ async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
|
||||
if _is_closed_websocket_send_error(exc):
|
||||
return
|
||||
raise
|
||||
await asyncio.sleep(0.5)
|
||||
await asyncio.sleep(DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS)
|
||||
except (PluginNotFoundError, PluginActionNotFoundError):
|
||||
await websocket.close(code=1008, reason="Device plugin is not available")
|
||||
except (PluginExecutionError, PluginRuntimeUnavailableError):
|
||||
|
||||
@@ -326,6 +326,7 @@ def test_live_rerun_process_and_close_failure_preserve_capture_then_recover(
|
||||
allow_second_bridge = threading.Event()
|
||||
second_bridge_ready = threading.Event()
|
||||
recovery_confirmed = threading.Event()
|
||||
raw_capture_completed = threading.Event()
|
||||
raw_sequences: list[int] = []
|
||||
should_stop_before_explicit_stop: list[bool] = []
|
||||
confirmed_attempts: list[int] = []
|
||||
@@ -389,6 +390,7 @@ def test_live_rerun_process_and_close_failure_preserve_capture_then_recover(
|
||||
assert second_bridge_ready.wait(timeout=2.0)
|
||||
raw_sequences.append(10)
|
||||
enqueue(_captured_live_point_cloud(10)) # type: ignore[operator]
|
||||
raw_capture_completed.set()
|
||||
assert recovery_confirmed.wait(timeout=2.0)
|
||||
|
||||
while not should_stop(): # type: ignore[operator]
|
||||
@@ -409,6 +411,7 @@ def test_live_rerun_process_and_close_failure_preserve_capture_then_recover(
|
||||
)
|
||||
|
||||
assert recovery_confirmed.wait(timeout=3.0)
|
||||
assert raw_capture_completed.wait(timeout=3.0)
|
||||
snapshot = runtime.snapshot()
|
||||
assert snapshot["phase"] == "live"
|
||||
assert snapshot["source_ready"] is True
|
||||
@@ -510,6 +513,66 @@ def test_live_latest_point_cloud_survives_pose_and_perception_pressure(
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_live_pose_is_published_during_continuous_point_cloud_pressure(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
producer_started = threading.Event()
|
||||
producer_finished = threading.Event()
|
||||
pose_published_during_pressure = threading.Event()
|
||||
|
||||
class SlowPointCloudBridge(MetricRuntimeBridgeStub):
|
||||
def process(self, envelope: object) -> None:
|
||||
if isinstance(envelope, DecodedPointCloudView):
|
||||
# Keep another PCL waiting in the latest-wins slot. The
|
||||
# scheduler must still admit pose while pressure continues.
|
||||
time.sleep(0.01)
|
||||
elif (
|
||||
isinstance(envelope, DecodedPoseView)
|
||||
and not producer_finished.is_set()
|
||||
):
|
||||
pose_published_during_pressure.set()
|
||||
super().process(envelope)
|
||||
|
||||
def fake_capture_mqtt(
|
||||
_host: str,
|
||||
_out_dir: Path,
|
||||
**callbacks: object,
|
||||
) -> dict[str, object]:
|
||||
callbacks["on_clock_established"]() # type: ignore[operator]
|
||||
callbacks["on_ready"]() # type: ignore[operator]
|
||||
enqueue = callbacks["on_message_recorded"]
|
||||
producer_started.set()
|
||||
for index in range(300):
|
||||
enqueue(_captured_live_point_cloud(index * 2 + 1)) # type: ignore[operator]
|
||||
enqueue(_captured_live_pose(index * 2 + 2)) # type: ignore[operator]
|
||||
time.sleep(0.001)
|
||||
producer_finished.set()
|
||||
|
||||
should_stop = callbacks["should_stop"]
|
||||
while not should_stop(): # type: ignore[operator]
|
||||
time.sleep(0.005)
|
||||
return {"message_count": 600}
|
||||
|
||||
monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt)
|
||||
runtime = VisualizationRuntime(
|
||||
bridge_factory=lambda **kwargs: SlowPointCloudBridge(kwargs["metrics"]), # type: ignore[arg-type]
|
||||
normalizer=normalize_k1_message,
|
||||
)
|
||||
runtime.start_live(
|
||||
"192.168.1.50",
|
||||
tmp_path / "pose-fairness",
|
||||
duration_seconds=None,
|
||||
project_name="POSEFAIRNESS001",
|
||||
)
|
||||
|
||||
assert producer_started.wait(timeout=2.0)
|
||||
assert producer_finished.wait(timeout=2.0)
|
||||
runtime.stop(wait_seconds=2.0)
|
||||
runtime.close()
|
||||
assert pose_published_during_pressure.is_set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attempt", [5, 1025, 10**100])
|
||||
def test_live_rerun_recovery_backoff_saturates(attempt: int) -> None:
|
||||
assert runtime_module._rerun_recovery_backoff_seconds(attempt) == 5.0 # noqa: SLF001
|
||||
|
||||
@@ -1817,7 +1817,7 @@ def test_same_path_positive_reducer_observation_resets_technical_failure_streak(
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak() -> None:
|
||||
def test_unrelated_supervisor_revisions_preserve_exact_configured_route() -> None:
|
||||
async def scenario() -> None:
|
||||
supervisor, epoch = _configured_unverified_supervisor()
|
||||
path = _available_path()
|
||||
@@ -1836,8 +1836,12 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
||||
async def host_probe(_target: EndpointTarget) -> HostPathProbeResult:
|
||||
return technical_timeout
|
||||
|
||||
tcp_calls = 0
|
||||
|
||||
async def tcp_probe(_target: EndpointTarget) -> bool:
|
||||
raise AssertionError("technical failure must skip TCP")
|
||||
nonlocal tcp_calls
|
||||
tcp_calls += 1
|
||||
return True
|
||||
|
||||
monitor = ReadOnlyConnectionMonitor(
|
||||
supervisor,
|
||||
@@ -1847,8 +1851,10 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
||||
)
|
||||
|
||||
first = await monitor.poll_once()
|
||||
assert first.revision == initial.revision
|
||||
assert first.host_path is initial.host_path
|
||||
assert first.host_path.available is True
|
||||
assert first.host_path.epoch == initial.host_path.epoch
|
||||
assert first.endpoint.tcp_state == "reachable"
|
||||
assert first.authority.control_allowed is False
|
||||
assert supervisor.observe_endpoint(
|
||||
target=TARGET,
|
||||
intent_id="bridge-1",
|
||||
@@ -1857,11 +1863,14 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
||||
)
|
||||
endpoint_refresh = supervisor.snapshot()
|
||||
assert endpoint_refresh.revision > first.revision
|
||||
assert endpoint_refresh.host_path is initial.host_path
|
||||
assert endpoint_refresh.host_path.epoch == initial.host_path.epoch
|
||||
assert endpoint_refresh.host_path.fingerprint == initial.host_path.fingerprint
|
||||
|
||||
second = await monitor.poll_once()
|
||||
assert second.revision == endpoint_refresh.revision
|
||||
assert second.host_path is initial.host_path
|
||||
assert second.host_path.available is True
|
||||
assert second.host_path.epoch == initial.host_path.epoch
|
||||
assert second.endpoint.tcp_state == "reachable"
|
||||
assert second.authority.control_allowed is False
|
||||
assert supervisor.observe_endpoint(
|
||||
target=TARGET,
|
||||
intent_id="bridge-1",
|
||||
@@ -1870,15 +1879,17 @@ def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak()
|
||||
)
|
||||
|
||||
confirmed = await monitor.poll_once()
|
||||
assert confirmed.host_path.available is False
|
||||
assert confirmed.host_path.reason_code == "host-wifi-operation-timeout"
|
||||
assert confirmed.host_path.available is True
|
||||
assert confirmed.host_path.epoch == initial.host_path.epoch
|
||||
assert confirmed.endpoint.tcp_state == "reachable"
|
||||
assert confirmed.authority.control_allowed is False
|
||||
assert tcp_calls == 3
|
||||
await monitor.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_inflight_third_timeout_cannot_overwrite_a_concurrent_positive_observation() -> None:
|
||||
def test_inflight_timeout_cannot_overwrite_a_concurrent_positive_observation() -> None:
|
||||
async def scenario() -> None:
|
||||
supervisor, initial_epoch = _configured_unverified_supervisor()
|
||||
path = _available_path()
|
||||
@@ -1892,20 +1903,20 @@ def test_inflight_third_timeout_cannot_overwrite_a_concurrent_positive_observati
|
||||
observation_failure_class="association-observer",
|
||||
kernel_route_fingerprint=path.kernel_route_fingerprint,
|
||||
)
|
||||
third_probe_entered = asyncio.Event()
|
||||
release_third_probe = asyncio.Event()
|
||||
timeout_probe_entered = asyncio.Event()
|
||||
release_timeout_probe = asyncio.Event()
|
||||
host_calls = 0
|
||||
|
||||
async def host_probe(_target: EndpointTarget) -> HostPathProbeResult:
|
||||
nonlocal host_calls
|
||||
host_calls += 1
|
||||
if host_calls == 3:
|
||||
third_probe_entered.set()
|
||||
await release_third_probe.wait()
|
||||
if host_calls == 1:
|
||||
timeout_probe_entered.set()
|
||||
await release_timeout_probe.wait()
|
||||
return technical_timeout
|
||||
|
||||
async def tcp_probe(_target: EndpointTarget) -> bool:
|
||||
raise AssertionError("technical failure must skip TCP")
|
||||
return True
|
||||
|
||||
monitor = ReadOnlyConnectionMonitor(
|
||||
supervisor,
|
||||
@@ -1913,32 +1924,20 @@ def test_inflight_third_timeout_cannot_overwrite_a_concurrent_positive_observati
|
||||
tcp_probe=tcp_probe,
|
||||
target_provider=lambda: TARGET,
|
||||
)
|
||||
first = await monitor.poll_once()
|
||||
second = await monitor.poll_once()
|
||||
assert first.authority.control_allowed is False
|
||||
assert second.authority.control_allowed is False
|
||||
|
||||
inflight_third = asyncio.create_task(monitor.poll_once())
|
||||
await third_probe_entered.wait()
|
||||
inflight_timeout = asyncio.create_task(monitor.poll_once())
|
||||
await timeout_probe_entered.wait()
|
||||
refreshed_epoch = supervisor.observe_host_path(path)
|
||||
assert refreshed_epoch == initial_epoch
|
||||
external_positive = supervisor.snapshot()
|
||||
release_third_probe.set()
|
||||
raced_timeout = await inflight_third
|
||||
release_timeout_probe.set()
|
||||
raced_timeout = await inflight_timeout
|
||||
|
||||
assert raced_timeout.revision == external_positive.revision
|
||||
assert raced_timeout.host_path is external_positive.host_path
|
||||
assert raced_timeout.revision >= external_positive.revision
|
||||
assert raced_timeout.host_path.available is True
|
||||
assert raced_timeout.host_path.epoch == external_positive.host_path.epoch
|
||||
assert raced_timeout.host_path.fingerprint == external_positive.host_path.fingerprint
|
||||
assert raced_timeout.endpoint.tcp_state == "reachable"
|
||||
assert raced_timeout.authority.control_allowed is False
|
||||
|
||||
second_after_positive = await monitor.poll_once()
|
||||
assert second_after_positive.revision == external_positive.revision
|
||||
assert second_after_positive.host_path is external_positive.host_path
|
||||
assert second_after_positive.authority.control_allowed is False
|
||||
|
||||
confirmed = await monitor.poll_once()
|
||||
assert confirmed.host_path.available is False
|
||||
assert confirmed.host_path.reason_code == "host-wifi-operation-timeout"
|
||||
assert confirmed.authority.control_allowed is False
|
||||
await monitor.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
@@ -37,6 +37,7 @@ from k1link.device_plugins.xgrids_k1.facade import (
|
||||
ACTION_PHYSICAL_COMMAND_RECONCILE,
|
||||
ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION,
|
||||
ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE,
|
||||
ACTION_STATE_READ,
|
||||
ACTION_STREAM_START_LIVE,
|
||||
ACTION_STREAM_START_REPLAY,
|
||||
ACTION_STREAM_STOP,
|
||||
@@ -1285,6 +1286,57 @@ def test_sync_runtime_actions_run_outside_the_api_event_loop() -> None:
|
||||
assert service.thread_id != event_loop_thread
|
||||
|
||||
|
||||
def test_state_reads_are_single_flight_and_survive_one_cancelled_waiter() -> None:
|
||||
class BlockingStateService(FakeXgridsService):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.state_calls = 0
|
||||
self.started = threading.Event()
|
||||
self.release = threading.Event()
|
||||
|
||||
def state(self) -> dict[str, Any]:
|
||||
self.state_calls += 1
|
||||
self.started.set()
|
||||
if not self.release.wait(timeout=2):
|
||||
raise AssertionError("state read was not released by the test")
|
||||
return {"phase": "idle", "revision": self.state_calls}
|
||||
|
||||
service = BlockingStateService()
|
||||
adapter = XgridsK1PluginFacade(service)
|
||||
|
||||
def invocation(invocation_id: str) -> RuntimeActionInvocation:
|
||||
return RuntimeActionInvocation(
|
||||
invocation_id=invocation_id,
|
||||
plugin_id=adapter.plugin_id,
|
||||
action_id=ACTION_STATE_READ,
|
||||
requested_at=datetime.now(UTC),
|
||||
parameters={},
|
||||
)
|
||||
|
||||
async def exercise() -> None:
|
||||
first = asyncio.create_task(adapter.invoke(invocation("state-read-first")))
|
||||
while not service.started.is_set():
|
||||
await asyncio.sleep(0)
|
||||
second = asyncio.create_task(adapter.invoke(invocation("state-read-second")))
|
||||
await asyncio.sleep(0.02)
|
||||
assert service.state_calls == 1
|
||||
|
||||
first.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await first
|
||||
assert service.state_calls == 1
|
||||
|
||||
service.release.set()
|
||||
assert await second == {"phase": "idle", "revision": 1}
|
||||
assert await adapter.invoke(invocation("state-read-fresh")) == {
|
||||
"phase": "idle",
|
||||
"revision": 2,
|
||||
}
|
||||
|
||||
asyncio.run(exercise())
|
||||
assert service.state_calls == 2
|
||||
|
||||
|
||||
def test_runtime_requires_successful_handshake_before_dispatch() -> None:
|
||||
adapter = XgridsK1PluginFacade(FakeXgridsService())
|
||||
runtime = _in_process_runtime(adapter, activate=False)
|
||||
|
||||
@@ -200,6 +200,33 @@ def test_startup_scan_baselines_historical_sessions_without_enqueuing(
|
||||
manager.close()
|
||||
|
||||
|
||||
def test_archive_revision_tracks_session_lifecycle_without_capture_churn(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
sessions = tmp_path / "sessions"
|
||||
sessions.mkdir()
|
||||
initial = app_module.observation_archive_revision((sessions,))
|
||||
|
||||
session = sessions / "20260716T205632Z_viewer_live"
|
||||
session.mkdir()
|
||||
started = app_module.observation_archive_revision((sessions,))
|
||||
assert started != initial
|
||||
|
||||
capture = session / "captures" / "mqtt_live"
|
||||
capture.mkdir(parents=True)
|
||||
(capture / "mqtt.raw.k1mqtt").write_bytes(b"growing-capture")
|
||||
assert app_module.observation_archive_revision((sessions,)) == started
|
||||
|
||||
marker = sessions / ".current_session"
|
||||
marker.write_text(f"{session.name}\n", encoding="utf-8")
|
||||
active = app_module.observation_archive_revision((sessions,))
|
||||
assert active != started
|
||||
|
||||
marker.unlink()
|
||||
finalized = app_module.observation_archive_revision((sessions,))
|
||||
assert finalized != active
|
||||
|
||||
|
||||
def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
@@ -40,6 +40,24 @@ class _FailingSendWebSocket:
|
||||
del code, reason
|
||||
|
||||
|
||||
class _DisconnectAfterSecondSendWebSocket:
|
||||
def __init__(self) -> None:
|
||||
self.accepted = False
|
||||
self.send_count = 0
|
||||
|
||||
async def accept(self) -> None:
|
||||
self.accepted = True
|
||||
|
||||
async def send_json(self, payload: dict[str, Any]) -> None:
|
||||
del payload
|
||||
self.send_count += 1
|
||||
if self.send_count == 2:
|
||||
raise WebSocketDisconnect(code=1001)
|
||||
|
||||
async def close(self, *, code: int, reason: str) -> None:
|
||||
del code, reason
|
||||
|
||||
|
||||
async def _post_json(path: str, payload: dict[str, Any]) -> tuple[int, str]:
|
||||
body = json.dumps(payload).encode()
|
||||
request_sent = False
|
||||
@@ -198,6 +216,25 @@ def test_device_plugin_events_treats_proven_transport_disconnect_as_completion(
|
||||
assert websocket.accepted is True
|
||||
|
||||
|
||||
def test_device_plugin_events_keeps_heavy_state_poll_off_live_media_cadence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
websocket = _DisconnectAfterSecondSendWebSocket()
|
||||
sleeps: list[float] = []
|
||||
|
||||
async def record_sleep(delay: float) -> None:
|
||||
sleeps.append(delay)
|
||||
|
||||
monkeypatch.setattr(app_module, "plugin_dispatcher", _StateDispatcher())
|
||||
monkeypatch.setattr(app_module.asyncio, "sleep", record_sleep)
|
||||
|
||||
asyncio.run(app_module.device_plugin_events(websocket, "test.plugin"))
|
||||
|
||||
assert websocket.accepted is True
|
||||
assert sleeps == [app_module.DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS]
|
||||
assert sleeps[0] == 2.0
|
||||
|
||||
|
||||
def test_device_plugin_events_propagates_arbitrary_send_runtime_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -5662,7 +5662,7 @@ def test_read_only_verify_scan_is_rejected_after_atomic_fence_before_admission(
|
||||
original_retire = service._retire_application_control_for_network_change # noqa: SLF001
|
||||
original_apply = service._apply_read_only_device_topology # noqa: SLF001
|
||||
|
||||
def retire_with_competing_scan() -> None:
|
||||
def retire_with_competing_scan(*, allow_terminal_failure: bool = False) -> None:
|
||||
assert service._provisioning_active is True # noqa: SLF001
|
||||
scan_outcomes.append(
|
||||
_attempt_competing_scan_from_sync_boundary(
|
||||
@@ -5670,7 +5670,7 @@ def test_read_only_verify_scan_is_rejected_after_atomic_fence_before_admission(
|
||||
operation_id="op-00000000-0000-4000-8000-000000000102",
|
||||
)
|
||||
)
|
||||
original_retire()
|
||||
original_retire(allow_terminal_failure=allow_terminal_failure)
|
||||
|
||||
def apply_with_fence_assertion(**kwargs: Any) -> str:
|
||||
nonlocal admission_calls
|
||||
@@ -7692,6 +7692,58 @@ def test_terminal_prepared_start_without_publish_settles_and_stops_local_capture
|
||||
assert service._acquisition_session_lease is None # noqa: SLF001
|
||||
|
||||
|
||||
def test_live_prepared_start_reduction_does_not_touch_publish_fence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
acquisition_id = "acq-live-prepared"
|
||||
operation_id = "op-live-prepared"
|
||||
monkeypatch.setattr(
|
||||
service._physical_command_coordinator, # noqa: SLF001
|
||||
"snapshot",
|
||||
lambda: {
|
||||
"status": "unresolved",
|
||||
"record": {
|
||||
"operation_id": operation_id,
|
||||
"acquisition_id": acquisition_id,
|
||||
"action": "start",
|
||||
"stage": "prepared",
|
||||
"resolution": None,
|
||||
"publish_call_returned": None,
|
||||
"packet_id": None,
|
||||
"qos2_completed": False,
|
||||
"application_response": None,
|
||||
},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service._application_control_session, # noqa: SLF001
|
||||
"snapshot",
|
||||
lambda: {"state": "initializing", "failure": None},
|
||||
)
|
||||
|
||||
class PublishFenceMustStayFree:
|
||||
def acquire(self, *args: object, **kwargs: object) -> bool:
|
||||
raise AssertionError("live START reduction touched the publish fence")
|
||||
|
||||
def release(self) -> None:
|
||||
raise AssertionError("live START reduction released an unowned fence")
|
||||
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_k1_command_dispatch_gate",
|
||||
PublishFenceMustStayFree(),
|
||||
)
|
||||
|
||||
outcome = service._settle_prepared_start_worker_failure( # noqa: SLF001
|
||||
acquisition_id=acquisition_id,
|
||||
start_operation_id=operation_id,
|
||||
)
|
||||
|
||||
assert outcome == "not-applicable"
|
||||
|
||||
|
||||
def _activate_real_checkpoint_for_prepared_stop_fixture(
|
||||
service: XgridsK1CompatibilityService,
|
||||
*,
|
||||
@@ -12273,6 +12325,7 @@ def _stop_response_without_terminal_status_fixture(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
qos2_completed: bool = True,
|
||||
application_response: bool = True,
|
||||
) -> SimpleNamespace:
|
||||
clock_value = [datetime(2026, 8, 10, 8, 10, tzinfo=UTC)]
|
||||
service, runtime = service_with_fake_runtime(tmp_path)
|
||||
@@ -12424,20 +12477,21 @@ def _stop_response_without_terminal_status_fixture(
|
||||
packet_id=82,
|
||||
)
|
||||
ledger.mark_qos2_completed(stop_operation_id, packet_id=82)
|
||||
ledger.record_application_response(
|
||||
stop_operation_id,
|
||||
PhysicalCommandApplicationResponse(
|
||||
operation_id=stop_operation_id,
|
||||
action="stop",
|
||||
control_session_id=physical_connection.control_session_id,
|
||||
host_path_epoch=physical_connection.host_path_epoch,
|
||||
producer_generation=physical_connection.producer_generation,
|
||||
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
|
||||
success=True,
|
||||
payload_sha256="a" * 64,
|
||||
observed_at_utc="2026-08-10T08:10:02.000Z",
|
||||
),
|
||||
)
|
||||
if application_response:
|
||||
ledger.record_application_response(
|
||||
stop_operation_id,
|
||||
PhysicalCommandApplicationResponse(
|
||||
operation_id=stop_operation_id,
|
||||
action="stop",
|
||||
control_session_id=physical_connection.control_session_id,
|
||||
host_path_epoch=physical_connection.host_path_epoch,
|
||||
producer_generation=physical_connection.producer_generation,
|
||||
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
|
||||
success=True,
|
||||
payload_sha256="a" * 64,
|
||||
observed_at_utc="2026-08-10T08:10:02.000Z",
|
||||
),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
service=service,
|
||||
runtime=runtime,
|
||||
@@ -12450,6 +12504,60 @@ def _stop_response_without_terminal_status_fixture(
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_stop_retires_only_terminal_control_and_keeps_live_capture(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fixture = _stop_response_without_terminal_status_fixture(
|
||||
tmp_path,
|
||||
qos2_completed=False,
|
||||
application_response=False,
|
||||
)
|
||||
service = fixture.service
|
||||
control = fixture.control
|
||||
close_calls = 0
|
||||
worker_retired = False
|
||||
original_snapshot = control.snapshot
|
||||
original_close = control.close
|
||||
|
||||
def terminal_snapshot() -> dict[str, object]:
|
||||
snapshot = original_snapshot()
|
||||
if control.state == "failed" and not worker_retired:
|
||||
snapshot["can_open"] = False
|
||||
return snapshot
|
||||
|
||||
def retire_failed_worker() -> None:
|
||||
nonlocal close_calls, worker_retired
|
||||
close_calls += 1
|
||||
worker_retired = True
|
||||
original_close()
|
||||
|
||||
monkeypatch.setattr(control, "snapshot", terminal_snapshot)
|
||||
monkeypatch.setattr(control, "close", retire_failed_worker)
|
||||
control.state = "failed"
|
||||
control.outcome_unknown = True
|
||||
control.failure = {
|
||||
"code": "ApplicationCommandOutcomeUnknown",
|
||||
"reason_code": "mqtt_response_timeout",
|
||||
"stop_command_attempted": True,
|
||||
"stop_publish_attempts": 1,
|
||||
"safe_to_retry": False,
|
||||
}
|
||||
service._acquire_application_control_process_lease() # noqa: SLF001
|
||||
|
||||
recovered = service.state()
|
||||
|
||||
assert close_calls == 1
|
||||
assert recovered["application_control_session"]["state"] == "idle"
|
||||
assert recovered["acquisition"]["state"] == "awaiting_external_stop"
|
||||
assert recovered["source_mode"] == "live"
|
||||
assert recovered["physical_command"]["status"] == "unresolved"
|
||||
assert recovered["physical_command"]["requires_reconciliation"] is True
|
||||
assert fixture.runtime.stop_calls == 0
|
||||
assert control.stop_calls == 1
|
||||
assert service._application_control_process_lease_holders == set() # noqa: SLF001
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qos2_completed", [True, False])
|
||||
def test_stop_success_without_terminal_status_times_out_into_local_only_recovery(
|
||||
tmp_path: Path,
|
||||
@@ -13729,7 +13837,18 @@ def test_confirmed_scanning_waits_for_first_authoritative_pcl_before_camera(
|
||||
binding=binding,
|
||||
resolved=True,
|
||||
)
|
||||
monkeypatch.setattr(service._physical_command_coordinator, "snapshot", lambda: physical) # noqa: SLF001
|
||||
physical_snapshot_calls = 0
|
||||
|
||||
def snapshot_physical() -> dict[str, object]:
|
||||
nonlocal physical_snapshot_calls
|
||||
physical_snapshot_calls += 1
|
||||
return physical
|
||||
|
||||
monkeypatch.setattr(
|
||||
service._physical_command_coordinator, # noqa: SLF001
|
||||
"snapshot",
|
||||
snapshot_physical,
|
||||
)
|
||||
runtime.pcl_frames = 1
|
||||
frame = DecodedPointCloudView(
|
||||
context=ConsumerFrameContext(
|
||||
@@ -13818,9 +13937,11 @@ def test_confirmed_scanning_waits_for_first_authoritative_pcl_before_camera(
|
||||
)
|
||||
|
||||
# Every later authoritative PCL for the same lineage is idempotent.
|
||||
physical_snapshot_calls_before = physical_snapshot_calls
|
||||
service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001
|
||||
time.sleep(0.05)
|
||||
assert len(events) == 2
|
||||
assert physical_snapshot_calls == physical_snapshot_calls_before
|
||||
|
||||
|
||||
def test_stale_post_publish_pcl_cannot_activate_camera(
|
||||
@@ -15316,6 +15437,85 @@ def test_active_stream_recovery_scanning_resumes_same_physical_lineage_without_c
|
||||
)
|
||||
|
||||
|
||||
def test_active_stream_recovery_privacy_bridge_retains_exact_route_for_read_only_rebind(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
service, runtime = service_with_fake_runtime(tmp_path)
|
||||
control, physical = _install_composite_active_recovery_fixture(
|
||||
service,
|
||||
runtime,
|
||||
monkeypatch,
|
||||
)
|
||||
service.state()
|
||||
lineage = service._active_stream_recovery_lineage # noqa: SLF001
|
||||
assert lineage is not None
|
||||
supervisor = service._connection_supervisor # noqa: SLF001
|
||||
baseline = supervisor.snapshot()
|
||||
candidate = replace(
|
||||
baseline,
|
||||
device_identity=replace(baseline.device_identity, state="unverified"),
|
||||
control_plane=replace(
|
||||
baseline.control_plane,
|
||||
state="lost",
|
||||
session_id=None,
|
||||
),
|
||||
lease=replace(baseline.lease, state="configured-unverified"),
|
||||
authority=replace(
|
||||
baseline.authority,
|
||||
control_allowed=False,
|
||||
acquisition_start_allowed=False,
|
||||
),
|
||||
)
|
||||
start_projects_before = list(control.start_projects)
|
||||
stop_calls_before = control.stop_calls
|
||||
operation_journal_before = service._operations.snapshot() # noqa: SLF001
|
||||
physical_ledger_before = service._physical_command_ledger.snapshot() # noqa: SLF001
|
||||
network_ledger_before = service._network_mutation_ledger.snapshot() # noqa: SLF001
|
||||
monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path)
|
||||
monkeypatch.setattr(
|
||||
service._host_wifi_association_probe, # noqa: SLF001
|
||||
"observe",
|
||||
lambda *_args, **_kwargs: {
|
||||
"schema_version": 1,
|
||||
"adapter": "CoreWLAN",
|
||||
"wifi_interface": True,
|
||||
"association_state": "unavailable",
|
||||
"evidence_quality": "unavailable",
|
||||
"continuity_proven": False,
|
||||
"continuity_token": "d" * 64,
|
||||
"reason_code": "association-identity-unavailable",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
supervisor,
|
||||
"association_timeout_retention_candidate",
|
||||
lambda *, expected_target: (
|
||||
candidate
|
||||
if expected_target
|
||||
== EndpointTarget(lineage.target_ipv4, lineage.target_port)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
sampled = service._sample_host_path(lineage.target_ipv4) # noqa: SLF001
|
||||
|
||||
assert sampled.available is True
|
||||
assert sampled.reason_code is None
|
||||
assert sampled.fingerprint == baseline.host_path.fingerprint
|
||||
assert sampled.kernel_route_fingerprint == (
|
||||
baseline.host_path.kernel_route_fingerprint
|
||||
)
|
||||
assert control.start_projects == start_projects_before
|
||||
assert control.stop_calls == stop_calls_before
|
||||
assert service._operations.snapshot() == operation_journal_before # noqa: SLF001
|
||||
assert ( # noqa: SLF001
|
||||
service._physical_command_ledger.snapshot() == physical_ledger_before
|
||||
)
|
||||
assert service._network_mutation_ledger.snapshot() == network_ledger_before # noqa: SLF001
|
||||
assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001
|
||||
|
||||
|
||||
def test_active_stream_control_adoption_gets_fresh_budget_after_slow_proof(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -15638,7 +15838,7 @@ def test_active_stream_recovery_wrong_identity_blocks_without_retry_or_camera_re
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_probe_control_endpoint",
|
||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
path=_direct_host_path(lineage.target_ipv4),
|
||||
reachable=True,
|
||||
reason_code=None,
|
||||
@@ -15699,8 +15899,14 @@ def test_active_stream_recovery_owned_path_is_read_only_and_resumes_exact_lineag
|
||||
events.append(("monitor", True))
|
||||
return True
|
||||
|
||||
def probe_exact_target(target_ipv4: str) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001
|
||||
events.append(("probe", target_ipv4))
|
||||
def probe_exact_target( # noqa: SLF001
|
||||
target_ipv4: str,
|
||||
*,
|
||||
association_timeout_seconds: float,
|
||||
) -> facade_module._CorrelatedEndpointObservation:
|
||||
events.append(
|
||||
("probe", (target_ipv4, association_timeout_seconds))
|
||||
)
|
||||
return facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
path=_direct_host_path(target_ipv4),
|
||||
reachable=True,
|
||||
@@ -15770,7 +15976,13 @@ def test_active_stream_recovery_owned_path_is_read_only_and_resumes_exact_lineag
|
||||
assert events == [
|
||||
("monitor", True),
|
||||
("lease-acquire", "network"),
|
||||
("probe", lineage.target_ipv4),
|
||||
(
|
||||
"probe",
|
||||
(
|
||||
lineage.target_ipv4,
|
||||
facade_module.ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS,
|
||||
),
|
||||
),
|
||||
(
|
||||
"device-info-status",
|
||||
(
|
||||
@@ -15910,6 +16122,7 @@ def test_incident_recovery_retires_epoch_one_control_and_retries_fresh_inspectio
|
||||
|
||||
def probe_current_epoch(
|
||||
target_ipv4: str,
|
||||
**_kwargs: object,
|
||||
) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001
|
||||
path = next(probe_paths)
|
||||
service._observe_connection_transport( # noqa: SLF001
|
||||
@@ -16065,7 +16278,7 @@ def test_active_stream_recovery_waits_for_terminal_control_worker_retirement(
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_probe_control_endpoint",
|
||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
path=_direct_host_path(lineage.target_ipv4),
|
||||
reachable=True,
|
||||
reason_code=None,
|
||||
@@ -16140,7 +16353,7 @@ def test_active_stream_recovery_exact_device_system_fault_is_terminal_without_co
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_probe_control_endpoint",
|
||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
path=_direct_host_path(lineage.target_ipv4),
|
||||
reachable=True,
|
||||
reason_code=None,
|
||||
@@ -16223,7 +16436,7 @@ def test_active_stream_recovery_bootstrap_system_error_is_normalized_terminal_fa
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_probe_control_endpoint",
|
||||
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
path=_direct_host_path(lineage.target_ipv4),
|
||||
reachable=True,
|
||||
reason_code=None,
|
||||
@@ -16476,6 +16689,111 @@ def test_active_stream_recovery_never_retries_invalid_fmp4_after_media_commit(
|
||||
assert control.stop_calls == 0
|
||||
|
||||
|
||||
def test_bound_scan_over_without_stop_dominates_late_empty_camera_epoch_failure(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
service, runtime = service_with_fake_runtime(tmp_path)
|
||||
control, physical = _install_composite_active_recovery_fixture(
|
||||
service,
|
||||
runtime,
|
||||
monkeypatch,
|
||||
runtime_phase="live",
|
||||
camera_phase="streaming",
|
||||
)
|
||||
camera = service.camera_preview.snapshot()
|
||||
recording = camera["recording"]
|
||||
assert isinstance(recording, dict)
|
||||
recording.update(
|
||||
{
|
||||
"active": True,
|
||||
"active_epoch": 2,
|
||||
"committed_media_segment_count": 4_833,
|
||||
"producer_alive": False,
|
||||
"completed_epochs": 2,
|
||||
"last_summary": {
|
||||
"codec_epoch": 2,
|
||||
"status": "failed",
|
||||
"media_segment_count": 0,
|
||||
"failure_code": "invalid-fmp4",
|
||||
},
|
||||
}
|
||||
)
|
||||
camera.update(
|
||||
{
|
||||
"phase": "error",
|
||||
"error": {
|
||||
"code": "invalid-fmp4",
|
||||
"message": "synthetic empty post-power-loss epoch",
|
||||
},
|
||||
}
|
||||
)
|
||||
physical.update(
|
||||
{
|
||||
"runtime_bound": True,
|
||||
"reconciliation_ready": True,
|
||||
"observed_session_state": "scan_over",
|
||||
}
|
||||
)
|
||||
control.state = "failed"
|
||||
control.state_revision += 1
|
||||
control.failure = {
|
||||
"reason_code": "application_acceptance_failed",
|
||||
"failed_phase": "scanning",
|
||||
"modeling_command_attempted": True,
|
||||
"stop_command_attempted": False,
|
||||
"diagnostic_snapshot_unavailable": [],
|
||||
"diagnostic_evidence_unavailable": [],
|
||||
"safe_to_retry": False,
|
||||
}
|
||||
|
||||
def scan_over_control_snapshot() -> dict[str, object]:
|
||||
snapshot = FakeInteractiveControlSession.snapshot(control)
|
||||
snapshot["transport"] = {
|
||||
"state": "failed",
|
||||
"publish_attempts": 7,
|
||||
"device_status_reports": 9,
|
||||
"latest_device_session_state": "scan_over",
|
||||
"latest_device_project_bound": True,
|
||||
"latest_device_init_ready": False,
|
||||
"latest_system_error_code": None,
|
||||
"automatic_retry": False,
|
||||
"automatic_reconnect": False,
|
||||
}
|
||||
return snapshot
|
||||
|
||||
monkeypatch.setattr(control, "snapshot", scan_over_control_snapshot)
|
||||
monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None)
|
||||
camera_stop_calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
service.camera_preview,
|
||||
"stop_current",
|
||||
lambda: camera_stop_calls.append("stop") or {},
|
||||
)
|
||||
start_projects_before = list(control.start_projects)
|
||||
stop_calls_before = control.stop_calls
|
||||
|
||||
terminal = service.state()
|
||||
|
||||
assert terminal["acquisition"]["state"] == "interrupted"
|
||||
assert terminal["acquisition"]["message_code"] == (
|
||||
"acquisition.recovery.device_standby_observed"
|
||||
)
|
||||
assert terminal["acquisition"]["result"] == {
|
||||
"receiver_stopped": True,
|
||||
"device_state": "scan_over",
|
||||
"device_stop": "not-sent",
|
||||
"automatic_command_retry": False,
|
||||
"read_only_recovery": True,
|
||||
"physical_reconciliation_required": True,
|
||||
}
|
||||
assert terminal["acquisition"]["cleanup_pending"] is False
|
||||
assert camera_stop_calls == ["stop"]
|
||||
assert runtime.stop_calls == 1
|
||||
assert control.start_projects == start_projects_before
|
||||
assert control.stop_calls == stop_calls_before == 0
|
||||
|
||||
|
||||
def test_active_stream_recovery_reopens_exact_camera_epoch_once_and_fences_stale_lineage(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -24955,6 +25273,59 @@ def test_configured_unverified_monitor_keeps_same_route_without_promoting_author
|
||||
assert "verify-control-device-info" in retained.allowed_actions
|
||||
|
||||
|
||||
def test_monitor_layer_retains_exact_configured_route_across_association_lock_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
binding = _seed_supervised_connection(service, with_control=False)
|
||||
baseline = service._connection_supervisor.snapshot() # noqa: SLF001
|
||||
assert baseline.host_path.kernel_route_fingerprint is not None
|
||||
tcp_calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_sample_host_path",
|
||||
lambda target, **_kwargs: HostPathProbeResult(
|
||||
available=False,
|
||||
fingerprint=None,
|
||||
interface=baseline.host_path.interface,
|
||||
source_ipv4=baseline.host_path.source_ipv4,
|
||||
route_class="unavailable",
|
||||
reason_code="host-wifi-operation-timeout",
|
||||
observation_failure_class="association-observer",
|
||||
kernel_route_fingerprint=baseline.host_path.kernel_route_fingerprint,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
facade_module,
|
||||
"_probe_control_endpoint_socket",
|
||||
lambda target: (
|
||||
tcp_calls.append(target)
|
||||
or facade_module.TcpReachabilityProbeResult(reachable=True)
|
||||
),
|
||||
)
|
||||
|
||||
async def poll_three_times() -> list[facade_module.ConnectionSupervisorSnapshot]:
|
||||
return [
|
||||
await service._connection_monitor.poll_once() # noqa: SLF001
|
||||
for _ in range(3)
|
||||
]
|
||||
|
||||
snapshots = asyncio.run(poll_three_times())
|
||||
|
||||
assert tcp_calls == [binding.target_ipv4] * 3
|
||||
for retained in snapshots:
|
||||
assert retained.host_path.available is True
|
||||
assert retained.host_path.epoch == baseline.host_path.epoch
|
||||
assert retained.host_path.fingerprint == baseline.host_path.fingerprint
|
||||
assert retained.endpoint.tcp_state == "reachable"
|
||||
assert retained.device_identity.state == "unverified"
|
||||
assert retained.authority.control_allowed is False
|
||||
assert retained.authority.acquisition_start_allowed is False
|
||||
assert "verify-control-device-info" in retained.allowed_actions
|
||||
|
||||
|
||||
def test_association_timeout_cannot_hide_a_real_kernel_route_change(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
@@ -29825,6 +30196,91 @@ def test_explicit_verify_reconciles_persisted_start_to_ready_without_device_io(
|
||||
assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001
|
||||
|
||||
|
||||
def test_long_first_verify_refreshes_aged_transport_before_exposing_prestart_control(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""A long checkpoint reconciliation cannot orphan its new ready socket."""
|
||||
|
||||
service, _ = service_with_fake_runtime(tmp_path)
|
||||
coordinator = _VerifyPhysicalRecoveryCoordinator(
|
||||
observed_session_state="ready",
|
||||
reconciliation_ready=True,
|
||||
)
|
||||
_install_synthetic_verify_recovery(service, coordinator)
|
||||
supervisor = service._connection_supervisor # noqa: SLF001
|
||||
monotonic_now = [100.0]
|
||||
suspend_aware_now = [1_000.0]
|
||||
supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001
|
||||
supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001
|
||||
before = supervisor.snapshot()
|
||||
assert before.intent is not None
|
||||
assert before.device_network.target is not None
|
||||
host_epoch = supervisor.observe_host_path(
|
||||
_association_bound_direct_host_path(before.device_network.target.ipv4)
|
||||
)
|
||||
assert supervisor.observe_endpoint(
|
||||
target=before.device_network.target,
|
||||
intent_id=before.intent.intent_id,
|
||||
host_path_epoch=host_epoch,
|
||||
reachable=True,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path)
|
||||
tcp_probes: list[str] = []
|
||||
|
||||
def reachable(target: str) -> bool:
|
||||
tcp_probes.append(target)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", reachable)
|
||||
reconcile = service._reconcile_physical_command_after_verify_owned # noqa: SLF001
|
||||
|
||||
async def reconcile_after_transport_ages(
|
||||
bound_service: XgridsK1CompatibilityService,
|
||||
*,
|
||||
verify_operation_id: str,
|
||||
allow_receiver_rehydrate: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
result = await reconcile(
|
||||
verify_operation_id=verify_operation_id,
|
||||
allow_receiver_rehydrate=allow_receiver_rehydrate,
|
||||
)
|
||||
monotonic_now[0] += 20.0
|
||||
suspend_aware_now[0] += 20.0
|
||||
return result
|
||||
|
||||
service._reconcile_physical_command_after_verify_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001
|
||||
reconcile_after_transport_ages,
|
||||
service,
|
||||
)
|
||||
operation_id = "op-00000000-0000-4000-8000-000000001202"
|
||||
|
||||
verified = asyncio.run(
|
||||
service.verify_connection(
|
||||
_retained_physical_recovery_verify_request(operation_id=operation_id)
|
||||
)
|
||||
)
|
||||
|
||||
assert tcp_probes == ["192.168.1.20"]
|
||||
assert verified["last_operation"]["status"] == "succeeded"
|
||||
assert verified["active_connection_mode"] == "bridge"
|
||||
assert verified["connection_policy"]["actions"]["start-acquisition"]["allowed"] is True
|
||||
assert service._application_control_session.snapshot()["state"] == "connection-ready" # noqa: SLF001
|
||||
assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001
|
||||
fresh = supervisor.snapshot()
|
||||
assert fresh.intent is not None
|
||||
assert fresh.device_network.target is not None
|
||||
assert supervisor.endpoint_observation_has_remaining_lease(
|
||||
target=fresh.device_network.target,
|
||||
intent_id=fresh.intent.intent_id,
|
||||
host_path_epoch=fresh.host_path.epoch,
|
||||
minimum_remaining_seconds=(
|
||||
facade_module.VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_verify_reconciliation_does_not_require_normal_command_authority(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
@@ -1843,6 +1843,25 @@ def test_chained_stop_facade_cessation_uses_real_ledger_ancestry(
|
||||
stale_reconciliation,
|
||||
final_record,
|
||||
)
|
||||
# Reproduce the append-only field shape: an otherwise valid historical
|
||||
# standby reconciliation is inherited by the newer ordinary STOP. The
|
||||
# checkpoint reducer must use the current STOP's own READY status, not the
|
||||
# last historical reconciliation merely because it is terminal standby.
|
||||
snapshot_with_stale_standby = replace(
|
||||
ledger.snapshot(),
|
||||
record=replace(
|
||||
final_record,
|
||||
reconciliations=(
|
||||
*final_record.reconciliations,
|
||||
stale_reconciliation,
|
||||
),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ledger,
|
||||
"snapshot",
|
||||
lambda: snapshot_with_stale_standby,
|
||||
)
|
||||
|
||||
service._application_control_session.snapshot = lambda: { # type: ignore[method-assign]
|
||||
"verified_control": _verified_control(first_rebind)
|
||||
@@ -1862,6 +1881,88 @@ def test_chained_stop_facade_cessation_uses_real_ledger_ancestry(
|
||||
assert checkpoint.cessation_physical_proof.ancestor_chain == ancestry
|
||||
|
||||
|
||||
def test_restart_ceases_active_checkpoint_after_composite_ordinary_stop(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
service = _service(tmp_path, monkeypatch)
|
||||
connection = _connection(
|
||||
control_session_id="control-before-ordinary-stop-restart",
|
||||
host_path_epoch=1,
|
||||
producer_generation=1,
|
||||
)
|
||||
_prepare_and_activate_checkpoint(service, connection)
|
||||
ledger = service._physical_command_ledger # noqa: SLF001
|
||||
start = ledger.snapshot().record
|
||||
assert start is not None and start.operation_id == START_OPERATION_ID
|
||||
identity = PhysicalCommandIdentity(
|
||||
vendor_device_id_sha256=VENDOR_SHA256,
|
||||
device_serial_sha256=SERIAL_SHA256,
|
||||
)
|
||||
stop_operation_id = "physical-stop-ordinary-restart"
|
||||
ledger.prepare(
|
||||
operation_id=stop_operation_id,
|
||||
parent_operation_id=START_OPERATION_ID,
|
||||
acquisition_id=ACQUISITION_ID,
|
||||
action="stop",
|
||||
identity=identity,
|
||||
connection=connection,
|
||||
compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID,
|
||||
payload_sha256="6" * 64,
|
||||
baseline_status=_status(
|
||||
connection,
|
||||
"scanning",
|
||||
observed_at_utc="2026-08-13T12:02:00.000Z",
|
||||
),
|
||||
)
|
||||
ledger.mark_dispatching(stop_operation_id)
|
||||
ledger.mark_observing(
|
||||
stop_operation_id,
|
||||
publish_call_returned=True,
|
||||
packet_id=43,
|
||||
)
|
||||
ledger.mark_qos2_completed(stop_operation_id, packet_id=43)
|
||||
ledger.record_application_response(
|
||||
stop_operation_id,
|
||||
PhysicalCommandApplicationResponse(
|
||||
operation_id=stop_operation_id,
|
||||
action="stop",
|
||||
control_session_id=connection.control_session_id,
|
||||
host_path_epoch=connection.host_path_epoch,
|
||||
producer_generation=connection.producer_generation,
|
||||
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
|
||||
success=True,
|
||||
payload_sha256="7" * 64,
|
||||
observed_at_utc="2026-08-13T12:02:01.000Z",
|
||||
),
|
||||
)
|
||||
ledger.record_status_observation(
|
||||
stop_operation_id,
|
||||
_status(
|
||||
connection,
|
||||
"ready",
|
||||
observed_at_utc="2026-08-13T12:02:02.000Z",
|
||||
),
|
||||
)
|
||||
stop = ledger.resolve(
|
||||
stop_operation_id,
|
||||
resolution="stop-standby-observed",
|
||||
)
|
||||
before_restart = ActiveAcquisitionRecoveryCheckpointStore(tmp_path).snapshot()
|
||||
assert before_restart.status == "active"
|
||||
|
||||
restarted = XgridsK1CompatibilityService(tmp_path)
|
||||
|
||||
after_restart = ActiveAcquisitionRecoveryCheckpointStore(tmp_path).snapshot()
|
||||
assert after_restart.status == "ceased"
|
||||
assert after_restart.checkpoint is not None
|
||||
assert after_restart.checkpoint.cessation_physical_proof is not None
|
||||
assert after_restart.checkpoint.cessation_physical_proof.operation_id == (
|
||||
stop.operation_id
|
||||
)
|
||||
assert restarted._active_acquisition_checkpoint_trust == "trusted" # noqa: SLF001
|
||||
|
||||
|
||||
def test_restart_ready_classification_ceases_active_checkpoint_after_undispatched_stop(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
@@ -887,7 +887,7 @@ def test_ambiguous_prepared_restart_ready_settles_after_dhcp_target_change(
|
||||
mqtt_retained=False,
|
||||
observed_at_utc="2026-08-13T12:20:01.000Z",
|
||||
)
|
||||
record = restarted._physical_command_ledger.reconcile_ambiguous( # noqa: SLF001
|
||||
restarted._physical_command_ledger.reconcile_ambiguous( # noqa: SLF001
|
||||
restart_support.START_OPERATION_ID,
|
||||
reconciliation_id="reconciliation-ready-new-dhcp-address",
|
||||
resolution="physical-standby-observed",
|
||||
@@ -1057,7 +1057,7 @@ def test_restart_receiver_link_loss_before_first_pcl_rebinds_read_only_then_prom
|
||||
monkeypatch.setattr(
|
||||
restarted,
|
||||
"_probe_control_endpoint",
|
||||
lambda target_ipv4: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
lambda target_ipv4, **_kwargs: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001
|
||||
reachable=True,
|
||||
reason_code=None,
|
||||
@@ -1237,7 +1237,7 @@ def test_ambiguous_prepared_restart_second_rebind_promotes_only_after_fresh_pcl(
|
||||
monkeypatch.setattr(
|
||||
restarted,
|
||||
"_probe_control_endpoint",
|
||||
lambda target_ipv4: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
lambda target_ipv4, **_kwargs: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001
|
||||
path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001
|
||||
reachable=True,
|
||||
reason_code=None,
|
||||
|
||||
@@ -394,7 +394,10 @@ def _wait_phase(
|
||||
session: InteractiveApplicationControlSession,
|
||||
expected: str,
|
||||
) -> dict[str, object]:
|
||||
deadline = time.monotonic() + 2.0
|
||||
# FakeExecutor may use its full two-second checkpoint timeout before the
|
||||
# worker publishes the terminal state. Keep the observer deadline strictly
|
||||
# larger so this helper does not race the transition it is asserting.
|
||||
deadline = time.monotonic() + 3.0
|
||||
while time.monotonic() < deadline:
|
||||
snapshot = session.snapshot()
|
||||
if snapshot["state"] == expected:
|
||||
|
||||
Reference in New Issue
Block a user