"""Per-camera recovery intent. No SDK ownership, recording replay or BLE access.""" import copy import hashlib import json import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from pathlib import Path from .http import request from .operations import Operations, atomic HELPER = Path("/run/mission-core-x4-recovery/driver.sock") class Recovery: def __init__(self, broker, root): self.broker, self.root = broker, root self.root.mkdir(mode=0o700, parents=True, exist_ok=True) self.lock = threading.RLock() self.states = {} self.current = {} self.absent = {} self.stable = {} self.jobs = {} self.stopping = threading.Event() self.pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="x4-recovery") for path in sorted(root.glob("instax4_*.json"))[:500]: state = json.loads(path.read_text()) self.broker.validate_identifier(path.stem) self.states[path.stem] = state def default(self): now = datetime.now(UTC).isoformat() return { "enabled": False, "preview_wanted": False, "attempts": 0, "phase": "disabled", "session_id": "x4_recovery_" + uuid.uuid4().hex, "opened_at": now, "updated_at": now, "revision": 1, "resume_session": None, "last_preview_request": "", "last_preview_id": "", } def save(self, ident, state): state["revision"] += 1 state["updated_at"] = datetime.now(UTC).isoformat() atomic(self.root / (ident + ".json"), state) def view(self, ident): with self.lock: state = self.states.get(ident, {}) return { "supported": True, "enabled": state.get("enabled", False), "preview_wanted": state.get("preview_wanted", False), "phase": state.get("phase", "disabled"), } def snapshot(self, ident): with self.lock: current = self.current.get(ident) if current is not None: return copy.deepcopy(current) state = self.states.get(ident) if state is None: raise ValueError("Camera identity is unavailable") return { "id": ident, "session_id": state["session_id"], "opened_at": state["opened_at"], "observed_at": state["updated_at"], "revision": state["revision"], "prepared": True, "verified": False, "online": False, "preparation_safe": False, "status": {}, "message": None, } def augment(self, snapshots): values = {x["id"]: copy.deepcopy(x) for x in snapshots} with self.lock: for ident in self.states: if ident not in values: values[ident] = self.snapshot(ident) # Never retain an online snapshot when its worker disappeared. values[ident].update(online=False, status={}, preparation_safe=False) for ident, value in values.items(): value["status"]["recovery"] = self.view(ident) return list(values.values()) def configure(self, ident, enabled): # Called under the camera capture lock, also used by preview restoration. reply = request(HELPER, "/control", {"device_id": ident, "enabled": enabled}, timeout=8) if reply.get("enabled") is not enabled: raise RuntimeError("Recovery setting was not confirmed") with self.lock: state = self.states.setdefault(ident, self.default()) state.update(enabled=enabled, attempts=0, phase="ready" if enabled else "disabled") if not enabled: state["preview_wanted"] = False current = self.current.get(ident) if current: state.update(session_id=current["session_id"], opened_at=current["opened_at"]) if enabled: state["preview_wanted"] = current.get("status", {}).get("preview") == 1 self.save(ident, state) self.absent.pop(ident, None) return {"state": "complete", "result": self.view(ident)} def operation(self, ident, command): with self.broker.capture_lock(ident): snapshot = self.snapshot(ident) operations = Operations( ident, snapshot["session_id"], self.root / ident / "operations", RecoveryCalls(self, ident, command), ) return operations.execute(command) def preview_intent(self, ident, command, result): start = command["action_id"] == "preview.start" if start and result.get("state") != "complete": return with self.lock: state = self.states.setdefault(ident, self.default()) requested = datetime.fromisoformat(command["requested_at"]) previous = state["last_preview_request"] if command["operation_id"] == state["last_preview_id"] or ( previous and requested <= datetime.fromisoformat(previous) ): return state.update( preview_wanted=start, last_preview_request=command["requested_at"], last_preview_id=command["operation_id"], ) self.save(ident, state) def observe(self, snapshots, now): """Pure scheduling decisions under one short lock; no device calls here.""" self.current = {x["id"]: copy.deepcopy(x) for x in snapshots} decisions = [] for ident, state in self.states.items(): current = self.current.get(ident) online = bool(current and current.get("online")) if online: self.absent.pop(ident, None) self.stable.setdefault(ident, now) if state["session_id"] != current["session_id"]: state.update(session_id=current["session_id"], opened_at=current["opened_at"]) self.save(ident, state) if now - self.stable[ident] >= 60 and state["attempts"]: state["attempts"] = 0 self.save(ident, state) if ( state["enabled"] and state["preview_wanted"] and state["resume_session"] != current["session_id"] and current.get("status", {}).get("preview") == 0 and current.get("status", {}).get("recording") == 0 ): decisions.append((ident, "preview")) else: self.stable.pop(ident, None) since = self.absent.setdefault(ident, now) if ( state["enabled"] and state["attempts"] < 3 and now - since >= (8, 40, 100)[state["attempts"]] ): decisions.append((ident, "wake")) return decisions def work(self, ident, kind): if kind == "preview": # Serialize restore with user START/STOP/configure for this camera only. with self.broker.capture_lock(ident): with self.lock: state = self.states[ident] current = self.current.get(ident) if not state["enabled"] or not state["preview_wanted"] or not current: return session = current["session_id"] if state["resume_session"] == session: return state.update(resume_session=session, phase="restoring_preview") self.save(ident, state) now = datetime.now(UTC) operation = ( "op_" + hashlib.sha256((ident + session + "restore").encode()).hexdigest()[:32] ) command = { "api_version": "missioncore.nodedc/plugin-sdk/v0alpha2", "kind": "OperationRequest", "operation_id": operation, "idempotency_key": operation, "session": {"device_id": ident, "session_id": session}, "action_id": "preview.start", "parameters": {}, "requested_at": now.isoformat(), "deadline_at": (now + timedelta(seconds=60)).isoformat(), } result = self.broker.capture_operation(ident, command, restoring=True) with self.lock: state["phase"] = ( "ready" if result.get("state") == "complete" else "preview_failed" ) self.save(ident, state) else: with self.lock: state = self.states[ident] if not state["enabled"] or state["attempts"] >= 3: return state["attempts"] += 1 state["phase"] = "waking" self.save(ident, state) result = request( HELPER, "/wake", {"device_id": ident, "attempt": uuid.uuid4().hex}, timeout=52 ) with self.lock: state = self.states[ident] if state["enabled"]: if result.get("state") == "busy": state["attempts"] -= 1 state["phase"] = ( "ready" if result.get("state") == "connected" else "exhausted" if state["attempts"] >= 3 else "waiting" ) self.save(ident, state) def loop(self): while not self.stopping.wait(2): try: snapshots = self.broker.snapshots() with self.lock: for ident, future in list(self.jobs.items()): if future.done(): if future.exception(): state = self.states[ident] state["phase"] = "unavailable" if state["enabled"] else "disabled" self.save(ident, state) del self.jobs[ident] for ident, kind in self.observe(snapshots, time.monotonic()): if ident not in self.jobs and len(self.jobs) < 2: self.jobs[ident] = self.pool.submit(self.work, ident, kind) except (OSError, ValueError, RuntimeError, KeyError): # No wake is inferred from failed model-wide inventory sampling. continue def start(self): threading.Thread(target=self.loop, name="x4-recovery", daemon=True).start() class RecoveryCalls: def __init__(self, recovery, ident, command): self.recovery, self.ident, self.command = recovery, ident, command def call(self, action, params): if action == "recovery.configure": return self.recovery.configure(self.ident, params["enabled"]) if action == "preview.stop": self.recovery.preview_intent(self.ident, self.command, {"state": "complete"}) return {"state": "complete", "result": {"preview_wanted": False}} raise ValueError("Unsupported recovery action")