329 lines
14 KiB
Python
329 lines
14 KiB
Python
"""Per-camera recovery intent. No SDK ownership, recording replay or BLE access."""
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
from .http import request
|
|
from .operations import Operations, atomic
|
|
|
|
HELPER = Path("/run/mission-core-x4-recovery/driver.sock")
|
|
|
|
|
|
class Recovery:
|
|
def __init__(self, broker, root):
|
|
self.broker, self.root = broker, root
|
|
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
self.lock = threading.RLock()
|
|
self.states = {}
|
|
self.current = {}
|
|
self.absent = {}
|
|
self.stable = {}
|
|
self.jobs = {}
|
|
self.wakeable = set()
|
|
self.stopping = threading.Event()
|
|
self.pool = ThreadPoolExecutor(max_workers=2, thread_name_prefix="x4-recovery")
|
|
for path in sorted(root.glob("instax4_*.json"))[:500]:
|
|
state = json.loads(path.read_text())
|
|
self.broker.validate_identifier(path.stem)
|
|
self.states[path.stem] = state
|
|
|
|
def default(self):
|
|
now = datetime.now(UTC).isoformat()
|
|
return {
|
|
"enabled": False,
|
|
"preview_wanted": False,
|
|
"attempts": 0,
|
|
"phase": "disabled",
|
|
"session_id": "x4_recovery_" + uuid.uuid4().hex,
|
|
"opened_at": now,
|
|
"updated_at": now,
|
|
"revision": 1,
|
|
"resume_session": None,
|
|
"last_preview_request": "",
|
|
"last_preview_id": "",
|
|
}
|
|
|
|
def save(self, ident, state):
|
|
state["revision"] += 1
|
|
state["updated_at"] = datetime.now(UTC).isoformat()
|
|
atomic(self.root / (ident + ".json"), state)
|
|
|
|
def view(self, ident):
|
|
with self.lock:
|
|
state = self.states.get(ident, {})
|
|
return {
|
|
"supported": True,
|
|
"enabled": state.get("enabled", False),
|
|
"preview_wanted": state.get("preview_wanted", False),
|
|
"phase": state.get("phase", "disabled"),
|
|
"wake_available": ident in self.wakeable,
|
|
}
|
|
|
|
def sync_registry(self):
|
|
values = request(HELPER, "/registry", {}, timeout=4)["items"]
|
|
if not isinstance(values, list) or len(values) > 500:
|
|
raise ValueError("Invalid wake registry")
|
|
for ident in values:
|
|
self.broker.validate_identifier(ident)
|
|
with self.lock:
|
|
self.wakeable = set(values)
|
|
for ident in values:
|
|
if ident not in self.states:
|
|
self.states[ident] = self.default()
|
|
self.save(ident, self.states[ident])
|
|
|
|
def wake_once(self, ident, command):
|
|
# Explicit operator action. Keep the automatic policy and preview intent intact.
|
|
deadline = time.monotonic() + 52
|
|
result = request(
|
|
HELPER,
|
|
"/wake-once",
|
|
{"device_id": ident, "attempt": command["operation_id"][3:]},
|
|
timeout=52,
|
|
)
|
|
state = result.get("state")
|
|
# A cold X4 can enumerate after the 20-second beacon has already ended.
|
|
# Observe the installed supervisor only; never send a second wake beacon.
|
|
if state == "unavailable":
|
|
while time.monotonic() < deadline and not self.stopping.is_set():
|
|
if any(x["id"] == ident and x.get("online") for x in self.broker.snapshots()):
|
|
state = "connected"
|
|
break
|
|
self.stopping.wait(1)
|
|
if state == "connected":
|
|
return {"state": "complete", "result": {"usb_connected": True}}
|
|
errors = {
|
|
"busy": "wake_adapter_busy",
|
|
"disabled": "wake_identity_missing",
|
|
"identity_conflict": "wake_identity_conflict",
|
|
"exhausted": "wake_rate_limited",
|
|
}
|
|
return {"state": "error", "error": errors.get(state, "wake_not_confirmed")}
|
|
|
|
def snapshot(self, ident):
|
|
with self.lock:
|
|
current = self.current.get(ident)
|
|
if current is not None:
|
|
return copy.deepcopy(current)
|
|
state = self.states.get(ident)
|
|
if state is None:
|
|
raise ValueError("Camera identity is unavailable")
|
|
return {
|
|
"id": ident,
|
|
"session_id": state["session_id"],
|
|
"opened_at": state["opened_at"],
|
|
"observed_at": state["updated_at"],
|
|
"revision": state["revision"],
|
|
"prepared": True,
|
|
"verified": False,
|
|
"online": False,
|
|
"preparation_safe": False,
|
|
"status": {},
|
|
"message": None,
|
|
}
|
|
|
|
def augment(self, snapshots):
|
|
values = {x["id"]: copy.deepcopy(x) for x in snapshots}
|
|
with self.lock:
|
|
for ident in self.states:
|
|
if ident not in values:
|
|
values[ident] = self.snapshot(ident)
|
|
# Never retain an online snapshot when its worker disappeared.
|
|
values[ident].update(online=False, status={}, preparation_safe=False)
|
|
for ident, value in values.items():
|
|
value["status"]["recovery"] = self.view(ident)
|
|
return list(values.values())
|
|
|
|
def configure(self, ident, enabled):
|
|
# Called under the camera capture lock, also used by preview restoration.
|
|
reply = request(HELPER, "/control", {"device_id": ident, "enabled": enabled}, timeout=8)
|
|
if reply.get("enabled") is not enabled:
|
|
raise RuntimeError("Recovery setting was not confirmed")
|
|
with self.lock:
|
|
state = self.states.setdefault(ident, self.default())
|
|
state.update(enabled=enabled, attempts=0, phase="ready" if enabled else "disabled")
|
|
if not enabled:
|
|
state["preview_wanted"] = False
|
|
current = self.current.get(ident)
|
|
if current:
|
|
state.update(session_id=current["session_id"], opened_at=current["opened_at"])
|
|
if enabled:
|
|
state["preview_wanted"] = current.get("status", {}).get("preview") == 1
|
|
self.save(ident, state)
|
|
self.absent.pop(ident, None)
|
|
return {"state": "complete", "result": self.view(ident)}
|
|
|
|
def operation(self, ident, command):
|
|
with self.broker.capture_lock(ident):
|
|
snapshot = self.snapshot(ident)
|
|
operations = Operations(
|
|
ident,
|
|
snapshot["session_id"],
|
|
self.root / ident / "operations",
|
|
RecoveryCalls(self, ident, command),
|
|
)
|
|
return operations.execute(command)
|
|
|
|
def preview_intent(self, ident, command, result):
|
|
start = command["action_id"] == "preview.start"
|
|
if start and result.get("state") != "complete":
|
|
return
|
|
with self.lock:
|
|
state = self.states.setdefault(ident, self.default())
|
|
requested = datetime.fromisoformat(command["requested_at"])
|
|
previous = state["last_preview_request"]
|
|
if command["operation_id"] == state["last_preview_id"] or (
|
|
previous and requested <= datetime.fromisoformat(previous)
|
|
):
|
|
return
|
|
state.update(
|
|
preview_wanted=start,
|
|
last_preview_request=command["requested_at"],
|
|
last_preview_id=command["operation_id"],
|
|
)
|
|
self.save(ident, state)
|
|
|
|
def observe(self, snapshots, now):
|
|
"""Pure scheduling decisions under one short lock; no device calls here."""
|
|
self.current = {x["id"]: copy.deepcopy(x) for x in snapshots}
|
|
decisions = []
|
|
for ident, state in self.states.items():
|
|
current = self.current.get(ident)
|
|
online = bool(current and current.get("online"))
|
|
if online:
|
|
self.absent.pop(ident, None)
|
|
self.stable.setdefault(ident, now)
|
|
if state["session_id"] != current["session_id"]:
|
|
state.update(session_id=current["session_id"], opened_at=current["opened_at"])
|
|
self.save(ident, state)
|
|
if now - self.stable[ident] >= 60 and state["attempts"]:
|
|
state["attempts"] = 0
|
|
self.save(ident, state)
|
|
if (
|
|
state["enabled"]
|
|
and state["preview_wanted"]
|
|
and state["resume_session"] != current["session_id"]
|
|
and current.get("status", {}).get("preview") == 0
|
|
and current.get("status", {}).get("recording") == 0
|
|
):
|
|
decisions.append((ident, "preview"))
|
|
else:
|
|
self.stable.pop(ident, None)
|
|
since = self.absent.setdefault(ident, now)
|
|
if (
|
|
state["enabled"]
|
|
and state["attempts"] < 3
|
|
and now - since >= (8, 40, 100)[state["attempts"]]
|
|
):
|
|
decisions.append((ident, "wake"))
|
|
return decisions
|
|
|
|
def work(self, ident, kind):
|
|
if kind == "preview":
|
|
# Serialize restore with user START/STOP/configure for this camera only.
|
|
with self.broker.capture_lock(ident):
|
|
with self.lock:
|
|
state = self.states[ident]
|
|
current = self.current.get(ident)
|
|
if not state["enabled"] or not state["preview_wanted"] or not current:
|
|
return
|
|
session = current["session_id"]
|
|
if state["resume_session"] == session:
|
|
return
|
|
state.update(resume_session=session, phase="restoring_preview")
|
|
self.save(ident, state)
|
|
now = datetime.now(UTC)
|
|
operation = (
|
|
"op_" + hashlib.sha256((ident + session + "restore").encode()).hexdigest()[:32]
|
|
)
|
|
command = {
|
|
"api_version": "missioncore.nodedc/plugin-sdk/v0alpha2",
|
|
"kind": "OperationRequest",
|
|
"operation_id": operation,
|
|
"idempotency_key": operation,
|
|
"session": {"device_id": ident, "session_id": session},
|
|
"action_id": "preview.start",
|
|
"parameters": {},
|
|
"requested_at": now.isoformat(),
|
|
"deadline_at": (now + timedelta(seconds=60)).isoformat(),
|
|
}
|
|
result = self.broker.capture_operation(ident, command, restoring=True)
|
|
with self.lock:
|
|
state["phase"] = (
|
|
"ready" if result.get("state") == "complete" else "preview_failed"
|
|
)
|
|
self.save(ident, state)
|
|
else:
|
|
with self.lock:
|
|
state = self.states[ident]
|
|
if not state["enabled"] or state["attempts"] >= 3:
|
|
return
|
|
state["attempts"] += 1
|
|
state["phase"] = "waking"
|
|
self.save(ident, state)
|
|
result = request(
|
|
HELPER, "/wake", {"device_id": ident, "attempt": uuid.uuid4().hex}, timeout=52
|
|
)
|
|
with self.lock:
|
|
state = self.states[ident]
|
|
if state["enabled"]:
|
|
if result.get("state") == "busy":
|
|
state["attempts"] -= 1
|
|
state["phase"] = (
|
|
"ready"
|
|
if result.get("state") == "connected"
|
|
else "exhausted"
|
|
if state["attempts"] >= 3
|
|
else "waiting"
|
|
)
|
|
self.save(ident, state)
|
|
|
|
def loop(self):
|
|
while not self.stopping.wait(2):
|
|
try:
|
|
snapshots = self.broker.snapshots()
|
|
try:
|
|
self.sync_registry()
|
|
except (OSError, ValueError, RuntimeError, KeyError, TypeError):
|
|
with self.lock:
|
|
self.wakeable = set()
|
|
with self.lock:
|
|
for ident, future in list(self.jobs.items()):
|
|
if future.done():
|
|
if future.exception():
|
|
state = self.states[ident]
|
|
state["phase"] = "unavailable" if state["enabled"] else "disabled"
|
|
self.save(ident, state)
|
|
del self.jobs[ident]
|
|
for ident, kind in self.observe(snapshots, time.monotonic()):
|
|
if ident not in self.jobs and len(self.jobs) < 2:
|
|
self.jobs[ident] = self.pool.submit(self.work, ident, kind)
|
|
except (OSError, ValueError, RuntimeError, KeyError):
|
|
# No wake is inferred from failed model-wide inventory sampling.
|
|
continue
|
|
|
|
def start(self):
|
|
threading.Thread(target=self.loop, name="x4-recovery", daemon=True).start()
|
|
|
|
|
|
class RecoveryCalls:
|
|
def __init__(self, recovery, ident, command):
|
|
self.recovery, self.ident, self.command = recovery, ident, command
|
|
|
|
def call(self, action, params):
|
|
if action == "power.wake":
|
|
return self.recovery.wake_once(self.ident, self.command)
|
|
if action == "recovery.configure":
|
|
return self.recovery.configure(self.ident, params["enabled"])
|
|
if action == "preview.stop":
|
|
self.recovery.preview_intent(self.ident, self.command, {"state": "complete"})
|
|
return {"state": "complete", "result": {"preview_wanted": False}}
|
|
raise ValueError("Unsupported recovery action")
|