fix(service): supervise canonical Mission Core lifecycle

src/k1link/device_plugins/xgrids_k1/cli.py

@@ -81,6 +81,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_authority import (
 )
 from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot
 from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
+from k1link.service_watchdog import MissionCoreSelfWatchdog, watchdog_enabled
 from k1link.sessions import SessionIntegrityError
 
 app = typer.Typer(
@@ -116,6 +117,7 @@ app.add_typer(artifact_app, name="artifact")
 
 _CANONICAL_MISSION_CORE_PORT = 8000
 _MISSION_CORE_SERVE_LOCK_FILENAME = ".serve.lock"
+_MISSION_CORE_GRACEFUL_SHUTDOWN_SECONDS = 10
 
 
 class _MissionCoreServeLeaseError(RuntimeError):
@@ -1166,13 +1168,23 @@ def serve_console(
             f"NODEDC MISSION CORE: http://127.0.0.1:{_CANONICAL_MISSION_CORE_PORT}"
         )
         console.print("The credential endpoint is bound to this Mac only.")
-        uvicorn.run(
-            "k1link.web.app:app",
-            host="127.0.0.1",
-            port=_CANONICAL_MISSION_CORE_PORT,
-            log_level="info",
-            access_log=True,
+        watchdog = (
+            MissionCoreSelfWatchdog(repository_root) if watchdog_enabled() else None
         )
+        if watchdog is not None:
+            watchdog.start()
+        try:
+            uvicorn.run(
+                "k1link.web.app:app",
+                host="127.0.0.1",
+                port=_CANONICAL_MISSION_CORE_PORT,
+                log_level="info",
+                access_log=True,
+                timeout_graceful_shutdown=_MISSION_CORE_GRACEFUL_SHUTDOWN_SECONDS,
+            )
+        finally:
+            if watchdog is not None:
+                watchdog.stop()
 
 
 def _print_existing_mission_core(port: int) -> None:

src/k1link/local_service_launchd.py

@@ -0,0 +1,132 @@
+"""Versioned launchd declaration for the canonical local Mission Core service."""
+
+from __future__ import annotations
+
+import hashlib
+import plistlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+MISSION_CORE_LAUNCH_AGENT_LABEL: Final = "com.nodedc.mission-core.local"
+MISSION_CORE_LAUNCH_AGENT_SCHEMA: Final = "missioncore.local-launch-agent-plan/v1"
+
+
+class MissionCoreLaunchAgentError(RuntimeError):
+    """The local launch agent cannot be planned without weakening its boundary."""
+
+
+@dataclass(frozen=True, slots=True)
+class MissionCoreLaunchAgentPlan:
+    agent_path: Path
+    current_sha256: str
+    desired_sha256: str
+    current_program_arguments: tuple[str, ...]
+    desired_program_arguments: tuple[str, ...]
+    desired_payload: bytes
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "schema_version": MISSION_CORE_LAUNCH_AGENT_SCHEMA,
+            "label": MISSION_CORE_LAUNCH_AGENT_LABEL,
+            "agent_path": str(self.agent_path),
+            "current_sha256": self.current_sha256,
+            "desired_sha256": self.desired_sha256,
+            "current_program_arguments": list(self.current_program_arguments),
+            "desired_program_arguments": list(self.desired_program_arguments),
+            "changes": {
+                "dependency_sync_disabled": "--no-sync"
+                in self.desired_program_arguments,
+                "self_health_watchdog": True,
+                "bounded_launchd_exit_timeout_seconds": 20,
+                "keep_alive": True,
+                "process_group_owned": True,
+            },
+        }
+
+
+def plan_mission_core_launch_agent(
+    *,
+    repository_root: Path,
+    agent_path: Path,
+) -> MissionCoreLaunchAgentPlan:
+    repository = repository_root.expanduser().resolve(strict=True)
+    path = agent_path.expanduser().absolute()
+    current_payload = _read_private_regular_file(path)
+    try:
+        current = plistlib.loads(current_payload)
+    except plistlib.InvalidFileException as exc:
+        raise MissionCoreLaunchAgentError("current Mission Core launch agent is invalid") from exc
+    if not isinstance(current, dict) or current.get("Label") != MISSION_CORE_LAUNCH_AGENT_LABEL:
+        raise MissionCoreLaunchAgentError("current launch agent identity changed")
+    current_arguments = _program_arguments(current)
+    current_working_directory = current.get("WorkingDirectory")
+    if current_working_directory != str(repository):
+        raise MissionCoreLaunchAgentError("current launch agent targets another repository")
+    environment = current.get("EnvironmentVariables")
+    if not isinstance(environment, dict) or any(
+        not isinstance(key, str) or not isinstance(value, str)
+        for key, value in environment.items()
+    ):
+        raise MissionCoreLaunchAgentError("current launch agent environment is invalid")
+    # A LaunchAgent started directly from this repository's venv is denied
+    # access to ``.venv/pyvenv.cfg`` by macOS privacy controls because the
+    # checkout is below Downloads.  The Homebrew uv launcher is already the
+    # accepted local execution boundary.  ``--no-sync`` keeps launch startup
+    # deterministic and prevents dependency mutation during recovery.
+    uv_entrypoint = Path(current_arguments[0])
+    if (
+        not uv_entrypoint.is_absolute()
+        or uv_entrypoint.name != "uv"
+        or not uv_entrypoint.exists()
+    ):
+        raise MissionCoreLaunchAgentError("Mission Core uv entrypoint is unavailable")
+    desired_environment = dict(environment)
+    desired_environment["MISSIONCORE_SERVICE_WATCHDOG"] = "1"
+    log_path = repository / ".runtime/mission-core/k1link-serve-launchd.log"
+    desired: dict[str, object] = {
+        "Label": MISSION_CORE_LAUNCH_AGENT_LABEL,
+        "ProgramArguments": [
+            str(uv_entrypoint),
+            "run",
+            "--no-sync",
+            "k1link",
+            "serve",
+        ],
+        "WorkingDirectory": str(repository),
+        "EnvironmentVariables": desired_environment,
+        "KeepAlive": True,
+        "RunAtLoad": True,
+        "AbandonProcessGroup": False,
+        "ProcessType": "Background",
+        "ThrottleInterval": 5,
+        "ExitTimeOut": 20,
+        "StandardOutPath": str(log_path),
+        "StandardErrorPath": str(log_path),
+    }
+    desired_payload = plistlib.dumps(desired, fmt=plistlib.FMT_XML, sort_keys=True)
+    return MissionCoreLaunchAgentPlan(
+        agent_path=path,
+        current_sha256=_sha256(current_payload),
+        desired_sha256=_sha256(desired_payload),
+        current_program_arguments=current_arguments,
+        desired_program_arguments=tuple(desired["ProgramArguments"]),
+        desired_payload=desired_payload,
+    )
+
+
+def _program_arguments(document: dict[str, object]) -> tuple[str, ...]:
+    value = document.get("ProgramArguments")
+    if not isinstance(value, list) or not value or any(not isinstance(item, str) for item in value):
+        raise MissionCoreLaunchAgentError("launch agent program arguments are invalid")
+    return tuple(value)
+
+
+def _read_private_regular_file(path: Path) -> bytes:
+    if path.is_symlink() or not path.is_file():
+        raise MissionCoreLaunchAgentError("Mission Core launch agent is unavailable")
+    return path.read_bytes()
+
+
+def _sha256(payload: bytes) -> str:
+    return hashlib.sha256(payload).hexdigest()

src/k1link/service_watchdog.py

@@ -0,0 +1,272 @@
+"""Fail-closed health watchdog for the canonical Mission Core service process."""
+
+from __future__ import annotations
+
+import http.client
+import json
+import os
+import signal
+import stat
+import time
+from collections.abc import Callable, Mapping
+from contextlib import suppress
+from dataclasses import dataclass
+from pathlib import Path
+from threading import Event, Lock, Thread
+from typing import Final
+
+WATCHDOG_SCHEMA: Final = "missioncore.local-service-watchdog/v1"
+WATCHDOG_ENV: Final = "MISSIONCORE_SERVICE_WATCHDOG"
+WATCHDOG_ENABLED_VALUE: Final = "1"
+MISSION_CORE_SERVICE_ID: Final = "mission-core-control-plane"
+DEFAULT_JOURNAL_MAX_BYTES: Final = 4 * 1024 * 1024
+
+
+class MissionCoreWatchdogError(RuntimeError):
+    """The watchdog cannot establish a trustworthy local safety boundary."""
+
+
+@dataclass(frozen=True, slots=True)
+class MissionCoreWatchdogPolicy:
+    startup_grace_seconds: float = 45.0
+    probe_interval_seconds: float = 2.0
+    probe_timeout_seconds: float = 1.0
+    consecutive_failure_limit: int = 3
+    graceful_shutdown_seconds: float = 12.0
+
+    def __post_init__(self) -> None:
+        if (
+            self.startup_grace_seconds <= 0
+            or self.probe_interval_seconds <= 0
+            or self.probe_timeout_seconds <= 0
+            or self.consecutive_failure_limit < 1
+            or self.graceful_shutdown_seconds <= 0
+        ):
+            raise ValueError("Mission Core watchdog policy must be positive")
+
+
+class ConsecutiveHealthGate:
+    """Trigger only after a bounded sequence of genuine probe failures."""
+
+    def __init__(self, failure_limit: int) -> None:
+        if failure_limit < 1:
+            raise ValueError("health failure limit must be positive")
+        self.failure_limit = failure_limit
+        self.consecutive_failures = 0
+
+    def observe(self, healthy: bool) -> bool:
+        if healthy:
+            self.consecutive_failures = 0
+            return False
+        self.consecutive_failures += 1
+        return self.consecutive_failures >= self.failure_limit
+
+
+class MissionCoreWatchdogJournal:
+    """Append bounded, private lifecycle evidence outside the application log."""
+
+    def __init__(
+        self,
+        path: Path,
+        *,
+        max_bytes: int = DEFAULT_JOURNAL_MAX_BYTES,
+    ) -> None:
+        if max_bytes < 1:
+            raise ValueError("watchdog journal limit must be positive")
+        self.path = path.expanduser().absolute()
+        self.max_bytes = max_bytes
+        self._guard = Lock()
+        self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+        metadata = self.path.parent.lstat()
+        if not stat.S_ISDIR(metadata.st_mode):
+            raise MissionCoreWatchdogError("watchdog journal parent is not a directory")
+
+    def append(self, event: str, **details: object) -> None:
+        document = {
+            "schema_version": WATCHDOG_SCHEMA,
+            "event": event,
+            "utc_ns": time.time_ns(),
+            "monotonic_ns": time.monotonic_ns(),
+            "pid": os.getpid(),
+            **details,
+        }
+        payload = json.dumps(
+            document,
+            ensure_ascii=True,
+            separators=(",", ":"),
+            sort_keys=True,
+        ).encode("utf-8") + b"\n"
+        with self._guard:
+            self._rotate_if_needed(len(payload))
+            flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_CLOEXEC", 0)
+            flags |= getattr(os, "O_NOFOLLOW", 0)
+            descriptor = os.open(self.path, flags, 0o600)
+            try:
+                metadata = os.fstat(descriptor)
+                if (
+                    not stat.S_ISREG(metadata.st_mode)
+                    or stat.S_IMODE(metadata.st_mode) != 0o600
+                    or metadata.st_nlink != 1
+                ):
+                    raise MissionCoreWatchdogError(
+                        "watchdog journal is not a private regular file"
+                    )
+                os.write(descriptor, payload)
+                os.fsync(descriptor)
+            finally:
+                os.close(descriptor)
+
+    def _rotate_if_needed(self, incoming_bytes: int) -> None:
+        try:
+            metadata = self.path.lstat()
+        except FileNotFoundError:
+            return
+        if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:
+            raise MissionCoreWatchdogError("watchdog journal identity changed")
+        if metadata.st_size + incoming_bytes <= self.max_bytes:
+            return
+        previous = self.path.with_name(f"{self.path.name}.1")
+        with suppress(FileNotFoundError):
+            previous.unlink()
+        os.replace(self.path, previous)
+
+
+Probe = Callable[[], bool]
+SignalAction = Callable[[], None]
+
+
+class MissionCoreSelfWatchdog:
+    """Terminate a live-but-unhealthy service so its init system can restart it."""
+
+    def __init__(
+        self,
+        repository_root: Path,
+        *,
+        policy: MissionCoreWatchdogPolicy | None = None,
+        probe: Probe | None = None,
+        request_shutdown: SignalAction | None = None,
+        force_shutdown: SignalAction | None = None,
+        journal: MissionCoreWatchdogJournal | None = None,
+    ) -> None:
+        self.policy = policy or MissionCoreWatchdogPolicy()
+        self.probe = probe or _mission_core_health_probe(
+            self.policy.probe_timeout_seconds
+        )
+        self.request_shutdown = request_shutdown or _process_group_signal(signal.SIGTERM)
+        self.force_shutdown = force_shutdown or _process_group_signal(signal.SIGKILL)
+        self.journal = journal or MissionCoreWatchdogJournal(
+            repository_root.expanduser().absolute()
+            / ".runtime/mission-core/service-watchdog.jsonl"
+        )
+        self._stop = Event()
+        self._thread = Thread(
+            target=self._run,
+            name="mission-core-self-health-watchdog",
+            daemon=True,
+        )
+        self._started = False
+
+    def start(self) -> None:
+        if self._started:
+            raise MissionCoreWatchdogError("Mission Core watchdog already started")
+        self._started = True
+        self.journal.append("watchdog-started", policy=_policy_dict(self.policy))
+        self._thread.start()
+
+    def stop(self) -> None:
+        if not self._started:
+            return
+        self._stop.set()
+        self._thread.join(timeout=self.policy.probe_timeout_seconds + 1.0)
+        self.journal.append(
+            "watchdog-stopped",
+            worker_alive=self._thread.is_alive(),
+        )
+
+    def _run(self) -> None:
+        if self._stop.wait(self.policy.startup_grace_seconds):
+            return
+        gate = ConsecutiveHealthGate(self.policy.consecutive_failure_limit)
+        last_reported_health: bool | None = None
+        while not self._stop.is_set():
+            healthy = False
+            try:
+                healthy = self.probe()
+            except Exception:
+                healthy = False
+            triggered = gate.observe(healthy)
+            if healthy != last_reported_health:
+                self.journal.append(
+                    "health-state-changed",
+                    healthy=healthy,
+                    consecutive_failures=gate.consecutive_failures,
+                )
+                last_reported_health = healthy
+            if triggered:
+                self.journal.append(
+                    "restart-requested",
+                    reason="consecutive-health-probe-failures",
+                    consecutive_failures=gate.consecutive_failures,
+                )
+                self.request_shutdown()
+                if not self._stop.wait(self.policy.graceful_shutdown_seconds):
+                    self.journal.append(
+                        "restart-escalated",
+                        reason="graceful-shutdown-timeout",
+                    )
+                    self.force_shutdown()
+                return
+            if self._stop.wait(self.policy.probe_interval_seconds):
+                return
+
+
+def watchdog_enabled(environ: Mapping[str, str] = os.environ) -> bool:
+    return environ.get(WATCHDOG_ENV) == WATCHDOG_ENABLED_VALUE
+
+
+def _mission_core_health_probe(timeout_seconds: float) -> Probe:
+    def probe() -> bool:
+        connection = http.client.HTTPConnection(
+            "127.0.0.1",
+            8000,
+            timeout=timeout_seconds,
+        )
+        try:
+            connection.request("GET", "/api/health", headers={"Connection": "close"})
+            response = connection.getresponse()
+            payload = response.read(64 * 1024 + 1)
+        except (OSError, TimeoutError, http.client.HTTPException):
+            return False
+        finally:
+            connection.close()
+        if response.status != 200 or len(payload) > 64 * 1024:
+            return False
+        try:
+            document = json.loads(payload)
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            return False
+        return bool(
+            isinstance(document, dict)
+            and document.get("ok") is True
+            and document.get("status") == "ok"
+            and document.get("service") == MISSION_CORE_SERVICE_ID
+        )
+
+    return probe
+
+
+def _process_group_signal(signal_number: signal.Signals) -> SignalAction:
+    def send() -> None:
+        os.killpg(os.getpgrp(), signal_number)
+
+    return send
+
+
+def _policy_dict(policy: MissionCoreWatchdogPolicy) -> dict[str, object]:
+    return {
+        "startup_grace_seconds": policy.startup_grace_seconds,
+        "probe_interval_seconds": policy.probe_interval_seconds,
+        "probe_timeout_seconds": policy.probe_timeout_seconds,
+        "consecutive_failure_limit": policy.consecutive_failure_limit,
+        "graceful_shutdown_seconds": policy.graceful_shutdown_seconds,
+    }