273 lines
9.5 KiB
Python
273 lines
9.5 KiB
Python
"""Fail-closed health watchdog for the canonical Mission Core service process."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import http.client
|
|
import json
|
|
import os
|
|
import signal
|
|
import stat
|
|
import time
|
|
from collections.abc import Callable, Mapping
|
|
from contextlib import suppress
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from threading import Event, Lock, Thread
|
|
from typing import Final
|
|
|
|
WATCHDOG_SCHEMA: Final = "missioncore.local-service-watchdog/v1"
|
|
WATCHDOG_ENV: Final = "MISSIONCORE_SERVICE_WATCHDOG"
|
|
WATCHDOG_ENABLED_VALUE: Final = "1"
|
|
MISSION_CORE_SERVICE_ID: Final = "mission-core-control-plane"
|
|
DEFAULT_JOURNAL_MAX_BYTES: Final = 4 * 1024 * 1024
|
|
|
|
|
|
class MissionCoreWatchdogError(RuntimeError):
|
|
"""The watchdog cannot establish a trustworthy local safety boundary."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class MissionCoreWatchdogPolicy:
|
|
startup_grace_seconds: float = 45.0
|
|
probe_interval_seconds: float = 2.0
|
|
probe_timeout_seconds: float = 1.0
|
|
consecutive_failure_limit: int = 3
|
|
graceful_shutdown_seconds: float = 12.0
|
|
|
|
def __post_init__(self) -> None:
|
|
if (
|
|
self.startup_grace_seconds <= 0
|
|
or self.probe_interval_seconds <= 0
|
|
or self.probe_timeout_seconds <= 0
|
|
or self.consecutive_failure_limit < 1
|
|
or self.graceful_shutdown_seconds <= 0
|
|
):
|
|
raise ValueError("Mission Core watchdog policy must be positive")
|
|
|
|
|
|
class ConsecutiveHealthGate:
|
|
"""Trigger only after a bounded sequence of genuine probe failures."""
|
|
|
|
def __init__(self, failure_limit: int) -> None:
|
|
if failure_limit < 1:
|
|
raise ValueError("health failure limit must be positive")
|
|
self.failure_limit = failure_limit
|
|
self.consecutive_failures = 0
|
|
|
|
def observe(self, healthy: bool) -> bool:
|
|
if healthy:
|
|
self.consecutive_failures = 0
|
|
return False
|
|
self.consecutive_failures += 1
|
|
return self.consecutive_failures >= self.failure_limit
|
|
|
|
|
|
class MissionCoreWatchdogJournal:
|
|
"""Append bounded, private lifecycle evidence outside the application log."""
|
|
|
|
def __init__(
|
|
self,
|
|
path: Path,
|
|
*,
|
|
max_bytes: int = DEFAULT_JOURNAL_MAX_BYTES,
|
|
) -> None:
|
|
if max_bytes < 1:
|
|
raise ValueError("watchdog journal limit must be positive")
|
|
self.path = path.expanduser().absolute()
|
|
self.max_bytes = max_bytes
|
|
self._guard = Lock()
|
|
self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
metadata = self.path.parent.lstat()
|
|
if not stat.S_ISDIR(metadata.st_mode):
|
|
raise MissionCoreWatchdogError("watchdog journal parent is not a directory")
|
|
|
|
def append(self, event: str, **details: object) -> None:
|
|
document = {
|
|
"schema_version": WATCHDOG_SCHEMA,
|
|
"event": event,
|
|
"utc_ns": time.time_ns(),
|
|
"monotonic_ns": time.monotonic_ns(),
|
|
"pid": os.getpid(),
|
|
**details,
|
|
}
|
|
payload = json.dumps(
|
|
document,
|
|
ensure_ascii=True,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
).encode("utf-8") + b"\n"
|
|
with self._guard:
|
|
self._rotate_if_needed(len(payload))
|
|
flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_CLOEXEC", 0)
|
|
flags |= getattr(os, "O_NOFOLLOW", 0)
|
|
descriptor = os.open(self.path, flags, 0o600)
|
|
try:
|
|
metadata = os.fstat(descriptor)
|
|
if (
|
|
not stat.S_ISREG(metadata.st_mode)
|
|
or stat.S_IMODE(metadata.st_mode) != 0o600
|
|
or metadata.st_nlink != 1
|
|
):
|
|
raise MissionCoreWatchdogError(
|
|
"watchdog journal is not a private regular file"
|
|
)
|
|
os.write(descriptor, payload)
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
def _rotate_if_needed(self, incoming_bytes: int) -> None:
|
|
try:
|
|
metadata = self.path.lstat()
|
|
except FileNotFoundError:
|
|
return
|
|
if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:
|
|
raise MissionCoreWatchdogError("watchdog journal identity changed")
|
|
if metadata.st_size + incoming_bytes <= self.max_bytes:
|
|
return
|
|
previous = self.path.with_name(f"{self.path.name}.1")
|
|
with suppress(FileNotFoundError):
|
|
previous.unlink()
|
|
os.replace(self.path, previous)
|
|
|
|
|
|
Probe = Callable[[], bool]
|
|
SignalAction = Callable[[], None]
|
|
|
|
|
|
class MissionCoreSelfWatchdog:
|
|
"""Terminate a live-but-unhealthy service so its init system can restart it."""
|
|
|
|
def __init__(
|
|
self,
|
|
repository_root: Path,
|
|
*,
|
|
policy: MissionCoreWatchdogPolicy | None = None,
|
|
probe: Probe | None = None,
|
|
request_shutdown: SignalAction | None = None,
|
|
force_shutdown: SignalAction | None = None,
|
|
journal: MissionCoreWatchdogJournal | None = None,
|
|
) -> None:
|
|
self.policy = policy or MissionCoreWatchdogPolicy()
|
|
self.probe = probe or _mission_core_health_probe(
|
|
self.policy.probe_timeout_seconds
|
|
)
|
|
self.request_shutdown = request_shutdown or _process_group_signal(signal.SIGTERM)
|
|
self.force_shutdown = force_shutdown or _process_group_signal(signal.SIGKILL)
|
|
self.journal = journal or MissionCoreWatchdogJournal(
|
|
repository_root.expanduser().absolute()
|
|
/ ".runtime/mission-core/service-watchdog.jsonl"
|
|
)
|
|
self._stop = Event()
|
|
self._thread = Thread(
|
|
target=self._run,
|
|
name="mission-core-self-health-watchdog",
|
|
daemon=True,
|
|
)
|
|
self._started = False
|
|
|
|
def start(self) -> None:
|
|
if self._started:
|
|
raise MissionCoreWatchdogError("Mission Core watchdog already started")
|
|
self._started = True
|
|
self.journal.append("watchdog-started", policy=_policy_dict(self.policy))
|
|
self._thread.start()
|
|
|
|
def stop(self) -> None:
|
|
if not self._started:
|
|
return
|
|
self._stop.set()
|
|
self._thread.join(timeout=self.policy.probe_timeout_seconds + 1.0)
|
|
self.journal.append(
|
|
"watchdog-stopped",
|
|
worker_alive=self._thread.is_alive(),
|
|
)
|
|
|
|
def _run(self) -> None:
|
|
if self._stop.wait(self.policy.startup_grace_seconds):
|
|
return
|
|
gate = ConsecutiveHealthGate(self.policy.consecutive_failure_limit)
|
|
last_reported_health: bool | None = None
|
|
while not self._stop.is_set():
|
|
healthy = False
|
|
try:
|
|
healthy = self.probe()
|
|
except Exception:
|
|
healthy = False
|
|
triggered = gate.observe(healthy)
|
|
if healthy != last_reported_health:
|
|
self.journal.append(
|
|
"health-state-changed",
|
|
healthy=healthy,
|
|
consecutive_failures=gate.consecutive_failures,
|
|
)
|
|
last_reported_health = healthy
|
|
if triggered:
|
|
self.journal.append(
|
|
"restart-requested",
|
|
reason="consecutive-health-probe-failures",
|
|
consecutive_failures=gate.consecutive_failures,
|
|
)
|
|
self.request_shutdown()
|
|
if not self._stop.wait(self.policy.graceful_shutdown_seconds):
|
|
self.journal.append(
|
|
"restart-escalated",
|
|
reason="graceful-shutdown-timeout",
|
|
)
|
|
self.force_shutdown()
|
|
return
|
|
if self._stop.wait(self.policy.probe_interval_seconds):
|
|
return
|
|
|
|
|
|
def watchdog_enabled(environ: Mapping[str, str] = os.environ) -> bool:
|
|
return environ.get(WATCHDOG_ENV) == WATCHDOG_ENABLED_VALUE
|
|
|
|
|
|
def _mission_core_health_probe(timeout_seconds: float) -> Probe:
|
|
def probe() -> bool:
|
|
connection = http.client.HTTPConnection(
|
|
"127.0.0.1",
|
|
8000,
|
|
timeout=timeout_seconds,
|
|
)
|
|
try:
|
|
connection.request("GET", "/api/health", headers={"Connection": "close"})
|
|
response = connection.getresponse()
|
|
payload = response.read(64 * 1024 + 1)
|
|
except (OSError, TimeoutError, http.client.HTTPException):
|
|
return False
|
|
finally:
|
|
connection.close()
|
|
if response.status != 200 or len(payload) > 64 * 1024:
|
|
return False
|
|
try:
|
|
document = json.loads(payload)
|
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
return False
|
|
return bool(
|
|
isinstance(document, dict)
|
|
and document.get("ok") is True
|
|
and document.get("status") == "ok"
|
|
and document.get("service") == MISSION_CORE_SERVICE_ID
|
|
)
|
|
|
|
return probe
|
|
|
|
|
|
def _process_group_signal(signal_number: signal.Signals) -> SignalAction:
|
|
def send() -> None:
|
|
os.killpg(os.getpgrp(), signal_number)
|
|
|
|
return send
|
|
|
|
|
|
def _policy_dict(policy: MissionCoreWatchdogPolicy) -> dict[str, object]:
|
|
return {
|
|
"startup_grace_seconds": policy.startup_grace_seconds,
|
|
"probe_interval_seconds": policy.probe_interval_seconds,
|
|
"probe_timeout_seconds": policy.probe_timeout_seconds,
|
|
"consecutive_failure_limit": policy.consecutive_failure_limit,
|
|
"graceful_shutdown_seconds": policy.graceful_shutdown_seconds,
|
|
}
|