117 lines
3.4 KiB
Python
117 lines
3.4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from threading import Event
|
|
|
|
from k1link.service_watchdog import (
|
|
ConsecutiveHealthGate,
|
|
MissionCoreSelfWatchdog,
|
|
MissionCoreWatchdogJournal,
|
|
MissionCoreWatchdogPolicy,
|
|
watchdog_enabled,
|
|
)
|
|
|
|
|
|
def test_consecutive_health_gate_resets_after_recovery() -> None:
|
|
gate = ConsecutiveHealthGate(3)
|
|
|
|
assert gate.observe(False) is False
|
|
assert gate.observe(False) is False
|
|
assert gate.observe(True) is False
|
|
assert gate.consecutive_failures == 0
|
|
assert gate.observe(False) is False
|
|
assert gate.observe(False) is False
|
|
assert gate.observe(False) is True
|
|
|
|
|
|
def test_self_watchdog_escalates_a_persistently_unhealthy_process(tmp_path: Path) -> None:
|
|
requested = Event()
|
|
forced = Event()
|
|
journal_path = tmp_path / "watchdog.jsonl"
|
|
watchdog = MissionCoreSelfWatchdog(
|
|
tmp_path,
|
|
policy=MissionCoreWatchdogPolicy(
|
|
startup_grace_seconds=0.01,
|
|
probe_interval_seconds=0.01,
|
|
probe_timeout_seconds=0.01,
|
|
consecutive_failure_limit=2,
|
|
graceful_shutdown_seconds=0.02,
|
|
),
|
|
probe=lambda: False,
|
|
request_shutdown=requested.set,
|
|
force_shutdown=forced.set,
|
|
journal=MissionCoreWatchdogJournal(journal_path),
|
|
)
|
|
|
|
watchdog.start()
|
|
|
|
assert requested.wait(0.5)
|
|
assert forced.wait(0.5)
|
|
watchdog.stop()
|
|
events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
|
|
assert events == [
|
|
"watchdog-started",
|
|
"health-state-changed",
|
|
"restart-requested",
|
|
"restart-escalated",
|
|
"watchdog-stopped",
|
|
]
|
|
|
|
|
|
def test_self_watchdog_leaves_a_healthy_process_running(tmp_path: Path) -> None:
|
|
probed = Event()
|
|
requested = Event()
|
|
forced = Event()
|
|
journal_path = tmp_path / "watchdog.jsonl"
|
|
|
|
def healthy_probe() -> bool:
|
|
probed.set()
|
|
return True
|
|
|
|
watchdog = MissionCoreSelfWatchdog(
|
|
tmp_path,
|
|
policy=MissionCoreWatchdogPolicy(
|
|
startup_grace_seconds=0.01,
|
|
probe_interval_seconds=0.01,
|
|
probe_timeout_seconds=0.01,
|
|
consecutive_failure_limit=2,
|
|
graceful_shutdown_seconds=0.02,
|
|
),
|
|
probe=healthy_probe,
|
|
request_shutdown=requested.set,
|
|
force_shutdown=forced.set,
|
|
journal=MissionCoreWatchdogJournal(journal_path),
|
|
)
|
|
|
|
watchdog.start()
|
|
|
|
assert probed.wait(0.5)
|
|
watchdog.stop()
|
|
assert requested.is_set() is False
|
|
assert forced.is_set() is False
|
|
events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
|
|
assert events == [
|
|
"watchdog-started",
|
|
"health-state-changed",
|
|
"watchdog-stopped",
|
|
]
|
|
|
|
|
|
def test_watchdog_journal_rotates_before_exceeding_bound(tmp_path: Path) -> None:
|
|
path = tmp_path / "watchdog.jsonl"
|
|
journal = MissionCoreWatchdogJournal(path, max_bytes=300)
|
|
|
|
journal.append("first", payload="x" * 180)
|
|
journal.append("second", payload="y" * 180)
|
|
|
|
assert path.is_file()
|
|
assert path.with_name("watchdog.jsonl.1").is_file()
|
|
assert json.loads(path.read_text())["event"] == "second"
|
|
|
|
|
|
def test_watchdog_requires_exact_enable_marker() -> None:
|
|
assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "1"}) is True
|
|
assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "true"}) is False
|
|
assert watchdog_enabled({}) is False
|