fix(service): supervise canonical Mission Core lifecycle

tests/test_cli.py

@@ -83,6 +83,7 @@ def test_serve_resolves_frontend_from_repository_root(monkeypatch: Any) -> None:
         "port": 8000,
         "log_level": "info",
         "access_log": True,
+        "timeout_graceful_shutdown": 10,
     }
     assert lease.active is False
 

tests/test_local_service_launchd.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import plistlib
+from pathlib import Path
+
+from k1link.local_service_launchd import plan_mission_core_launch_agent
+
+
+def test_launch_agent_plan_disables_sync_and_enables_watchdog(tmp_path: Path) -> None:
+    repository = tmp_path / "repo"
+    repository.mkdir()
+    uv_entrypoint = tmp_path / "uv"
+    uv_entrypoint.write_text("#!/bin/sh\n")
+    uv_entrypoint.chmod(0o700)
+    agent = tmp_path / "agent.plist"
+    agent.write_bytes(
+        plistlib.dumps(
+            {
+                "Label": "com.nodedc.mission-core.local",
+                "ProgramArguments": [str(uv_entrypoint), "run", "k1link", "serve"],
+                "WorkingDirectory": str(repository),
+                "EnvironmentVariables": {"PATH": "/usr/bin:/bin"},
+            }
+        )
+    )
+    agent.chmod(0o600)
+
+    plan = plan_mission_core_launch_agent(
+        repository_root=repository,
+        agent_path=agent,
+    )
+    desired = plistlib.loads(plan.desired_payload)
+
+    assert desired["ProgramArguments"] == [
+        str(uv_entrypoint),
+        "run",
+        "--no-sync",
+        "k1link",
+        "serve",
+    ]
+    assert desired["EnvironmentVariables"]["MISSIONCORE_SERVICE_WATCHDOG"] == "1"
+    assert desired["KeepAlive"] is True
+    assert desired["RunAtLoad"] is True
+    assert desired["AbandonProcessGroup"] is False
+    assert desired["ExitTimeOut"] == 20
+    assert plan.current_sha256 != plan.desired_sha256

tests/test_manage_mission_core_launch_agent.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import importlib.util
+import subprocess
+from pathlib import Path
+from typing import Any
+
+_SCRIPT = Path(__file__).parents[1] / "scripts/manage_mission_core_launch_agent.py"
+_SPEC = importlib.util.spec_from_file_location(
+    "manage_mission_core_launch_agent",
+    _SCRIPT,
+)
+assert _SPEC is not None and _SPEC.loader is not None
+manager = importlib.util.module_from_spec(_SPEC)
+_SPEC.loader.exec_module(manager)
+
+
+def test_reload_waits_for_launchd_transition_before_bootstrap(
+    monkeypatch: Any,
+    tmp_path: Path,
+) -> None:
+    commands: list[tuple[str, ...]] = []
+    print_results = iter((0, 0, 1))
+
+    def fake_run(arguments: list[str], **_: object) -> subprocess.CompletedProcess[str]:
+        command = tuple(arguments)
+        commands.append(command)
+        if arguments[1] == "print":
+            return subprocess.CompletedProcess(arguments, next(print_results), "", "")
+        return subprocess.CompletedProcess(arguments, 0, "", "")
+
+    monkeypatch.setattr(manager.subprocess, "run", fake_run)
+    monkeypatch.setattr(manager.time, "sleep", lambda _: None)
+
+    agent = tmp_path / "agent.plist"
+    manager._reload_launch_agent(agent)
+
+    assert [command[1] for command in commands] == [
+        "bootout",
+        "print",
+        "print",
+        "print",
+        "bootstrap",
+    ]
+
+
+def test_reload_rejects_failed_bootout_while_job_is_still_loaded(
+    monkeypatch: Any,
+    tmp_path: Path,
+) -> None:
+    def fake_run(arguments: list[str], **_: object) -> subprocess.CompletedProcess[str]:
+        if arguments[1] == "bootout":
+            return subprocess.CompletedProcess(arguments, 5, "", "")
+        return subprocess.CompletedProcess(arguments, 0, "", "")
+
+    monkeypatch.setattr(manager.subprocess, "run", fake_run)
+
+    try:
+        manager._reload_launch_agent(tmp_path / "agent.plist")
+    except manager.MissionCoreLaunchAgentError as exc:
+        assert "bootout failed with exit code 5" in str(exc)
+    else:
+        raise AssertionError("failed bootout was accepted")

tests/test_service_watchdog.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from threading import Event
+
+from k1link.service_watchdog import (
+    ConsecutiveHealthGate,
+    MissionCoreSelfWatchdog,
+    MissionCoreWatchdogJournal,
+    MissionCoreWatchdogPolicy,
+    watchdog_enabled,
+)
+
+
+def test_consecutive_health_gate_resets_after_recovery() -> None:
+    gate = ConsecutiveHealthGate(3)
+
+    assert gate.observe(False) is False
+    assert gate.observe(False) is False
+    assert gate.observe(True) is False
+    assert gate.consecutive_failures == 0
+    assert gate.observe(False) is False
+    assert gate.observe(False) is False
+    assert gate.observe(False) is True
+
+
+def test_self_watchdog_escalates_a_persistently_unhealthy_process(tmp_path: Path) -> None:
+    requested = Event()
+    forced = Event()
+    journal_path = tmp_path / "watchdog.jsonl"
+    watchdog = MissionCoreSelfWatchdog(
+        tmp_path,
+        policy=MissionCoreWatchdogPolicy(
+            startup_grace_seconds=0.01,
+            probe_interval_seconds=0.01,
+            probe_timeout_seconds=0.01,
+            consecutive_failure_limit=2,
+            graceful_shutdown_seconds=0.02,
+        ),
+        probe=lambda: False,
+        request_shutdown=requested.set,
+        force_shutdown=forced.set,
+        journal=MissionCoreWatchdogJournal(journal_path),
+    )
+
+    watchdog.start()
+
+    assert requested.wait(0.5)
+    assert forced.wait(0.5)
+    watchdog.stop()
+    events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
+    assert events == [
+        "watchdog-started",
+        "health-state-changed",
+        "restart-requested",
+        "restart-escalated",
+        "watchdog-stopped",
+    ]
+
+
+def test_self_watchdog_leaves_a_healthy_process_running(tmp_path: Path) -> None:
+    probed = Event()
+    requested = Event()
+    forced = Event()
+    journal_path = tmp_path / "watchdog.jsonl"
+
+    def healthy_probe() -> bool:
+        probed.set()
+        return True
+
+    watchdog = MissionCoreSelfWatchdog(
+        tmp_path,
+        policy=MissionCoreWatchdogPolicy(
+            startup_grace_seconds=0.01,
+            probe_interval_seconds=0.01,
+            probe_timeout_seconds=0.01,
+            consecutive_failure_limit=2,
+            graceful_shutdown_seconds=0.02,
+        ),
+        probe=healthy_probe,
+        request_shutdown=requested.set,
+        force_shutdown=forced.set,
+        journal=MissionCoreWatchdogJournal(journal_path),
+    )
+
+    watchdog.start()
+
+    assert probed.wait(0.5)
+    watchdog.stop()
+    assert requested.is_set() is False
+    assert forced.is_set() is False
+    events = [json.loads(line)["event"] for line in journal_path.read_text().splitlines()]
+    assert events == [
+        "watchdog-started",
+        "health-state-changed",
+        "watchdog-stopped",
+    ]
+
+
+def test_watchdog_journal_rotates_before_exceeding_bound(tmp_path: Path) -> None:
+    path = tmp_path / "watchdog.jsonl"
+    journal = MissionCoreWatchdogJournal(path, max_bytes=300)
+
+    journal.append("first", payload="x" * 180)
+    journal.append("second", payload="y" * 180)
+
+    assert path.is_file()
+    assert path.with_name("watchdog.jsonl.1").is_file()
+    assert json.loads(path.read_text())["event"] == "second"
+
+
+def test_watchdog_requires_exact_enable_marker() -> None:
+    assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "1"}) is True
+    assert watchdog_enabled({"MISSIONCORE_SERVICE_WATCHDOG": "true"}) is False
+    assert watchdog_enabled({}) is False