From a7c64e009d1b1f482b96da101f14c99e171b6a9b Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Fri, 25 Sep 2026 16:40:45 +0300 Subject: [PATCH] fix(core): keep launchd service logs outside protected Downloads --- .../audits/2026-09-22-core-launch-recovery.md | 38 +++++++ .../MISSION_CORE_LOCAL_SERVICE_RECOVERY.md | 18 ++++ scripts/manage_mission_core_launch_agent.py | 2 + scripts/migrate_worker_tunnel_logs.py | 101 ++++++++++++++++++ src/k1link/launchd_logs.py | 33 ++++++ src/k1link/local_service_launchd.py | 5 +- .../observatory/worker_tunnel_launchd.py | 4 +- tests/test_launchd_logs.py | 31 ++++++ 8 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 docs/audits/2026-09-22-core-launch-recovery.md create mode 100644 scripts/migrate_worker_tunnel_logs.py create mode 100644 src/k1link/launchd_logs.py create mode 100644 tests/test_launchd_logs.py diff --git a/docs/audits/2026-09-22-core-launch-recovery.md b/docs/audits/2026-09-22-core-launch-recovery.md new file mode 100644 index 0000000..907cb89 --- /dev/null +++ b/docs/audits/2026-09-22-core-launch-recovery.md @@ -0,0 +1,38 @@ +# Core и Worker · восстановление запуска 22 сентября + +После перезапуска Mac штатные LaunchAgents Core и двух старых Worker-туннелей +завершались до запуска приложения с EX_CONFIG/78. Журнал macOS показал +`xpcproxy deny file-read-data` для startup-журналов внутри Downloads и затем +`posix_spawn Operation not permitted`. Фоновые элементы при этом были enabled/allowed. +Это отдельный сбой запуска операторского контура, не ошибка навигации Isaac. + +Исправление вынесло только launchd stdout/stderr в +`~/Library/Logs/NODE.DC/MissionCore/`. Исходные журналы и evidence оставлены на месте. +Путь checkout, данные, env, SSH arguments/ports и полномочия не менялись. +Разрешения macOS не расширялись. Tailscale включил владелец; после этого SSH +к Worker 006 снова заработал. Применение выполнено с проверкой обеих идентичностей +plist, резервной копией и ожиданием полного bootout перед bootstrap. + +Версионированная реализация: `src/k1link/launchd_logs.py`, +`local_service_launchd.py`, `observatory/worker_tunnel_launchd.py`, +`scripts/manage_mission_core_launch_agent.py`, `scripts/migrate_worker_tunnel_logs.py`. +Планирование не создаёт файлы; apply создаёт только новый приватный журнал, +отклоняет symlink и не обнуляет существующее содержимое. Старый Core startup-log +остаётся в `.runtime/mission-core/k1link-serve-launchd.log` как исторический файл. + +| Служба | Предыдущий plist SHA-256 | Применённый plist SHA-256 | +|---|---|---| +| Core | `5d7a58122c016f8021fede27d1b21891ee20238fc38c281e5b33574ff8204e3f` | `182a87853bc363555edea6af5911c288ec2e83e45354ba2e22a4090d50064486` | +| Observatory tunnel | `db1df305e9ca135218e782b57e59a0ebb354c5267fdfe8a2379a1f4bc1a82880` | `1449901c566f576ca7a21accc60d12012c4f0a1c5d7f380c72e7b9210297b31b` | +| Gaussian tunnel | `88d5672b1a3620e68e0d622e00ebc22ee04aba5d774ed92dd84ed580d9ce3fad` | `3ac365e2cd173283d30c85dac38585440849e47ebbe9cd6b153778095adf6b80` | + +Проверки: 16 focused launchd/tunnel tests прошли. Core health вернул +`ok=true`, `status=ok`, `service=mission-core-control-plane`, `operational=true`. +Единственный Python listener — 127.0.0.1:8000; на8765 слушателя нет. +Оба прежних SSH-туннеля имеют устойчивые launchd PID. Каталог AI-полигона +подтвердил `runtime.available=true`, активных заездов нет. + +С задачей «Mission Core - PC - SINHRON» согласован один владелец запуска: +эта задача восстанавливает службы, соседняя не выполняет параллельных рестартов. +Готовность передана туда после health/worker проверки. Повторный reboot-drill +не проводился; подтверждён именно текущий штатный запуск после исправления. diff --git a/docs/runbooks/MISSION_CORE_LOCAL_SERVICE_RECOVERY.md b/docs/runbooks/MISSION_CORE_LOCAL_SERVICE_RECOVERY.md index 793a88c..4dc2eb1 100644 --- a/docs/runbooks/MISSION_CORE_LOCAL_SERVICE_RECOVERY.md +++ b/docs/runbooks/MISSION_CORE_LOCAL_SERVICE_RECOVERY.md @@ -151,3 +151,21 @@ and PGID `42895` produced a new LaunchAgent PID `42942` and exact health in operator-station evidence only. An onboard Linux deployment must express the same contract in its init system and pass its own power-loss, crash, hang and durable-state reconciliation qualification. + +## Startup journals outside Downloads + +Launch-time stdout/stderr now live in +`~/Library/Logs/NODE.DC/MissionCore/k1link-serve-launchd.log`. +The application's own evidence and watchdog journals remain in their configured +private data directory. launchd/xpcproxy opens stdout before application folder +permissions apply; putting this startup journal in a Downloads checkout caused +EX_CONFIG/78 after the 22 September restart. Do not broaden privacy permissions +or start a duplicate backend to mask this failure. + +The normal hash-bound Core plan/apply prepares this private log. For the two +existing Worker SSH declarations, use `scripts/migrate_worker_tunnel_logs.py` +`plan observatory` / `plan gaussian`, review hashes, then `apply` with the two +`--expected-…-sha256` arguments. The migration changes only stdout/stderr paths, +preserves every SSH option, retains a backup and checks a stable exact PID. +Confirm Worker registration in the Core catalog after tunnel recovery. +See `docs/audits/2026-09-22-core-launch-recovery.md` for the observed incident. diff --git a/scripts/manage_mission_core_launch_agent.py b/scripts/manage_mission_core_launch_agent.py index b9b1f0b..7e7a9ba 100644 --- a/scripts/manage_mission_core_launch_agent.py +++ b/scripts/manage_mission_core_launch_agent.py @@ -14,6 +14,7 @@ import time from contextlib import suppress from pathlib import Path +from k1link.launchd_logs import launchd_log_path, prepare_launchd_log from k1link.local_service_launchd import ( MISSION_CORE_LAUNCH_AGENT_LABEL, MissionCoreLaunchAgentError, @@ -81,6 +82,7 @@ def main() -> int: raise MissionCoreLaunchAgentError("launch agent backup identity collision") if not backup.exists(): _write_atomic(backup, previous) + prepare_launchd_log(launchd_log_path("k1link-serve-launchd.log")) _write_atomic(plan.agent_path, plan.desired_payload) try: _reload_launch_agent(plan.agent_path) diff --git a/scripts/migrate_worker_tunnel_logs.py b/scripts/migrate_worker_tunnel_logs.py new file mode 100644 index 0000000..c558d4c --- /dev/null +++ b/scripts/migrate_worker_tunnel_logs.py @@ -0,0 +1,101 @@ +"""Move only the two installed Worker tunnel startup journals to Library/Logs. + +Preserves every SSH argument and all other launchd fields. Plan/apply is bound +to both hashes; failed health acceptance restores the previous declaration. +""" + +import argparse +import hashlib +import json +import os +import plistlib +import subprocess +import time +from pathlib import Path + +from manage_mission_core_launch_agent import _write_atomic + +from k1link.launchd_logs import launchd_log_path, prepare_launchd_log + +LABELS = { + "observatory": "com.nodedc.observatory-worker-tunnel.local", + "gaussian": "com.nodedc.gaussian-pipeline-tunnel.local", +} + + +def reload_agent(path, label): + domain = f"gui/{os.getuid()}" + target = f"{domain}/{label}" + subprocess.run(["launchctl", "bootout", target], capture_output=True, timeout=20) + deadline = time.monotonic() + 20 + while subprocess.run(["launchctl", "print", target], capture_output=True).returncode == 0: + if time.monotonic() > deadline: + raise RuntimeError("Previous tunnel did not unload") + time.sleep(0.2) + subprocess.run(["launchctl", "bootstrap", domain, str(path)], check=True, timeout=20) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("action", choices=["plan", "apply"]) + p.add_argument("tunnel", choices=LABELS) + p.add_argument("--expected-current-sha256") + p.add_argument("--expected-desired-sha256") + a = p.parse_args() + label = LABELS[a.tunnel] + agent = Path.home() / "Library/LaunchAgents" / (label + ".plist") + if agent.is_symlink(): + raise ValueError("LaunchAgent cannot be a symlink") + previous = agent.read_bytes() + desired = plistlib.loads(previous) + if desired["Label"] != label or desired["ProgramArguments"][0] != "/usr/bin/ssh": + raise ValueError("Unexpected tunnel declaration") + log = launchd_log_path(a.tunnel + "-worker-tunnel.log") + desired.update(StandardOutPath=str(log), StandardErrorPath=str(log)) + payload = plistlib.dumps(desired, sort_keys=True) + plan = dict( + label=label, + current_sha256=hashlib.sha256(previous).hexdigest(), + desired_sha256=hashlib.sha256(payload).hexdigest(), + log_path=str(log), + ) + if a.action == "plan": + print(json.dumps(plan)) + return + if ( + a.expected_current_sha256 != plan["current_sha256"] + or a.expected_desired_sha256 != plan["desired_sha256"] + ): + raise ValueError("Tunnel plan changed before apply") + prepare_launchd_log(log) + backup = log.parent / (label + "." + plan["current_sha256"] + ".plist") + if backup.exists() and backup.read_bytes() != previous: + raise ValueError("Backup identity collision") + _write_atomic(backup, previous) + _write_atomic(agent, payload) + try: + reload_agent(agent, label) + # ExitOnForwardFailure makes an occupied remote listener terminate SSH. + # Check a stable exact launchd PID over several reconnect intervals. + target = f"gui/{os.getuid()}/{label}" + pid, stable = None, 0 + for _ in range(12): + time.sleep(1) + state = subprocess.check_output(["launchctl", "print", target], text=True) + found = next( + (x.strip() for x in state.splitlines() if x.strip().startswith("pid =")), None + ) + stable = stable + 1 if found and found == pid and "state = running" in state else 0 + pid = found + if stable >= 6: + print(json.dumps({**plan, "applied": True, "pid": pid})) + return + raise RuntimeError("Tunnel process did not remain running") + except BaseException: + _write_atomic(agent, previous) + reload_agent(agent, label) + raise + + +if __name__ == "__main__": + main() diff --git a/src/k1link/launchd_logs.py b/src/k1link/launchd_logs.py new file mode 100644 index 0000000..7ee9051 --- /dev/null +++ b/src/k1link/launchd_logs.py @@ -0,0 +1,33 @@ +"""Launch-time journals belong in Library/Logs, outside protected Documents/Downloads. + +launchd opens these before the application starts, so an application's existing +folder grant cannot authorize xpcproxy to open a journal inside the checkout. +This does not move evidence or change access to the application's data directory. +""" + +import os +import stat +from pathlib import Path + + +def launchd_log_path(name: str) -> Path: + if Path(name).name != name or not name.endswith(".log"): + raise ValueError("A single journal filename is required") + return Path.home() / "Library/Logs/NODE.DC/MissionCore" / name + + +def prepare_launchd_log(path: Path) -> None: + if path != launchd_log_path(path.name): + raise ValueError("Unexpected launch journal directory") + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + info = path.parent.lstat() + if path.parent.resolve() != path.parent or info.st_uid != os.getuid(): + raise ValueError("Launch journal directory must be owned and canonical") + descriptor = os.open(path, os.O_CREAT | os.O_APPEND | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid(): + raise ValueError("Launch journal must be an owned regular file") + os.fchmod(descriptor, 0o600) + finally: + os.close(descriptor) diff --git a/src/k1link/local_service_launchd.py b/src/k1link/local_service_launchd.py index 2e59a8f..86b32b9 100644 --- a/src/k1link/local_service_launchd.py +++ b/src/k1link/local_service_launchd.py @@ -10,6 +10,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Final, cast +from k1link.launchd_logs import launchd_log_path + MISSION_CORE_LAUNCH_AGENT_LABEL: Final = "com.nodedc.mission-core.local" MISSION_CORE_LAUNCH_AGENT_SCHEMA: Final = "missioncore.local-launch-agent-plan/v1" OBSERVATORY_LOCAL_WORKER_ENABLED_ENV: Final = ( @@ -60,6 +62,7 @@ class MissionCoreLaunchAgentPlan: "desired_program_arguments": list(self.desired_program_arguments), "current_process_type": self.current_process_type, "desired_process_type": "Interactive", + "desired_log_path": plistlib.loads(self.desired_payload)["StandardOutPath"], "changes": { "repository_migration": self.current_working_directory != self.desired_working_directory, @@ -172,7 +175,7 @@ def plan_mission_core_launch_agent( desired_environment[OBSERVATORY_SOURCE_CAS_ROOT_ENV] = str(source_cas) desired_environment[OBSERVATORY_RESULT_STAGING_ROOT_ENV] = str(result_staging) desired_environment[OBSERVATORY_LOCAL_WORKER_ENABLED_ENV] = "1" - log_path = repository / ".runtime/mission-core/k1link-serve-launchd.log" + log_path = launchd_log_path("k1link-serve-launchd.log") desired_program_arguments = ( str(uv_entrypoint), "run", diff --git a/src/k1link/observatory/worker_tunnel_launchd.py b/src/k1link/observatory/worker_tunnel_launchd.py index 3507ec7..7d0bfb0 100644 --- a/src/k1link/observatory/worker_tunnel_launchd.py +++ b/src/k1link/observatory/worker_tunnel_launchd.py @@ -10,6 +10,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Final +from k1link.launchd_logs import launchd_log_path + OBSERVATORY_WORKER_TUNNEL_LABEL: Final = ( "com.nodedc.observatory-worker-tunnel.local" ) @@ -64,7 +66,7 @@ def plan_observatory_worker_tunnel_launch_agent( data_root = _private_directory(data_directory) ssh = _exact_executable(ssh_path) target_path = agent_path.expanduser().absolute() - log_path = data_root / "observatory-worker-tunnel.log" + log_path = launchd_log_path("observatory-worker-tunnel.log") arguments = [ str(ssh), "-o", diff --git a/tests/test_launchd_logs.py b/tests/test_launchd_logs.py new file mode 100644 index 0000000..ec2908c --- /dev/null +++ b/tests/test_launchd_logs.py @@ -0,0 +1,31 @@ +import stat +from pathlib import Path + +import pytest + +from k1link.launchd_logs import launchd_log_path, prepare_launchd_log + + +def test_journal_is_private_and_keeps_existing_evidence(tmp_path, monkeypatch): + monkeypatch.setattr(Path, "home", lambda: tmp_path) + path = launchd_log_path("core.log") + prepare_launchd_log(path) + path.write_text("existing evidence\n") + prepare_launchd_log(path) + assert path.read_text() == "existing evidence\n" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert path == tmp_path / "Library/Logs/NODE.DC/MissionCore/core.log" + + +def test_journal_rejects_symlink_and_unexpected_destination(tmp_path, monkeypatch): + monkeypatch.setattr(Path, "home", lambda: tmp_path) + target = tmp_path / "evidence" + target.write_text("immutable") + path = launchd_log_path("core.log") + path.parent.mkdir(parents=True) + path.symlink_to(target) + with pytest.raises(OSError): + prepare_launchd_log(path) + with pytest.raises(ValueError): + prepare_launchd_log(tmp_path / "Downloads/core.log") + assert target.read_text() == "immutable"