fix(telemetry): recover worker agent and Tailscale transport automatically

2026-09-25 23:24:27 +03:00
parent bd5873ef34
commit f3640026cf
11 changed files with 755 additions and 4 deletions
@@ -0,0 +1,99 @@
+"""Bounded operator acceptance of installed telemetry recovery, no inference jobs.
+
+Each case injects one failure in an exact Mission Core telemetry component.
+An existing service's automatic supervisor must restore fresh worker telemetry.
+"""
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import time
+import urllib.request
+from datetime import datetime
+
+DOCKER = "/Applications/Docker.app/Contents/Resources/bin/docker"
+BASE = "http://127.0.0.1:8000/api/v1/system/contours/worker-006/telemetry?history=1"
+
+
+def read():
+    return json.load(urllib.request.urlopen(BASE, timeout=5))
+
+
+def tunnel_pid():
+    r = subprocess.run(
+        ["launchctl", "print", f"gui/{os.getuid()}/com.nodedc.telemetry-tunnel.local"],
+        capture_output=True,
+        text=True,
+    )
+    m = re.search(r"\bpid = (\d+)", r.stdout)
+    return int(m.group(1)) if m else None
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("case", choices=["tunnel", "receiver"])
+    args = parser.parse_args()
+    baseline = read()["connection"]
+    if not baseline["reachable"] or not baseline["identity_matches"]:
+        raise RuntimeError("Fresh confirmed baseline required")
+    started = time.monotonic()
+    baseline_time = datetime.fromisoformat(baseline["observed_at_utc"].replace("Z", "+00:00"))
+    before_pid = tunnel_pid()
+    if args.case == "tunnel":
+        subprocess.run(
+            [
+                "launchctl",
+                "kill",
+                "SIGKILL",
+                f"gui/{os.getuid()}/com.nodedc.telemetry-tunnel.local",
+            ],
+            check=True,
+            capture_output=True,
+        )
+    else:
+        subprocess.run(
+            [DOCKER, "stop", "--time", "5", "ndc-mission-core-telemetry-normalizer"],
+            check=True,
+            capture_output=True,
+            timeout=15,
+        )
+    seen = set()
+    while time.monotonic() - started < 90:
+        time.sleep(2)
+        try:
+            probe = read()["connection"]
+            code = probe.get("error_code")
+            seen.add(code or "fresh")
+            observed = datetime.fromisoformat(probe["observed_at_utc"].replace("Z", "+00:00"))
+            if (
+                probe["reachable"]
+                and probe["identity_matches"]
+                and (observed - baseline_time).total_seconds() >= 10
+                and time.monotonic() - started >= 10
+                and (
+                    args.case != "tunnel"
+                    or (tunnel_pid() is not None and tunnel_pid() != before_pid)
+                )
+                and (args.case != "receiver" or "telemetry-receiver-unavailable" in seen)
+            ):
+                print(
+                    json.dumps(
+                        {
+                            "case": args.case,
+                            "automaticRecovery": True,
+                            "elapsedSeconds": round(time.monotonic() - started, 2),
+                            "observedStates": sorted(seen),
+                            "observedAt": probe["observed_at_utc"],
+                        }
+                    )
+                )
+                return
+        except (OSError, ValueError):
+            seen.add("query-unavailable")
+    raise RuntimeError("Automatic telemetry recovery not accepted within 90 seconds")
+
+
+if __name__ == "__main__":
+    main()