"""Bounded operator acceptance of installed telemetry recovery, no inference jobs. Each case injects one failure in an exact Mission Core telemetry component. An existing service's automatic supervisor must restore fresh worker telemetry. """ import argparse import json import os import re import subprocess import time import urllib.request from datetime import datetime DOCKER = "/Applications/Docker.app/Contents/Resources/bin/docker" BASE = "http://127.0.0.1:8000/api/v1/system/contours/worker-006/telemetry?history=1" def read(): return json.load(urllib.request.urlopen(BASE, timeout=5)) def tunnel_pid(): r = subprocess.run( ["launchctl", "print", f"gui/{os.getuid()}/com.nodedc.telemetry-tunnel.local"], capture_output=True, text=True, ) m = re.search(r"\bpid = (\d+)", r.stdout) return int(m.group(1)) if m else None def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("case", choices=["tunnel", "receiver"]) args = parser.parse_args() baseline = read()["connection"] if not baseline["reachable"] or not baseline["identity_matches"]: raise RuntimeError("Fresh confirmed baseline required") started = time.monotonic() baseline_time = datetime.fromisoformat(baseline["observed_at_utc"].replace("Z", "+00:00")) before_pid = tunnel_pid() if args.case == "tunnel": subprocess.run( [ "launchctl", "kill", "SIGKILL", f"gui/{os.getuid()}/com.nodedc.telemetry-tunnel.local", ], check=True, capture_output=True, ) else: subprocess.run( [DOCKER, "stop", "--time", "5", "ndc-mission-core-telemetry-normalizer"], check=True, capture_output=True, timeout=15, ) seen = set() while time.monotonic() - started < 90: time.sleep(2) try: probe = read()["connection"] code = probe.get("error_code") seen.add(code or "fresh") observed = datetime.fromisoformat(probe["observed_at_utc"].replace("Z", "+00:00")) if ( probe["reachable"] and probe["identity_matches"] and (observed - baseline_time).total_seconds() >= 10 and time.monotonic() - started >= 10 and ( args.case != "tunnel" or (tunnel_pid() is not None and tunnel_pid() != before_pid) ) and (args.case != "receiver" or "telemetry-receiver-unavailable" in seen) ): print( json.dumps( { "case": args.case, "automaticRecovery": True, "elapsedSeconds": round(time.monotonic() - started, 2), "observedStates": sorted(seen), "observedAt": probe["observed_at_utc"], } ) ) return except (OSError, ValueError): seen.add("query-unavailable") raise RuntimeError("Automatic telemetry recovery not accepted within 90 seconds") if __name__ == "__main__": main()