100 lines
3.3 KiB
Python
100 lines
3.3 KiB
Python
"""Bounded operator acceptance of installed telemetry recovery, no inference jobs.
|
|
|
|
Each case injects one failure in an exact Mission Core telemetry component.
|
|
An existing service's automatic supervisor must restore fresh worker telemetry.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import urllib.request
|
|
from datetime import datetime
|
|
|
|
DOCKER = "/Applications/Docker.app/Contents/Resources/bin/docker"
|
|
BASE = "http://127.0.0.1:8000/api/v1/system/contours/worker-006/telemetry?history=1"
|
|
|
|
|
|
def read():
|
|
return json.load(urllib.request.urlopen(BASE, timeout=5))
|
|
|
|
|
|
def tunnel_pid():
|
|
r = subprocess.run(
|
|
["launchctl", "print", f"gui/{os.getuid()}/com.nodedc.telemetry-tunnel.local"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
m = re.search(r"\bpid = (\d+)", r.stdout)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("case", choices=["tunnel", "receiver"])
|
|
args = parser.parse_args()
|
|
baseline = read()["connection"]
|
|
if not baseline["reachable"] or not baseline["identity_matches"]:
|
|
raise RuntimeError("Fresh confirmed baseline required")
|
|
started = time.monotonic()
|
|
baseline_time = datetime.fromisoformat(baseline["observed_at_utc"].replace("Z", "+00:00"))
|
|
before_pid = tunnel_pid()
|
|
if args.case == "tunnel":
|
|
subprocess.run(
|
|
[
|
|
"launchctl",
|
|
"kill",
|
|
"SIGKILL",
|
|
f"gui/{os.getuid()}/com.nodedc.telemetry-tunnel.local",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
else:
|
|
subprocess.run(
|
|
[DOCKER, "stop", "--time", "5", "ndc-mission-core-telemetry-normalizer"],
|
|
check=True,
|
|
capture_output=True,
|
|
timeout=15,
|
|
)
|
|
seen = set()
|
|
while time.monotonic() - started < 90:
|
|
time.sleep(2)
|
|
try:
|
|
probe = read()["connection"]
|
|
code = probe.get("error_code")
|
|
seen.add(code or "fresh")
|
|
observed = datetime.fromisoformat(probe["observed_at_utc"].replace("Z", "+00:00"))
|
|
if (
|
|
probe["reachable"]
|
|
and probe["identity_matches"]
|
|
and (observed - baseline_time).total_seconds() >= 10
|
|
and time.monotonic() - started >= 10
|
|
and (
|
|
args.case != "tunnel"
|
|
or (tunnel_pid() is not None and tunnel_pid() != before_pid)
|
|
)
|
|
and (args.case != "receiver" or "telemetry-receiver-unavailable" in seen)
|
|
):
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"case": args.case,
|
|
"automaticRecovery": True,
|
|
"elapsedSeconds": round(time.monotonic() - started, 2),
|
|
"observedStates": sorted(seen),
|
|
"observedAt": probe["observed_at_utc"],
|
|
}
|
|
)
|
|
)
|
|
return
|
|
except (OSError, ValueError):
|
|
seen.add("query-unavailable")
|
|
raise RuntimeError("Automatic telemetry recovery not accepted within 90 seconds")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|