"""Persistent host coordinator. Control/telemetry I/O never runs in the simulator. The Windows task owns the coordinator; a Job Object fences its native children. Closing the viewer or the private control tunnel cannot stop the physics clock. Runs are bounded locally; reconnect reconciles the same identity, without replay. """ import argparse import ctypes import hashlib import json import logging import os import subprocess import threading import time from pathlib import Path from uuid import uuid4 from core_client import CoreClient from local_state import StateChannel, read_json, write_json from model_stack import ROOT, ModelStack, docker, sha256 from worker import terminate_episode class NativeJob: """Windows kills all native descendants if the coordinator unexpectedly exits.""" def __init__(self): from ctypes import wintypes self.kernel = ctypes.WinDLL("kernel32", use_last_error=True) self.kernel.CreateJobObjectW.restype = wintypes.HANDLE self.kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR] self.kernel.SetInformationJobObject.argtypes = [ wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD, ] self.kernel.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE] self.kernel.CloseHandle.argtypes = [wintypes.HANDLE] self.handle = self.kernel.CreateJobObjectW(None, None) # JOBOBJECT_EXTENDED_LIMIT_INFORMATION: 144 bytes on 64-bit Windows; # BasicLimitInformation.LimitFlags is DWORD at offset 16. limits = ctypes.create_string_buffer(144) ctypes.c_uint32.from_buffer(limits, 16).value = 0x2000 # KILL_ON_JOB_CLOSE if not self.handle or not self.kernel.SetInformationJobObject(self.handle, 9, limits, 144): raise ctypes.WinError(ctypes.get_last_error()) def assign(self, child): if not self.kernel.AssignProcessToJobObject(self.handle, int(child._handle)): terminate_episode(child) raise ctypes.WinError(ctypes.get_last_error()) def close(self): if self.handle: self.kernel.CloseHandle(self.handle) self.handle = None def acquire_service_lock(path): import msvcrt stream = path.open("a+b") stream.seek(0) if not stream.read(1): stream.write(b"0") stream.flush() stream.seek(0) msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) return stream def main(): parser = argparse.ArgumentParser() parser.add_argument("--core", default="http://127.0.0.1:18080") parser.add_argument("--token-file", type=Path, required=True) parser.add_argument("--state", type=Path, required=True) parser.add_argument("--isaac", type=Path, required=True) parser.add_argument("--stream-address", required=True) args = parser.parse_args() args.state.mkdir(parents=True, exist_ok=True) lock = acquire_service_lock(args.state / "realtime-service.lock") logging.basicConfig( filename=args.state / "realtime-service.log", level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", ) identity_path = args.state / "realtime-identity.json" identity = read_json(identity_path) release = sha256(ROOT / "realtime_worker.py") active_path = args.state / "active.json" if identity is None or (identity["release"] != release and not active_path.exists()): identity = {"instance_id": uuid4().hex, "release": release} write_json(identity_path, identity) instance = identity["instance_id"] client = CoreClient(args.core, args.token_file, instance) stack = ModelStack() stack.preflight() sources = { "worker": release, "scene": hashlib.sha256( "".join( sha256(ROOT / n) for n in ( "run_realtime.py", "motion_control.py", "realtime_ai.py", "local_state.py", "terrain.py", "spawn_clearance.py", "prepare_terrain.py", "navigation_client.py", "navigation/server.py", "navigation/footprint.py", "navigation/fastdds.xml", "navigation/Prepare-Terrain.ps1", ) ).encode() + "".join( sha256(ROOT.parents[1] / "src/k1link/simulation/ai_polygon" / n) for n in ( "inference.py", "policy.py", "contracts.py", "composition.py", "terrain_contract.py", ) ).encode() + "".join( sha256(ROOT.parents[1] / "src/k1link" / n) for n in ( "artifacts.py", "observatory/__init__.py", "observatory/modular_composition.py", "simulation/__init__.py", ) ).encode() ).hexdigest(), "models": hashlib.sha256( "".join( sha256(ROOT / name) for name in ( "models.worker-006.json", "model_stack.py", "compose.models.yaml", "ddrnet_server.py", "segformer/server.py", ) ).encode() ).hexdigest(), "robot": sha256(ROOT / "rover_profile.py"), } hello = { "worker_id": "worker-006-ai-polygon", "instance_id": instance, "runtime": "isaac-sim-6.1", "execution_modes": ["realtime"], "model_ids": [m["id"] for m in stack.profile["models"]] + [stack.profile["navigation"]["id"]], "profile_sha256": hashlib.sha256(json.dumps(sources, sort_keys=True).encode()).hexdigest(), "runtime_sources": sources, "stream": { "server": args.stream_address, "signaling_port": 49100, "media_port": 47998, "width": 1280, "height": 720, "fps": 30, }, } write_json( args.state / "service-status.json", {"pid": os.getpid(), "instance_id": instance, "hello": hello}, ) def connect(): while True: try: client.request("/worker/register", hello) return except Exception as exc: logging.warning("Control unavailable: %s", type(exc).__name__) time.sleep(3) # A previous process died. Its Job Object has killed native children. # Reconcile only exact persisted container IDs; never infer release from a lease. orphan = read_json(active_path) if orphan: if orphan.get("instance_id") != instance: raise RuntimeError("Reconciliation identity mismatch") for cid in orphan.get("model_container_ids", []): record = docker("container", "inspect", cid, check=False) if record.returncode == 0: data = json.loads(record.stdout)[0] if data["Config"]["Labels"].get("com.nodedc.stack") != "ai-polygon": raise RuntimeError("Reconciliation resource owner changed") stack.ids.append(cid) stack.stop() connect() client.request( "/worker/runs/" + orphan["run_id"] + "/finish", { "instance_id": instance, "outcome": "failed", "resources_released": True, "message": "Worker перезапустился. Прогон завершён без повторного запуска.", }, ) active_path.unlink() connect() while True: try: polled = client.request("/worker/poll", {"instance_id": instance}) except Exception: connect() continue if polled["action"] == "idle": time.sleep(0.5) continue run = polled["run"] if run.get("clock") != "realtime": raise RuntimeError("A realtime worker cannot execute lockstep work") episode = args.state / run["run_id"] if episode.exists(): raise RuntimeError("An existing episode must never be replayed") episode.mkdir() run_file = episode / "run.json" write_json(run_file, run) channel = StateChannel(episode) channel.write("control", run) active = {"run_id": run["run_id"], "instance_id": instance, "model_container_ids": []} write_json(active_path, active) child = job = starter = None cancelled = threading.Event() model_failure = [] result = {"outcome": "failed", "message": "Прогон прерван. Журнал сохранён на Worker."} try: cache = args.state / "worlds" cache.mkdir(exist_ok=True) source = cache / (run["world"]["sha256"] + ".ply") client.download(run["world"], source) client.request("/worker/runs/" + run["run_id"] + "/progress", {"phase": "scene"}) from prepare_terrain import prepare_terrain job = NativeJob() def preparation_pulse(run=run): desired = client.request( "/worker/poll", {"instance_id": instance, "run_id": run["run_id"]} ) if desired["run"]["control"] == "stop": raise InterruptedError("Terrain preparation cancelled") terrain_path = prepare_terrain( args.state.parent, episode, run["world"], job, preparation_pulse ) run["terrain_manifest"] = str(terrain_path) write_json(run_file, run) with (episode / "isaac.log").open("wb") as output: child = subprocess.Popen( [ str(args.isaac / "python.bat"), str(ROOT / "run_realtime.py"), "--run", str(run_file), "--source", str(source), "--stream-address", args.stream_address, ], stdout=output, stderr=subprocess.STDOUT, ) job.assign(child) def acquired(ids, active=active): active["model_container_ids"] = ids write_json(active_path, active) def start_models( cancelled=cancelled, model_failure=model_failure, acquired=acquired, run=run ): try: stack.start( cancelled=cancelled.is_set, on_acquired=acquired, selection=run["request"].get("composition"), ) except InterruptedError: pass except Exception as exc: model_failure.append(type(exc).__name__) logging.exception("Model startup failed") stop_deadline = None deadline = time.monotonic() + 300 + run["request"]["duration_seconds"] last_snapshot = -1 registered = True while child.poll() is None: now = time.monotonic() if now >= deadline or (stop_deadline and now >= stop_deadline): terminate_episode(child) break if model_failure: raise RuntimeError("Local model startup failed") try: if not registered: client.request("/worker/register", hello) registered = True polled = client.request( "/worker/poll", {"instance_id": instance, "run_id": run["run_id"]} ) desired = polled["run"] channel.write("control", desired) if desired["control"] == "stop": cancelled.set() stop_deadline = stop_deadline or now + 15 if desired["control"] == "play" and starter is None: starter = threading.Thread( target=start_models, name="polygon-model-start", daemon=True ) starter.start() snapshot = channel.read("snapshot") if snapshot and snapshot["sequence"] > last_snapshot: client.request("/worker/runs/" + run["run_id"] + "/snapshot", snapshot) last_snapshot = snapshot["sequence"] except Exception as exc: registered = False logging.warning( "Episode control/telemetry disconnected: %s", type(exc).__name__ ) time.sleep(0.25) result = read_json(episode / "result.json", result) if cancelled.is_set(): result = {"outcome": "stopped", "message": "Движение и inference остановлены."} except InterruptedError: result = {"outcome": "stopped", "message": "Подготовка симуляции остановлена."} except Exception: logging.exception("Episode failed") finally: channel.close() cancelled.set() if child is not None and child.poll() is None: terminate_episode(child) if job is not None: job.close() if starter is not None: starter.join(timeout=150) if starter.is_alive(): raise RuntimeError("Model startup has not released ownership") stack.stop() active["resources_released"] = True write_json(active_path, active) write_json(episode / "worker-result.json", result) while True: try: client.request("/worker/register", hello) client.request( "/worker/runs/" + run["run_id"] + "/finish", {**result, "instance_id": instance, "resources_released": True}, ) active_path.unlink() break except Exception: logging.warning("Finish pending reconciliation") time.sleep(3) lock.close() if __name__ == "__main__": try: main() except Exception: logging.exception("Simulation coordinator failed") raise