feat(simulation): add Worker AI polygon runtime and terrain navigation
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,377 @@
|
||||
"""Persistent host coordinator. Control/telemetry I/O never runs in the simulator.
|
||||
|
||||
The Windows task owns the coordinator; a Job Object fences its native children.
|
||||
Closing the viewer or the private control tunnel cannot stop the physics clock.
|
||||
Runs are bounded locally; reconnect reconciles the same identity, without replay.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from core_client import CoreClient
|
||||
from local_state import StateChannel, read_json, write_json
|
||||
from model_stack import ROOT, ModelStack, docker, sha256
|
||||
from worker import terminate_episode
|
||||
|
||||
|
||||
class NativeJob:
|
||||
"""Windows kills all native descendants if the coordinator unexpectedly exits."""
|
||||
|
||||
def __init__(self):
|
||||
from ctypes import wintypes
|
||||
|
||||
self.kernel = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
self.kernel.CreateJobObjectW.restype = wintypes.HANDLE
|
||||
self.kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR]
|
||||
self.kernel.SetInformationJobObject.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
ctypes.c_int,
|
||||
ctypes.c_void_p,
|
||||
wintypes.DWORD,
|
||||
]
|
||||
self.kernel.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
|
||||
self.kernel.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
self.handle = self.kernel.CreateJobObjectW(None, None)
|
||||
# JOBOBJECT_EXTENDED_LIMIT_INFORMATION: 144 bytes on 64-bit Windows;
|
||||
# BasicLimitInformation.LimitFlags is DWORD at offset 16.
|
||||
limits = ctypes.create_string_buffer(144)
|
||||
ctypes.c_uint32.from_buffer(limits, 16).value = 0x2000 # KILL_ON_JOB_CLOSE
|
||||
if not self.handle or not self.kernel.SetInformationJobObject(self.handle, 9, limits, 144):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
|
||||
def assign(self, child):
|
||||
if not self.kernel.AssignProcessToJobObject(self.handle, int(child._handle)):
|
||||
terminate_episode(child)
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
|
||||
def close(self):
|
||||
if self.handle:
|
||||
self.kernel.CloseHandle(self.handle)
|
||||
self.handle = None
|
||||
|
||||
|
||||
def acquire_service_lock(path):
|
||||
import msvcrt
|
||||
|
||||
stream = path.open("a+b")
|
||||
stream.seek(0)
|
||||
if not stream.read(1):
|
||||
stream.write(b"0")
|
||||
stream.flush()
|
||||
stream.seek(0)
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
return stream
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--core", default="http://127.0.0.1:18080")
|
||||
parser.add_argument("--token-file", type=Path, required=True)
|
||||
parser.add_argument("--state", type=Path, required=True)
|
||||
parser.add_argument("--isaac", type=Path, required=True)
|
||||
parser.add_argument("--stream-address", required=True)
|
||||
args = parser.parse_args()
|
||||
args.state.mkdir(parents=True, exist_ok=True)
|
||||
lock = acquire_service_lock(args.state / "realtime-service.lock")
|
||||
logging.basicConfig(
|
||||
filename=args.state / "realtime-service.log",
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
identity_path = args.state / "realtime-identity.json"
|
||||
identity = read_json(identity_path)
|
||||
release = sha256(ROOT / "realtime_worker.py")
|
||||
active_path = args.state / "active.json"
|
||||
if identity is None or (identity["release"] != release and not active_path.exists()):
|
||||
identity = {"instance_id": uuid4().hex, "release": release}
|
||||
write_json(identity_path, identity)
|
||||
instance = identity["instance_id"]
|
||||
client = CoreClient(args.core, args.token_file, instance)
|
||||
stack = ModelStack()
|
||||
stack.preflight()
|
||||
sources = {
|
||||
"worker": release,
|
||||
"scene": hashlib.sha256(
|
||||
"".join(
|
||||
sha256(ROOT / n)
|
||||
for n in (
|
||||
"run_realtime.py",
|
||||
"motion_control.py",
|
||||
"realtime_ai.py",
|
||||
"local_state.py",
|
||||
"terrain.py",
|
||||
"spawn_clearance.py",
|
||||
"prepare_terrain.py",
|
||||
"navigation_client.py",
|
||||
"navigation/server.py",
|
||||
"navigation/footprint.py",
|
||||
"navigation/fastdds.xml",
|
||||
"navigation/Prepare-Terrain.ps1",
|
||||
)
|
||||
).encode()
|
||||
+ "".join(
|
||||
sha256(ROOT.parents[1] / "src/k1link/simulation/ai_polygon" / n)
|
||||
for n in (
|
||||
"inference.py",
|
||||
"policy.py",
|
||||
"contracts.py",
|
||||
"composition.py",
|
||||
"terrain_contract.py",
|
||||
)
|
||||
).encode()
|
||||
+ "".join(
|
||||
sha256(ROOT.parents[1] / "src/k1link" / n)
|
||||
for n in (
|
||||
"artifacts.py",
|
||||
"observatory/__init__.py",
|
||||
"observatory/modular_composition.py",
|
||||
"simulation/__init__.py",
|
||||
)
|
||||
).encode()
|
||||
).hexdigest(),
|
||||
"models": hashlib.sha256(
|
||||
"".join(
|
||||
sha256(ROOT / name)
|
||||
for name in (
|
||||
"models.worker-006.json",
|
||||
"model_stack.py",
|
||||
"compose.models.yaml",
|
||||
"ddrnet_server.py",
|
||||
"segformer/server.py",
|
||||
)
|
||||
).encode()
|
||||
).hexdigest(),
|
||||
"robot": sha256(ROOT / "rover_profile.py"),
|
||||
}
|
||||
hello = {
|
||||
"worker_id": "worker-006-ai-polygon",
|
||||
"instance_id": instance,
|
||||
"runtime": "isaac-sim-6.1",
|
||||
"execution_modes": ["realtime"],
|
||||
"model_ids": [m["id"] for m in stack.profile["models"]]
|
||||
+ [stack.profile["navigation"]["id"]],
|
||||
"profile_sha256": hashlib.sha256(json.dumps(sources, sort_keys=True).encode()).hexdigest(),
|
||||
"runtime_sources": sources,
|
||||
"stream": {
|
||||
"server": args.stream_address,
|
||||
"signaling_port": 49100,
|
||||
"media_port": 47998,
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 30,
|
||||
},
|
||||
}
|
||||
write_json(
|
||||
args.state / "service-status.json",
|
||||
{"pid": os.getpid(), "instance_id": instance, "hello": hello},
|
||||
)
|
||||
|
||||
def connect():
|
||||
while True:
|
||||
try:
|
||||
client.request("/worker/register", hello)
|
||||
return
|
||||
except Exception as exc:
|
||||
logging.warning("Control unavailable: %s", type(exc).__name__)
|
||||
time.sleep(3)
|
||||
|
||||
# A previous process died. Its Job Object has killed native children.
|
||||
# Reconcile only exact persisted container IDs; never infer release from a lease.
|
||||
orphan = read_json(active_path)
|
||||
if orphan:
|
||||
if orphan.get("instance_id") != instance:
|
||||
raise RuntimeError("Reconciliation identity mismatch")
|
||||
for cid in orphan.get("model_container_ids", []):
|
||||
record = docker("container", "inspect", cid, check=False)
|
||||
if record.returncode == 0:
|
||||
data = json.loads(record.stdout)[0]
|
||||
if data["Config"]["Labels"].get("com.nodedc.stack") != "ai-polygon":
|
||||
raise RuntimeError("Reconciliation resource owner changed")
|
||||
stack.ids.append(cid)
|
||||
stack.stop()
|
||||
connect()
|
||||
client.request(
|
||||
"/worker/runs/" + orphan["run_id"] + "/finish",
|
||||
{
|
||||
"instance_id": instance,
|
||||
"outcome": "failed",
|
||||
"resources_released": True,
|
||||
"message": "Worker перезапустился. Прогон завершён без повторного запуска.",
|
||||
},
|
||||
)
|
||||
active_path.unlink()
|
||||
connect()
|
||||
while True:
|
||||
try:
|
||||
polled = client.request("/worker/poll", {"instance_id": instance})
|
||||
except Exception:
|
||||
connect()
|
||||
continue
|
||||
if polled["action"] == "idle":
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
run = polled["run"]
|
||||
if run.get("clock") != "realtime":
|
||||
raise RuntimeError("A realtime worker cannot execute lockstep work")
|
||||
episode = args.state / run["run_id"]
|
||||
if episode.exists():
|
||||
raise RuntimeError("An existing episode must never be replayed")
|
||||
episode.mkdir()
|
||||
run_file = episode / "run.json"
|
||||
write_json(run_file, run)
|
||||
channel = StateChannel(episode)
|
||||
channel.write("control", run)
|
||||
active = {"run_id": run["run_id"], "instance_id": instance, "model_container_ids": []}
|
||||
write_json(active_path, active)
|
||||
child = job = starter = None
|
||||
cancelled = threading.Event()
|
||||
model_failure = []
|
||||
result = {"outcome": "failed", "message": "Прогон прерван. Журнал сохранён на Worker."}
|
||||
try:
|
||||
cache = args.state / "worlds"
|
||||
cache.mkdir(exist_ok=True)
|
||||
source = cache / (run["world"]["sha256"] + ".ply")
|
||||
client.download(run["world"], source)
|
||||
client.request("/worker/runs/" + run["run_id"] + "/progress", {"phase": "scene"})
|
||||
from prepare_terrain import prepare_terrain
|
||||
|
||||
job = NativeJob()
|
||||
|
||||
def preparation_pulse(run=run):
|
||||
desired = client.request(
|
||||
"/worker/poll", {"instance_id": instance, "run_id": run["run_id"]}
|
||||
)
|
||||
if desired["run"]["control"] == "stop":
|
||||
raise InterruptedError("Terrain preparation cancelled")
|
||||
|
||||
terrain_path = prepare_terrain(
|
||||
args.state.parent, episode, run["world"], job, preparation_pulse
|
||||
)
|
||||
run["terrain_manifest"] = str(terrain_path)
|
||||
write_json(run_file, run)
|
||||
with (episode / "isaac.log").open("wb") as output:
|
||||
child = subprocess.Popen(
|
||||
[
|
||||
str(args.isaac / "python.bat"),
|
||||
str(ROOT / "run_realtime.py"),
|
||||
"--run",
|
||||
str(run_file),
|
||||
"--source",
|
||||
str(source),
|
||||
"--stream-address",
|
||||
args.stream_address,
|
||||
],
|
||||
stdout=output,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
job.assign(child)
|
||||
|
||||
def acquired(ids, active=active):
|
||||
active["model_container_ids"] = ids
|
||||
write_json(active_path, active)
|
||||
|
||||
def start_models(
|
||||
cancelled=cancelled, model_failure=model_failure, acquired=acquired, run=run
|
||||
):
|
||||
try:
|
||||
stack.start(
|
||||
cancelled=cancelled.is_set,
|
||||
on_acquired=acquired,
|
||||
selection=run["request"].get("composition"),
|
||||
)
|
||||
except InterruptedError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
model_failure.append(type(exc).__name__)
|
||||
logging.exception("Model startup failed")
|
||||
|
||||
stop_deadline = None
|
||||
deadline = time.monotonic() + 300 + run["request"]["duration_seconds"]
|
||||
last_snapshot = -1
|
||||
registered = True
|
||||
while child.poll() is None:
|
||||
now = time.monotonic()
|
||||
if now >= deadline or (stop_deadline and now >= stop_deadline):
|
||||
terminate_episode(child)
|
||||
break
|
||||
if model_failure:
|
||||
raise RuntimeError("Local model startup failed")
|
||||
try:
|
||||
if not registered:
|
||||
client.request("/worker/register", hello)
|
||||
registered = True
|
||||
polled = client.request(
|
||||
"/worker/poll", {"instance_id": instance, "run_id": run["run_id"]}
|
||||
)
|
||||
desired = polled["run"]
|
||||
channel.write("control", desired)
|
||||
if desired["control"] == "stop":
|
||||
cancelled.set()
|
||||
stop_deadline = stop_deadline or now + 15
|
||||
if desired["control"] == "play" and starter is None:
|
||||
starter = threading.Thread(
|
||||
target=start_models, name="polygon-model-start", daemon=True
|
||||
)
|
||||
starter.start()
|
||||
snapshot = channel.read("snapshot")
|
||||
if snapshot and snapshot["sequence"] > last_snapshot:
|
||||
client.request("/worker/runs/" + run["run_id"] + "/snapshot", snapshot)
|
||||
last_snapshot = snapshot["sequence"]
|
||||
except Exception as exc:
|
||||
registered = False
|
||||
logging.warning(
|
||||
"Episode control/telemetry disconnected: %s", type(exc).__name__
|
||||
)
|
||||
time.sleep(0.25)
|
||||
result = read_json(episode / "result.json", result)
|
||||
if cancelled.is_set():
|
||||
result = {"outcome": "stopped", "message": "Движение и inference остановлены."}
|
||||
except InterruptedError:
|
||||
result = {"outcome": "stopped", "message": "Подготовка симуляции остановлена."}
|
||||
except Exception:
|
||||
logging.exception("Episode failed")
|
||||
finally:
|
||||
channel.close()
|
||||
cancelled.set()
|
||||
if child is not None and child.poll() is None:
|
||||
terminate_episode(child)
|
||||
if job is not None:
|
||||
job.close()
|
||||
if starter is not None:
|
||||
starter.join(timeout=150)
|
||||
if starter.is_alive():
|
||||
raise RuntimeError("Model startup has not released ownership")
|
||||
stack.stop()
|
||||
active["resources_released"] = True
|
||||
write_json(active_path, active)
|
||||
write_json(episode / "worker-result.json", result)
|
||||
while True:
|
||||
try:
|
||||
client.request("/worker/register", hello)
|
||||
client.request(
|
||||
"/worker/runs/" + run["run_id"] + "/finish",
|
||||
{**result, "instance_id": instance, "resources_released": True},
|
||||
)
|
||||
active_path.unlink()
|
||||
break
|
||||
except Exception:
|
||||
logging.warning("Finish pending reconciliation")
|
||||
time.sleep(3)
|
||||
lock.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception:
|
||||
logging.exception("Simulation coordinator failed")
|
||||
raise
|
||||
Reference in New Issue
Block a user