feat(simulation): add Gaussian UGV runtime pipeline

This commit is contained in:
DCCONSTRUCTIONS
2026-08-28 11:50:47 +03:00
parent c4c2392c79
commit 722e60e5ef
21 changed files with 4685 additions and 628 deletions
+397 -70
View File
@@ -4,11 +4,13 @@ from __future__ import annotations
import hashlib
import json
import math
import os
import re
import shutil
import threading
import time
from collections import deque
from collections.abc import Callable
from contextlib import suppress
from pathlib import Path
@@ -28,10 +30,13 @@ from k1link.simulation.gaussian_pipeline_gateway import (
PROJECT_SCHEMA: Final = "missioncore.simulation-project/v1"
WORLD_MANIFEST_SCHEMA: Final = "missioncore.simulation-world-manifest/v1"
VIEWER_SETTINGS_SCHEMA: Final = "missioncore.simulation-viewer-settings/v1"
VIEWER_SETTINGS_SCHEMA: Final = "missioncore.simulation-viewer-settings/v3"
PROJECT_ID_PATTERN: Final = re.compile(r"^sim-[a-f0-9]{32}$")
SOURCE_FILE_ID_PATTERN: Final = re.compile(r"^src-[0-9]{5}-[a-f0-9]{8}$")
PROVIDER_JOB_ID_PATTERN: Final = re.compile(r"^gsp-[0-9]{14}-[a-f0-9]{8}$")
PROVIDER_TIMESTAMP_PATTERN: Final = re.compile(
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z$"
)
MAX_SOURCE_FILES: Final = 10_000
MAX_SOURCE_BYTES: Final = 16 * 1024 * 1024 * 1024
MAX_UPLOAD_CHUNK_BYTES: Final = 16 * 1024 * 1024
@@ -64,6 +69,10 @@ class SimulationProjectConflictError(SimulationProjectError):
"""The requested transition conflicts with current project state."""
class _SimulationProcessingCancelled(RuntimeError):
"""Internal cooperative stop after an operator deletes an active project."""
class SimulationProjectStore:
def __init__(self, data_dir: Path) -> None:
self.root = data_dir.expanduser().resolve() / "simulation-worlds"
@@ -105,13 +114,15 @@ class SimulationProjectStore:
raise SimulationProjectError("simulation source paths must be unique")
logical_paths.add(logical_path)
total_bytes += byte_length
normalized.append({
"file_id": f"src-{index:05d}-{uuid4().hex[:8]}",
"logical_path": logical_path,
"byte_length": byte_length,
"uploaded_bytes": 0,
"sha256": None,
})
normalized.append(
{
"file_id": f"src-{index:05d}-{uuid4().hex[:8]}",
"logical_path": logical_path,
"byte_length": byte_length,
"uploaded_bytes": 0,
"sha256": None,
}
)
if total_bytes > MAX_SOURCE_BYTES:
raise SimulationProjectError("simulation source exceeds byte admission")
if source_kind == "archive":
@@ -141,7 +152,9 @@ class SimulationProjectStore:
"provider": {
"provider_id": "gaussian-pipeline",
"job_id": None,
"job_created_at_utc": None,
"state": None,
"state_started_at_utc": None,
"progress": None,
"runtime": None,
},
@@ -269,7 +282,9 @@ class SimulationProjectStore:
document["provider"] = {
"provider_id": "gaussian-pipeline",
"job_id": None,
"job_created_at_utc": None,
"state": None,
"state_started_at_utc": None,
"progress": None,
"runtime": None,
}
@@ -286,7 +301,9 @@ class SimulationProjectStore:
*,
status: str,
provider_job_id: str | None = None,
provider_job_created_at_utc: str | None = None,
provider_state: str | None = None,
provider_state_started_at_utc: str | None = None,
progress: object = None,
bundle_sha256: str | None = None,
) -> dict[str, Any]:
@@ -299,8 +316,26 @@ class SimulationProjectStore:
if PROVIDER_JOB_ID_PATTERN.fullmatch(provider_job_id) is None:
raise SimulationProjectError("simulation provider job id is invalid")
document["provider"]["job_id"] = provider_job_id
if provider_job_created_at_utc is not None:
document["provider"]["job_created_at_utc"] = _provider_timestamp(
provider_job_created_at_utc,
"job creation",
)
if provider_state is not None:
previous_state = document["provider"].get("state")
document["provider"]["state"] = provider_state
if (
previous_state != provider_state
or document["provider"].get("state_started_at_utc") is None
):
document["provider"]["state_started_at_utc"] = (
_provider_timestamp(
provider_state_started_at_utc,
"state start",
)
if provider_state_started_at_utc is not None
else utc_now_iso()
)
if progress is not None:
document["provider"]["progress"] = progress
if bundle_sha256 is not None:
@@ -362,13 +397,15 @@ class SimulationProjectStore:
raise SimulationProjectNotFoundError("simulation artifact is unavailable")
return target, descriptor
def delete(self, project_id: str) -> None:
def delete(self, project_id: str, *, allow_active: bool = False) -> None:
with self._lock:
document = self._read(project_id)
if document["status"] not in TERMINAL_STATES and document["status"] != "uploading":
raise SimulationProjectConflictError(
"active simulation project cannot be deleted"
)
if (
not allow_active
and document["status"] not in TERMINAL_STATES
and document["status"] != "uploading"
):
raise SimulationProjectConflictError("active simulation project cannot be deleted")
shutil.rmtree(self._project_root(project_id))
def source_root(self, project_id: str) -> Path:
@@ -391,7 +428,11 @@ class SimulationProjectStore:
or document.get("project_id") != project_id
):
raise SimulationProjectError("persisted simulation project identity is invalid")
document.setdefault("viewer_settings", _default_viewer_settings())
document["viewer_settings"] = _migrate_viewer_settings(document.get("viewer_settings"))
provider = document.get("provider")
if isinstance(provider, dict):
provider.setdefault("job_created_at_utc", None)
provider.setdefault("state_started_at_utc", None)
return document
def _write(self, document: dict[str, Any]) -> None:
@@ -425,21 +466,67 @@ class SimulationProjectService:
) -> None:
self.store = store
self.provider_factory = provider_factory
self._condition = threading.Condition()
self._queue: deque[str] = deque()
self._queued_ids: set[str] = set()
self._active_project_id: str | None = None
self._worker: threading.Thread | None = None
self._cancel_events: dict[str, threading.Event] = {}
self._inflight_jobs: dict[str, str] = {}
def recover_pending(self) -> int:
pending = [
project for project in self.store.list()
if project.get("status") in ACTIVE_STATES
]
pending = sorted(
[project for project in self.store.list() if project.get("status") in ACTIVE_STATES],
key=lambda project: (
str(project.get("created_at_utc")),
str(project.get("project_id")),
),
)
for project in pending:
threading.Thread(
target=self.process,
args=(str(project["project_id"]),),
name=f"simulation-recovery-{str(project['project_id'])[-8:]}",
daemon=True,
).start()
self.enqueue(str(project["project_id"]))
return len(pending)
def enqueue(self, project_id: str) -> None:
project = self.store.get(project_id)
if project.get("status") not in ACTIVE_STATES:
raise SimulationProjectConflictError(
"simulation project is not ready for queued processing"
)
with self._condition:
if project_id in self._queued_ids or project_id == self._active_project_id:
return
event = self._cancel_events.setdefault(project_id, threading.Event())
event.clear()
self._queue.append(project_id)
self._queued_ids.add(project_id)
if self._worker is None or not self._worker.is_alive():
self._worker = threading.Thread(
target=self._run_queue,
name="simulation-build-queue",
daemon=True,
)
self._worker.start()
self._condition.notify_all()
def _run_queue(self) -> None:
while True:
with self._condition:
if not self._queue:
self._worker = None
self._condition.notify_all()
return
project_id = self._queue.popleft()
self._queued_ids.discard(project_id)
self._active_project_id = project_id
try:
self.process(project_id)
finally:
with self._condition:
self._active_project_id = None
self._inflight_jobs.pop(project_id, None)
self._cancel_events.pop(project_id, None)
self._condition.notify_all()
def begin_build(self, project_id: str) -> dict[str, Any]:
project = self.store.get(project_id)
if project.get("status") == "failed":
@@ -460,15 +547,20 @@ class SimulationProjectService:
finally:
provider.close()
current_state = job.get("state")
if (
isinstance(current_state, str)
and current_state in PROVIDER_JOB_STATES - {"failed"}
):
if isinstance(current_state, str) and current_state in PROVIDER_JOB_STATES - {
"failed"
}:
return self.store.update_processing(
project_id,
status="processing",
provider_job_id=job_id,
provider_job_created_at_utc=_optional_provider_timestamp(
job.get("created_at_utc")
),
provider_state=current_state,
provider_state_started_at_utc=_optional_provider_timestamp(
job.get("updated_at_utc")
),
progress=job.get("progress"),
)
if project.get("status") in {"failed", "ready"}:
@@ -488,14 +580,17 @@ class SimulationProjectService:
def process(self, project_id: str) -> None:
provider: GaussianPipelineGateway | None = None
try:
self._raise_if_cancelled(project_id)
project = self.store.get(project_id)
provider = self.provider_factory()
if provider is None:
raise SimulationProjectError("Gaussian Pipeline не настроен.")
_retry_provider_unavailable(provider.capabilities)
self._raise_if_cancelled(project_id)
existing_job_id = project["provider"].get("job_id")
if isinstance(existing_job_id, str):
job_id = existing_job_id
self._register_inflight_job(project_id, job_id, provider)
else:
if project["status"] != "queued":
raise SimulationProjectError(
@@ -522,25 +617,33 @@ class SimulationProjectService:
"outputs": {
"preview_sog": True,
"streamed_sog": True,
"collision": True,
"collision": False,
},
"preview_lod": "coarsest",
"collision_profile": _collision_profile(str(project["scene_type"])),
"collision_profile": None,
}
submitted = provider.submit_build(request)
job_id = submitted.get("job_id")
if not isinstance(job_id, str):
raise SimulationProjectError("Gaussian Pipeline не вернул job id.")
self._register_inflight_job(project_id, job_id, provider)
self.store.update_processing(
project_id,
status="processing",
provider_job_id=job_id,
provider_job_created_at_utc=_optional_provider_timestamp(
submitted.get("created_at_utc")
),
provider_state=str(submitted.get("state") or "queued"),
provider_state_started_at_utc=_optional_provider_timestamp(
submitted.get("updated_at_utc")
),
progress=submitted.get("progress"),
bundle_sha256=source.bundle_sha256,
)
deadline = time.monotonic() + PROVIDER_POLL_TIMEOUT_SECONDS
while True:
self._raise_if_cancelled(project_id)
if time.monotonic() >= deadline:
raise SimulationProjectError(
"Gaussian Pipeline превысил лимит ожидания сборки."
@@ -554,7 +657,13 @@ class SimulationProjectService:
self.store.update_processing(
project_id,
status="processing",
provider_job_created_at_utc=_optional_provider_timestamp(
job.get("created_at_utc")
),
provider_state=str(state or "unknown"),
provider_state_started_at_utc=_optional_provider_timestamp(
job.get("updated_at_utc")
),
progress=job.get("progress"),
)
if state == "failed":
@@ -572,9 +681,11 @@ class SimulationProjectService:
provider_state="ready",
)
result = _retry_provider_unavailable(lambda: provider.get_result(job_id))
self._raise_if_cancelled(project_id)
artifacts = _artifact_descriptors(result.get("artifacts"))
artifacts_root = self.store.artifacts_root(project_id)
for descriptor in artifacts:
self._raise_if_cancelled(project_id)
_retry_provider_unavailable(
lambda descriptor=descriptor: provider.download_artifact(
job_id,
@@ -592,27 +703,78 @@ class SimulationProjectService:
artifacts=artifacts,
world_manifest=world_manifest,
)
except _SimulationProcessingCancelled:
pass
except (GaussianPipelineGatewayError, SimulationProjectError, OSError) as exc:
with suppress(SimulationProjectError):
self.store.fail(project_id, str(exc))
finally:
with self._condition:
self._inflight_jobs.pop(project_id, None)
if provider is not None:
provider.close()
def delete(self, project_id: str) -> None:
project = self.store.get(project_id)
job_id = project["provider"].get("job_id")
with self._condition:
cancel = self._cancel_events.setdefault(project_id, threading.Event())
cancel.set()
if project_id in self._queued_ids:
self._queue = deque(item for item in self._queue if item != project_id)
self._queued_ids.discard(project_id)
job_id = self._inflight_jobs.pop(
project_id,
project["provider"].get("job_id"),
)
self._condition.notify_all()
if isinstance(job_id, str):
provider = self.provider_factory()
if provider is None:
with suppress(SimulationProjectError):
self.store.fail(
project_id,
"Gaussian Pipeline недоступен для удаления серверных артефактов.",
)
raise SimulationProjectConflictError(
"Gaussian Pipeline недоступен для удаления серверных артефактов."
)
try:
provider.delete_job(job_id)
except GaussianPipelineGatewayError:
with suppress(SimulationProjectError):
self.store.fail(
project_id,
"Не удалось удалить активную сборку на Worker 006.",
)
raise
finally:
provider.close()
self.store.delete(project_id)
self.store.delete(project_id, allow_active=True)
with self._condition:
self._cancel_events.pop(project_id, None)
self._condition.notify_all()
def _raise_if_cancelled(self, project_id: str) -> None:
with self._condition:
event = self._cancel_events.get(project_id)
if event is not None and event.is_set():
raise _SimulationProcessingCancelled(project_id)
def _register_inflight_job(
self,
project_id: str,
job_id: str,
provider: GaussianPipelineGateway,
) -> None:
with self._condition:
event = self._cancel_events.get(project_id)
cancelled = event is not None and event.is_set()
if not cancelled:
self._inflight_jobs[project_id] = job_id
if cancelled:
with suppress(GaussianPipelineGatewayError):
provider.delete_job(job_id)
raise _SimulationProcessingCancelled(project_id)
def _retry_provider_unavailable(operation: Callable[[], _T]) -> _T:
@@ -650,13 +812,15 @@ def _artifact_descriptors(value: object) -> list[dict[str, Any]]:
or not isinstance(role, str)
):
raise SimulationProjectError("Gaussian artifact fields are invalid")
descriptors.append({
"role": role,
"logical_path": logical_path,
"media_type": media_type,
"sha256": sha256,
"byte_length": byte_length,
})
descriptors.append(
{
"role": role,
"logical_path": logical_path,
"media_type": media_type,
"sha256": sha256,
"byte_length": byte_length,
}
)
return descriptors
@@ -666,16 +830,12 @@ def _world_manifest(project_id: str, artifacts: list[dict[str, Any]]) -> dict[st
if descriptor is None:
return None
encoded = "/".join(
quote(part, safe="")
for part in str(descriptor["logical_path"]).split("/")
quote(part, safe="") for part in str(descriptor["logical_path"]).split("/")
)
return f"/api/v1/simulation-worlds/projects/{project_id}/artifacts/{encoded}"
collision_mesh_url = (
url_for("rover-collision-mesh")
or url_for("rover-terrain-mesh")
or url_for("collision-mesh")
)
rover_collision_mesh_url = url_for("rover-collision-mesh") or url_for("rover-terrain-mesh")
collision_mesh_url = rover_collision_mesh_url or url_for("collision-mesh")
return {
"schema_version": WORLD_MANIFEST_SCHEMA,
"project_id": project_id,
@@ -689,39 +849,53 @@ def _world_manifest(project_id: str, artifacts: list[dict[str, Any]]) -> dict[st
},
"transforms": {
"world_from_visual": [1, 0, 0, 0, 0, -1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1],
"world_from_collision": [-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1],
"world_from_collision": (
[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]
if rover_collision_mesh_url is not None
else [-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1]
),
},
}
def _collision_profile(scene_type: str) -> dict[str, Any]:
"""Build the portable walkable-volume contract around the scanner origin."""
return {
"scene_type": scene_type,
"seed_position": [0, 1, 0],
"capsule_height": 1.6,
"capsule_radius": 0.2,
"voxel_size": 0.2 if scene_type == "outdoor" else 0.1,
"mesh_shape": "smooth",
}
def _default_viewer_settings() -> dict[str, Any]:
return {
"schema_version": VIEWER_SETTINGS_SCHEMA,
"quality": "high",
"visual": {"inverted": True, "axis": "x"},
"collision": {"inverted": True, "axis": "y"},
"visual": {"rotation_degrees": {"x": 180.0, "y": 0.0, "z": 0.0}},
"collision": {"rotation_degrees": {"x": 0.0, "y": 0.0, "z": 0.0}},
"camera": {
"invert_horizontal": True,
"invert_vertical": False,
},
"ugv": _default_ugv_settings(),
}
def _default_ugv_settings() -> dict[str, Any]:
return {
"preset_name": "UGV 100 кг",
"mass_kg": 100.0,
"dimensions_m": {
"length": 1.0,
"width": 0.8,
"height": 0.4,
"ground_clearance": 0.15,
},
"max_speed_mps": 1.2,
"max_turn_rate_degrees": 45.0,
"invert_steering": False,
}
def _viewer_settings(value: object) -> dict[str, Any]:
if not isinstance(value, dict) or set(value) != {
"schema_version", "quality", "visual", "collision", "camera"
"schema_version",
"quality",
"visual",
"collision",
"camera",
"ugv",
}:
raise SimulationProjectError("simulation viewer settings contract is invalid")
if value.get("schema_version") != VIEWER_SETTINGS_SCHEMA:
@@ -732,23 +906,104 @@ def _viewer_settings(value: object) -> dict[str, Any]:
def layer(name: str) -> dict[str, Any]:
raw = value.get(name)
if not isinstance(raw, dict) or set(raw) != {"inverted", "axis"}:
if not isinstance(raw, dict) or set(raw) != {"rotation_degrees"}:
raise SimulationProjectError(f"simulation {name} transform is invalid")
inverted = raw.get("inverted")
axis = raw.get("axis")
if not isinstance(inverted, bool) or axis not in {"x", "y", "z"}:
raw_rotation = raw.get("rotation_degrees")
if not isinstance(raw_rotation, dict) or set(raw_rotation) != {"x", "y", "z"}:
raise SimulationProjectError(f"simulation {name} transform is invalid")
return {"inverted": inverted, "axis": axis}
rotation: dict[str, float] = {}
for axis in ("x", "y", "z"):
angle = raw_rotation.get(axis)
if (
not isinstance(angle, (int, float))
or isinstance(angle, bool)
or not math.isfinite(angle)
or angle < -360.0
or angle > 360.0
):
raise SimulationProjectError(f"simulation {name} transform is invalid")
rotation[axis] = float(angle)
return {"rotation_degrees": rotation}
camera = value.get("camera")
if not isinstance(camera, dict) or set(camera) != {
"invert_horizontal", "invert_vertical"
}:
if not isinstance(camera, dict) or set(camera) != {"invert_horizontal", "invert_vertical"}:
raise SimulationProjectError("simulation camera settings are invalid")
horizontal = camera.get("invert_horizontal")
vertical = camera.get("invert_vertical")
if not isinstance(horizontal, bool) or not isinstance(vertical, bool):
raise SimulationProjectError("simulation camera settings are invalid")
ugv = value.get("ugv")
if not isinstance(ugv, dict) or set(ugv) != {
"preset_name",
"mass_kg",
"dimensions_m",
"max_speed_mps",
"max_turn_rate_degrees",
"invert_steering",
}:
raise SimulationProjectError("simulation UGV settings are invalid")
preset_name = " ".join(str(ugv.get("preset_name", "")).split())
if not 1 <= len(preset_name) <= 80:
raise SimulationProjectError("simulation UGV preset name is invalid")
dimensions = ugv.get("dimensions_m")
if not isinstance(dimensions, dict) or set(dimensions) != {
"length",
"width",
"height",
"ground_clearance",
}:
raise SimulationProjectError("simulation UGV dimensions are invalid")
def bounded_number(
raw: object,
*,
minimum: float,
maximum: float,
label: str,
) -> float:
if (
not isinstance(raw, (int, float))
or isinstance(raw, bool)
or not math.isfinite(raw)
or raw < minimum
or raw > maximum
):
raise SimulationProjectError(f"simulation UGV {label} is invalid")
return float(raw)
mass_kg = bounded_number(
ugv.get("mass_kg"), minimum=0.01, maximum=1_000_000_000, label="mass"
)
length = bounded_number(
dimensions.get("length"), minimum=0.01, maximum=1_000_000_000, label="length"
)
width = bounded_number(
dimensions.get("width"), minimum=0.01, maximum=1_000_000_000, label="width"
)
height = bounded_number(
dimensions.get("height"), minimum=0.07, maximum=1_000_000_000, label="height"
)
ground_clearance = bounded_number(
dimensions.get("ground_clearance"),
minimum=0.01,
maximum=1_000_000_000,
label="ground clearance",
)
if ground_clearance >= height - 0.05:
raise SimulationProjectError("simulation UGV ground clearance exceeds its height")
max_speed_mps = bounded_number(
ugv.get("max_speed_mps"), minimum=0, maximum=1_000_000_000, label="maximum speed"
)
max_turn_rate_degrees = bounded_number(
ugv.get("max_turn_rate_degrees"),
minimum=0,
maximum=1_000_000_000,
label="maximum turn rate",
)
invert_steering = ugv.get("invert_steering")
if not isinstance(invert_steering, bool):
raise SimulationProjectError("simulation UGV steering inversion is invalid")
return {
"schema_version": VIEWER_SETTINGS_SCHEMA,
"quality": quality,
@@ -758,9 +1013,69 @@ def _viewer_settings(value: object) -> dict[str, Any]:
"invert_horizontal": horizontal,
"invert_vertical": vertical,
},
"ugv": {
"preset_name": preset_name,
"mass_kg": mass_kg,
"dimensions_m": {
"length": length,
"width": width,
"height": height,
"ground_clearance": ground_clearance,
},
"max_speed_mps": max_speed_mps,
"max_turn_rate_degrees": max_turn_rate_degrees,
"invert_steering": invert_steering,
},
}
def _migrate_viewer_settings(value: object) -> dict[str, Any]:
if not isinstance(value, dict):
return _default_viewer_settings()
if value.get("schema_version") == VIEWER_SETTINGS_SCHEMA:
return _viewer_settings(value)
if value.get("schema_version") == "missioncore.simulation-viewer-settings/v2":
migrated_v2 = {
**value,
"schema_version": VIEWER_SETTINGS_SCHEMA,
"ugv": _default_ugv_settings(),
}
try:
return _viewer_settings(migrated_v2)
except SimulationProjectError:
return _default_viewer_settings()
if value.get("schema_version") != "missioncore.simulation-viewer-settings/v1":
return _default_viewer_settings()
def legacy_layer(name: str) -> dict[str, Any]:
raw = value.get(name)
rotation = {"x": 0.0, "y": 0.0, "z": 0.0}
if isinstance(raw, dict) and raw.get("inverted") is True:
axis = raw.get("axis")
if axis in rotation:
rotation[axis] = 180.0
return {"rotation_degrees": rotation}
camera = value.get("camera")
if not isinstance(camera, dict):
camera = {}
migrated = {
"schema_version": VIEWER_SETTINGS_SCHEMA,
"quality": value.get("quality", "high"),
"visual": legacy_layer("visual"),
"collision": legacy_layer("collision"),
"camera": {
"invert_horizontal": camera.get("invert_horizontal", True),
"invert_vertical": camera.get("invert_vertical", False),
},
"ugv": _default_ugv_settings(),
}
try:
return _viewer_settings(migrated)
except SimulationProjectError:
return _default_viewer_settings()
def _project_name(value: str) -> str:
normalized = " ".join(value.split())
if not 1 <= len(normalized) <= 120:
@@ -768,6 +1083,18 @@ def _project_name(value: str) -> str:
return normalized
def _provider_timestamp(value: str, label: str) -> str:
if PROVIDER_TIMESTAMP_PATTERN.fullmatch(value) is None:
raise SimulationProjectError(f"simulation provider {label} timestamp is invalid")
return value
def _optional_provider_timestamp(value: object) -> str | None:
if isinstance(value, str) and PROVIDER_TIMESTAMP_PATTERN.fullmatch(value) is not None:
return value
return None
def _logical_path(value: object) -> str:
if not isinstance(value, str) or not value or len(value) > 1024:
raise SimulationProjectError("simulation logical path is invalid")