feat(system): add compute contour telemetry APIs
This commit is contained in:
@@ -33,6 +33,7 @@ from k1link.sessions import (
|
|||||||
SessionStore,
|
SessionStore,
|
||||||
)
|
)
|
||||||
from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router
|
from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router
|
||||||
|
from k1link.web.compute_contour_api import build_compute_contour_router
|
||||||
from k1link.web.device_plugin_composition import load_installed_device_plugins
|
from k1link.web.device_plugin_composition import load_installed_device_plugins
|
||||||
from k1link.web.e30_engineering_api import build_e30_engineering_router
|
from k1link.web.e30_engineering_api import build_e30_engineering_router
|
||||||
from k1link.web.e30_human_review_api import build_e30_human_review_router
|
from k1link.web.e30_human_review_api import build_e30_human_review_router
|
||||||
@@ -624,6 +625,11 @@ app.include_router(
|
|||||||
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
app.include_router(
|
||||||
|
build_compute_contour_router(
|
||||||
|
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||||
|
|||||||
@@ -0,0 +1,321 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Callable
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final, Literal
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||||
|
|
||||||
|
CONTOUR_SCHEMA: Final = "missioncore.compute-contour/v1"
|
||||||
|
CATALOG_SCHEMA: Final = "missioncore.compute-contour-catalog/v1"
|
||||||
|
INSTALL_SCHEMA: Final = "missioncore.compute-contour-agent-install/v1"
|
||||||
|
CATALOG_FILE_NAME: Final = "compute-contours.json"
|
||||||
|
SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
|
||||||
|
SAFE_NODE_ID: Final = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,126}[A-Za-z0-9])?$")
|
||||||
|
|
||||||
|
RootProvider = Callable[[], Path]
|
||||||
|
TelemetryMode = Literal["agent-mqtt", "legacy-ssh"]
|
||||||
|
ContourPlatform = Literal["windows", "linux", "unknown"]
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now() -> str:
|
||||||
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
|
class StrictModel(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeContour(StrictModel):
|
||||||
|
schema_version: Literal["missioncore.compute-contour/v1"] = CONTOUR_SCHEMA
|
||||||
|
contour_id: str
|
||||||
|
display_name: str = Field(min_length=1, max_length=80)
|
||||||
|
expected_node_id: str = Field(min_length=1, max_length=128)
|
||||||
|
agent_id: str
|
||||||
|
platform: ContourPlatform = "unknown"
|
||||||
|
telemetry_mode: TelemetryMode = "agent-mqtt"
|
||||||
|
address: str = Field(default="", max_length=253)
|
||||||
|
ssh_port: int = Field(default=22, ge=1, le=65535)
|
||||||
|
mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253)
|
||||||
|
mqtt_port: int = Field(default=1883, ge=1, le=65535)
|
||||||
|
revision: int = Field(default=0, ge=0)
|
||||||
|
updated_at_utc: str | None = None
|
||||||
|
|
||||||
|
@field_validator("contour_id", "agent_id")
|
||||||
|
@classmethod
|
||||||
|
def validate_identifier(cls, value: str) -> str:
|
||||||
|
normalized = value.strip().lower()
|
||||||
|
if SAFE_IDENTIFIER.fullmatch(normalized) is None:
|
||||||
|
raise ValueError("identifier must be a lowercase DNS-safe token")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
@field_validator("display_name")
|
||||||
|
@classmethod
|
||||||
|
def normalize_display_name(cls, value: str) -> str:
|
||||||
|
normalized = " ".join(value.split())
|
||||||
|
if not normalized:
|
||||||
|
raise ValueError("display name is required")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
@field_validator("expected_node_id")
|
||||||
|
@classmethod
|
||||||
|
def validate_node_id(cls, value: str) -> str:
|
||||||
|
normalized = value.strip()
|
||||||
|
if SAFE_NODE_ID.fullmatch(normalized) is None:
|
||||||
|
raise ValueError("node id contains unsupported characters")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
@field_validator("address", "mqtt_host")
|
||||||
|
@classmethod
|
||||||
|
def validate_host(cls, value: str) -> str:
|
||||||
|
normalized = value.strip()
|
||||||
|
if any(character.isspace() or ord(character) < 32 for character in normalized):
|
||||||
|
raise ValueError("host contains whitespace or control characters")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeContourCreate(StrictModel):
|
||||||
|
display_name: str = Field(min_length=1, max_length=80)
|
||||||
|
expected_node_id: str = Field(min_length=1, max_length=128)
|
||||||
|
platform: ContourPlatform = "unknown"
|
||||||
|
address: str = Field(default="", max_length=253)
|
||||||
|
ssh_port: int = Field(default=22, ge=1, le=65535)
|
||||||
|
mqtt_host: str = Field(default="127.0.0.1", min_length=1, max_length=253)
|
||||||
|
mqtt_port: int = Field(default=1883, ge=1, le=65535)
|
||||||
|
|
||||||
|
@field_validator("display_name")
|
||||||
|
@classmethod
|
||||||
|
def normalize_display_name(cls, value: str) -> str:
|
||||||
|
return ComputeContour.normalize_display_name(value)
|
||||||
|
|
||||||
|
@field_validator("expected_node_id")
|
||||||
|
@classmethod
|
||||||
|
def validate_node_id(cls, value: str) -> str:
|
||||||
|
return ComputeContour.validate_node_id(value)
|
||||||
|
|
||||||
|
@field_validator("address", "mqtt_host")
|
||||||
|
@classmethod
|
||||||
|
def validate_host(cls, value: str) -> str:
|
||||||
|
return ComputeContour.validate_host(value)
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeContourPut(ComputeContourCreate):
|
||||||
|
revision: int = Field(ge=0)
|
||||||
|
telemetry_mode: TelemetryMode = "agent-mqtt"
|
||||||
|
|
||||||
|
|
||||||
|
def default_compute_contour() -> ComputeContour:
|
||||||
|
return ComputeContour(
|
||||||
|
contour_id="worker-006",
|
||||||
|
display_name="Worker 006",
|
||||||
|
expected_node_id="DESKTOP-OPJ8J04",
|
||||||
|
agent_id="worker-006",
|
||||||
|
platform="windows",
|
||||||
|
telemetry_mode="agent-mqtt",
|
||||||
|
address="",
|
||||||
|
ssh_port=22,
|
||||||
|
mqtt_host="127.0.0.1",
|
||||||
|
mqtt_port=1883,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ComputeContourStore:
|
||||||
|
def __init__(self, root: Path) -> None:
|
||||||
|
self.root = root
|
||||||
|
self.path = root / CATALOG_FILE_NAME
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
def list_contours(self) -> list[ComputeContour]:
|
||||||
|
with self._lock:
|
||||||
|
return self._read_unlocked()
|
||||||
|
|
||||||
|
def create(self, request: ComputeContourCreate) -> ComputeContour:
|
||||||
|
with self._lock:
|
||||||
|
contours = self._read_unlocked()
|
||||||
|
contour_id = f"worker-{uuid.uuid4().hex[:8]}"
|
||||||
|
contour = ComputeContour(
|
||||||
|
contour_id=contour_id,
|
||||||
|
display_name=request.display_name,
|
||||||
|
expected_node_id=request.expected_node_id,
|
||||||
|
agent_id=contour_id,
|
||||||
|
platform=request.platform,
|
||||||
|
telemetry_mode="agent-mqtt",
|
||||||
|
address=request.address,
|
||||||
|
ssh_port=request.ssh_port,
|
||||||
|
mqtt_host=request.mqtt_host,
|
||||||
|
mqtt_port=request.mqtt_port,
|
||||||
|
revision=0,
|
||||||
|
updated_at_utc=_utc_now(),
|
||||||
|
)
|
||||||
|
contours.append(contour)
|
||||||
|
self._write_unlocked(contours)
|
||||||
|
return contour
|
||||||
|
|
||||||
|
def update(self, contour_id: str, request: ComputeContourPut) -> ComputeContour:
|
||||||
|
with self._lock:
|
||||||
|
contours = self._read_unlocked()
|
||||||
|
for index, current in enumerate(contours):
|
||||||
|
if current.contour_id != contour_id:
|
||||||
|
continue
|
||||||
|
if current.revision != request.revision:
|
||||||
|
raise RuntimeError("compute contour revision changed")
|
||||||
|
updated = current.model_copy(
|
||||||
|
update={
|
||||||
|
"display_name": request.display_name,
|
||||||
|
"expected_node_id": request.expected_node_id,
|
||||||
|
"platform": request.platform,
|
||||||
|
"telemetry_mode": request.telemetry_mode,
|
||||||
|
"address": request.address,
|
||||||
|
"ssh_port": request.ssh_port,
|
||||||
|
"mqtt_host": request.mqtt_host,
|
||||||
|
"mqtt_port": request.mqtt_port,
|
||||||
|
"revision": current.revision + 1,
|
||||||
|
"updated_at_utc": _utc_now(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
contours[index] = updated
|
||||||
|
self._write_unlocked(contours)
|
||||||
|
return updated
|
||||||
|
raise KeyError(contour_id)
|
||||||
|
|
||||||
|
def get(self, contour_id: str) -> ComputeContour:
|
||||||
|
for contour in self.list_contours():
|
||||||
|
if contour.contour_id == contour_id:
|
||||||
|
return contour
|
||||||
|
raise KeyError(contour_id)
|
||||||
|
|
||||||
|
def _read_unlocked(self) -> list[ComputeContour]:
|
||||||
|
if not self.path.is_file():
|
||||||
|
return [default_compute_contour()]
|
||||||
|
try:
|
||||||
|
document = json.loads(self.path.read_text(encoding="utf-8"))
|
||||||
|
rows = document.get("contours") if isinstance(document, dict) else None
|
||||||
|
if not isinstance(rows, list):
|
||||||
|
return [default_compute_contour()]
|
||||||
|
contours = [ComputeContour.model_validate(row) for row in rows]
|
||||||
|
return contours or [default_compute_contour()]
|
||||||
|
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
||||||
|
return [default_compute_contour()]
|
||||||
|
|
||||||
|
def _write_unlocked(self, contours: list[ComputeContour]) -> None:
|
||||||
|
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
descriptor, temporary_name = tempfile.mkstemp(
|
||||||
|
prefix=".compute-contours.",
|
||||||
|
suffix=".tmp",
|
||||||
|
dir=self.root,
|
||||||
|
)
|
||||||
|
temporary = Path(temporary_name)
|
||||||
|
try:
|
||||||
|
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
|
||||||
|
json.dump(
|
||||||
|
{
|
||||||
|
"schema_version": CATALOG_SCHEMA,
|
||||||
|
"contours": [
|
||||||
|
contour.model_dump(mode="json") for contour in contours
|
||||||
|
],
|
||||||
|
},
|
||||||
|
stream,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
stream.write("\n")
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
os.chmod(temporary, 0o600)
|
||||||
|
os.replace(temporary, self.path)
|
||||||
|
finally:
|
||||||
|
temporary.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _catalog_document(contours: list[ComputeContour]) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": CATALOG_SCHEMA,
|
||||||
|
"contours": [contour.model_dump(mode="json") for contour in contours],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _agent_install_document(contour: ComputeContour) -> dict[str, object]:
|
||||||
|
environment = {
|
||||||
|
"MISSIONCORE_CONTOUR_ID": contour.contour_id,
|
||||||
|
"MISSIONCORE_AGENT_ID": contour.agent_id,
|
||||||
|
"MISSIONCORE_NODE_ID": contour.expected_node_id,
|
||||||
|
"MISSIONCORE_MQTT_HOST": contour.mqtt_host,
|
||||||
|
"MISSIONCORE_MQTT_PORT": str(contour.mqtt_port),
|
||||||
|
"MISSIONCORE_MQTT_USERNAME": contour.agent_id,
|
||||||
|
}
|
||||||
|
if contour.platform == "windows":
|
||||||
|
command = (
|
||||||
|
"$env:MISSIONCORE_MQTT_PASSWORD=Read-Host 'MQTT password'; "
|
||||||
|
"telegraf.exe --service install "
|
||||||
|
'--config "C:\\ProgramData\\MissionCore\\telegraf.conf"'
|
||||||
|
)
|
||||||
|
template = "deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl"
|
||||||
|
else:
|
||||||
|
command = (
|
||||||
|
"read -s MISSIONCORE_MQTT_PASSWORD && export MISSIONCORE_MQTT_PASSWORD && "
|
||||||
|
"sudo telegraf --config /etc/telegraf/telegraf.d/mission-core.conf --test"
|
||||||
|
)
|
||||||
|
template = "deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl"
|
||||||
|
return {
|
||||||
|
"schema_version": INSTALL_SCHEMA,
|
||||||
|
"contour_id": contour.contour_id,
|
||||||
|
"platform": contour.platform,
|
||||||
|
"agent": {
|
||||||
|
"distribution": "Telegraf",
|
||||||
|
"configuration_template": template,
|
||||||
|
"environment": environment,
|
||||||
|
"secret_delivery": "interactive-prompt",
|
||||||
|
},
|
||||||
|
"command": command,
|
||||||
|
"ready": False,
|
||||||
|
"blocked_reason": (
|
||||||
|
"Сначала выпустите scoped MQTT credential в локальном telemetry plane. "
|
||||||
|
"Пароль не передаётся через Mission Core UI."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_compute_contour_router(*, root_provider: RootProvider) -> APIRouter:
|
||||||
|
store = ComputeContourStore(root_provider())
|
||||||
|
router = APIRouter(prefix="/api/v1/system", tags=["system"])
|
||||||
|
|
||||||
|
@router.get("/contours")
|
||||||
|
def list_contours() -> dict[str, object]:
|
||||||
|
return _catalog_document(store.list_contours())
|
||||||
|
|
||||||
|
@router.post("/contours", status_code=201)
|
||||||
|
def create_contour(request: ComputeContourCreate) -> dict[str, object]:
|
||||||
|
return store.create(request).model_dump(mode="json")
|
||||||
|
|
||||||
|
@router.put("/contours/{contour_id}")
|
||||||
|
def update_contour(
|
||||||
|
contour_id: str,
|
||||||
|
request: ComputeContourPut,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
return store.update(contour_id, request).model_dump(mode="json")
|
||||||
|
except KeyError as exc:
|
||||||
|
raise HTTPException(status_code=404, detail="Контур не найден.") from exc
|
||||||
|
except RuntimeError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Конфигурация контура была изменена в другой сессии.",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
@router.get("/contours/{contour_id}/agent-install")
|
||||||
|
def get_agent_install(contour_id: str) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
return _agent_install_document(store.get(contour_id))
|
||||||
|
except KeyError as exc:
|
||||||
|
raise HTTPException(status_code=404, detail="Контур не найден.") from exc
|
||||||
|
|
||||||
|
return router
|
||||||
@@ -10,6 +10,8 @@ import subprocess
|
|||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
from collections import deque
|
from collections import deque
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
@@ -27,11 +29,40 @@ DEFAULT_DISPLAY_NAME: Final = "Worker 006"
|
|||||||
EXPECTED_NODE_ID: Final = "DESKTOP-OPJ8J04"
|
EXPECTED_NODE_ID: Final = "DESKTOP-OPJ8J04"
|
||||||
SSH_HOST_ALIAS: Final = "mission-gpu"
|
SSH_HOST_ALIAS: Final = "mission-gpu"
|
||||||
PROFILE_FILE_NAME: Final = "worker-006.json"
|
PROFILE_FILE_NAME: Final = "worker-006.json"
|
||||||
CONTAINER_NAMES: Final = (
|
CONTAINER_GROUPS: Final = (
|
||||||
"mission-core-triton",
|
(
|
||||||
"mission-core-perception-worker",
|
"ndc-mission-core-triton",
|
||||||
"sentinel-frigate",
|
("ndc-mission-core-triton", "mission-core-triton"),
|
||||||
"sentinel-ollama",
|
"Inference Runtime",
|
||||||
|
"mission-core",
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"ndc-mission-core-perception-worker",
|
||||||
|
("ndc-mission-core-perception-worker", "mission-core-perception-worker"),
|
||||||
|
"Perception Pipeline",
|
||||||
|
"mission-core",
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"sentinel-frigate",
|
||||||
|
("sentinel-frigate",),
|
||||||
|
"Sentinel Frigate",
|
||||||
|
"external",
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"sentinel-ollama",
|
||||||
|
("sentinel-ollama",),
|
||||||
|
"Sentinel Ollama",
|
||||||
|
"external",
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
CONTAINER_NAMES: Final = tuple(
|
||||||
|
alias
|
||||||
|
for _canonical, aliases, _role, _owner, _external in CONTAINER_GROUPS
|
||||||
|
for alias in aliases
|
||||||
)
|
)
|
||||||
SAFE_HOSTNAME = re.compile(
|
SAFE_HOSTNAME = re.compile(
|
||||||
r"^(?=.{1,253}\.?$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*"
|
r"^(?=.{1,253}\.?$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*"
|
||||||
@@ -41,6 +72,8 @@ PROMETHEUS_SAMPLE = re.compile(
|
|||||||
r"^(?P<name>nv_inference_(?:request_success|request_failure|count))"
|
r"^(?P<name>nv_inference_(?:request_success|request_failure|count))"
|
||||||
r"(?:\{[^}]*\})?\s+(?P<value>[0-9.eE+-]+)$"
|
r"(?:\{[^}]*\})?\s+(?P<value>[0-9.eE+-]+)$"
|
||||||
)
|
)
|
||||||
|
TELEMETRY_QUERY_SCHEMA: Final = "missioncore.telemetry-query/v1"
|
||||||
|
DEFAULT_TELEMETRY_QUERY_URL: Final = "http://127.0.0.1:18030"
|
||||||
|
|
||||||
RootProvider = Callable[[], Path]
|
RootProvider = Callable[[], Path]
|
||||||
ProbeRunner = Callable[["WorkerConnectionProfile"], dict[str, Any]]
|
ProbeRunner = Callable[["WorkerConnectionProfile"], dict[str, Any]]
|
||||||
@@ -160,7 +193,9 @@ class WorkerProfileStore:
|
|||||||
WORKER_PROBE_POWERSHELL: Final = r"""
|
WORKER_PROBE_POWERSHELL: Final = r"""
|
||||||
$ErrorActionPreference = "Stop"
|
$ErrorActionPreference = "Stop"
|
||||||
$containers = @(
|
$containers = @(
|
||||||
|
"ndc-mission-core-triton",
|
||||||
"mission-core-triton",
|
"mission-core-triton",
|
||||||
|
"ndc-mission-core-perception-worker",
|
||||||
"mission-core-perception-worker",
|
"mission-core-perception-worker",
|
||||||
"sentinel-frigate",
|
"sentinel-frigate",
|
||||||
"sentinel-ollama"
|
"sentinel-ollama"
|
||||||
@@ -170,9 +205,9 @@ $cpu = Get-CimInstance Win32_Processor
|
|||||||
$computer = Get-CimInstance Win32_ComputerSystem
|
$computer = Get-CimInstance Win32_ComputerSystem
|
||||||
$dockerStats = @{}
|
$dockerStats = @{}
|
||||||
try {
|
try {
|
||||||
$statsLines = @(docker stats --no-stream --format "{{json .}}" $containers)
|
foreach ($name in $containers) {
|
||||||
foreach ($line in $statsLines) {
|
$line = docker stats --no-stream --format "{{json .}}" $name 2>$null
|
||||||
if ($line) {
|
if ($LASTEXITCODE -eq 0 -and $line) {
|
||||||
$row = $line | ConvertFrom-Json
|
$row = $line | ConvertFrom-Json
|
||||||
$dockerStats[$row.Name] = $row
|
$dockerStats[$row.Name] = $row
|
||||||
}
|
}
|
||||||
@@ -243,10 +278,23 @@ try {
|
|||||||
)
|
)
|
||||||
} catch {}
|
} catch {}
|
||||||
$perceptionHealth = $null
|
$perceptionHealth = $null
|
||||||
try {
|
$perceptionContainer = $null
|
||||||
$healthJson = docker exec mission-core-perception-worker python3 -c "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
|
foreach ($candidate in @(
|
||||||
$perceptionHealth = $healthJson | ConvertFrom-Json
|
"ndc-mission-core-perception-worker",
|
||||||
} catch {}
|
"mission-core-perception-worker"
|
||||||
|
)) {
|
||||||
|
docker inspect $candidate *> $null
|
||||||
|
if ($LASTEXITCODE -eq 0) {
|
||||||
|
$perceptionContainer = $candidate
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($perceptionContainer) {
|
||||||
|
try {
|
||||||
|
$healthJson = docker exec $perceptionContainer python3 -c "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
|
||||||
|
$perceptionHealth = $healthJson | ConvertFrom-Json
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
$disks = @(
|
$disks = @(
|
||||||
Get-CimInstance Win32_LogicalDisk -Filter "DriveType=3" |
|
Get-CimInstance Win32_LogicalDisk -Filter "DriveType=3" |
|
||||||
ForEach-Object {
|
ForEach-Object {
|
||||||
@@ -355,6 +403,7 @@ def run_worker_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"schema_version": PROBE_SCHEMA,
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"source": "legacy-ssh",
|
||||||
"reachable": True,
|
"reachable": True,
|
||||||
"identity_matches": True,
|
"identity_matches": True,
|
||||||
"node_id": node_id,
|
"node_id": node_id,
|
||||||
@@ -365,13 +414,265 @@ def run_worker_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _metric_payload(sample: object) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
||||||
|
row = _mapping(sample)
|
||||||
|
payload = _mapping(row.get("payload"))
|
||||||
|
return payload, _mapping(payload.get("fields")), _mapping(payload.get("tags"))
|
||||||
|
|
||||||
|
|
||||||
|
def _format_bytes(value: float | None) -> str:
|
||||||
|
if value is None:
|
||||||
|
return "—"
|
||||||
|
scaled = max(0.0, value)
|
||||||
|
units = ("B", "KiB", "MiB", "GiB", "TiB")
|
||||||
|
for unit in units:
|
||||||
|
if scaled < 1024 or unit == units[-1]:
|
||||||
|
precision = 0 if unit == "B" else 2
|
||||||
|
return f"{scaled:.{precision}f}{unit}"
|
||||||
|
scaled /= 1024
|
||||||
|
return "—"
|
||||||
|
|
||||||
|
|
||||||
|
def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
samples = [
|
||||||
|
sample for sample in _items(document.get("samples")) if isinstance(sample, dict)
|
||||||
|
]
|
||||||
|
node_ids = {
|
||||||
|
value
|
||||||
|
for sample in samples
|
||||||
|
if isinstance((value := sample.get("node_id")), str)
|
||||||
|
}
|
||||||
|
node_id = next(iter(node_ids)) if len(node_ids) == 1 else None
|
||||||
|
observed_values = [
|
||||||
|
value
|
||||||
|
for sample in samples
|
||||||
|
if isinstance((value := sample.get("observed_at_utc")), str)
|
||||||
|
]
|
||||||
|
raw: dict[str, Any] = {
|
||||||
|
"node_id": node_id,
|
||||||
|
"observed_at_utc": max(observed_values, default=_utc_now()),
|
||||||
|
"os": {},
|
||||||
|
"cpu": {},
|
||||||
|
"memory": {},
|
||||||
|
"disks": [],
|
||||||
|
"gpu": None,
|
||||||
|
"network": [],
|
||||||
|
"docker_stats": {},
|
||||||
|
"container_states": {},
|
||||||
|
"triton": {},
|
||||||
|
"perception": {},
|
||||||
|
}
|
||||||
|
docker_stats: dict[str, dict[str, str]] = raw["docker_stats"]
|
||||||
|
container_states: dict[str, dict[str, object]] = raw["container_states"]
|
||||||
|
for sample in samples:
|
||||||
|
measurement = sample.get("measurement")
|
||||||
|
payload, fields, tags = _metric_payload(sample)
|
||||||
|
if sample.get("kind") == "pipeline":
|
||||||
|
raw["perception"] = _mapping(payload.get("payload")) or payload
|
||||||
|
continue
|
||||||
|
if sample.get("kind") == "runtime":
|
||||||
|
runtime_payload = _mapping(payload.get("payload")) or payload
|
||||||
|
if isinstance(runtime_payload.get("triton"), dict):
|
||||||
|
raw["triton"] = runtime_payload["triton"]
|
||||||
|
continue
|
||||||
|
if measurement == "cpu":
|
||||||
|
raw["cpu"] = {
|
||||||
|
"name": "Processor",
|
||||||
|
"logical_processors": None,
|
||||||
|
"load_percent": _number(fields.get("usage_active")),
|
||||||
|
}
|
||||||
|
elif measurement == "system":
|
||||||
|
cpu = _mapping(raw.get("cpu"))
|
||||||
|
cpu["logical_processors"] = _number(fields.get("n_cpus"))
|
||||||
|
raw["cpu"] = cpu
|
||||||
|
raw["os"] = {
|
||||||
|
"caption": fields.get("platform") or fields.get("os") or "Windows",
|
||||||
|
"version": fields.get("platform_version"),
|
||||||
|
"uptime_seconds": _number(fields.get("uptime")),
|
||||||
|
}
|
||||||
|
elif measurement == "mem":
|
||||||
|
total = _number(fields.get("total"))
|
||||||
|
available = _number(fields.get("available"))
|
||||||
|
if available is None:
|
||||||
|
available = _number(fields.get("free"))
|
||||||
|
raw["memory"] = {
|
||||||
|
"total_bytes": total,
|
||||||
|
"free_bytes": available,
|
||||||
|
}
|
||||||
|
elif measurement == "disk":
|
||||||
|
raw["disks"].append(
|
||||||
|
{
|
||||||
|
"name": tags.get("path") or tags.get("device"),
|
||||||
|
"size_bytes": _number(fields.get("total")),
|
||||||
|
"free_bytes": _number(fields.get("free")),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
elif measurement == "net":
|
||||||
|
raw["network"].append(
|
||||||
|
{
|
||||||
|
"name": tags.get("interface") or "unknown",
|
||||||
|
"description": None,
|
||||||
|
"status": "Up",
|
||||||
|
"link_speed_bps": _number(fields.get("speed")),
|
||||||
|
"mac_address": None,
|
||||||
|
"addresses": [],
|
||||||
|
"received_bytes": _number(fields.get("bytes_recv")),
|
||||||
|
"sent_bytes": _number(fields.get("bytes_sent")),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
elif measurement == "nvidia_smi":
|
||||||
|
memory_used = _number(fields.get("memory_used"))
|
||||||
|
memory_total = _number(fields.get("memory_total"))
|
||||||
|
raw["gpu"] = {
|
||||||
|
"name": tags.get("name") or tags.get("gpu_name"),
|
||||||
|
"utilization_percent": _number(fields.get("utilization_gpu")),
|
||||||
|
"memory_used_mib": memory_used,
|
||||||
|
"memory_total_mib": memory_total,
|
||||||
|
"power_watts": _number(fields.get("power_draw")),
|
||||||
|
"temperature_celsius": _number(fields.get("temperature_gpu")),
|
||||||
|
}
|
||||||
|
elif measurement == "missioncore_triton_health":
|
||||||
|
raw["triton"] = {
|
||||||
|
"ready": fields.get("response_status_code_match") == 1
|
||||||
|
or fields.get("http_response_code") == 200,
|
||||||
|
"metrics": _items(_mapping(raw.get("triton")).get("metrics")),
|
||||||
|
}
|
||||||
|
elif measurement in {
|
||||||
|
"nv_inference_request_success",
|
||||||
|
"nv_inference_request_failure",
|
||||||
|
"nv_inference_count",
|
||||||
|
}:
|
||||||
|
value = next(
|
||||||
|
(
|
||||||
|
candidate
|
||||||
|
for candidate in (
|
||||||
|
_number(fields.get("gauge")),
|
||||||
|
_number(fields.get("counter")),
|
||||||
|
_number(fields.get("value")),
|
||||||
|
)
|
||||||
|
if candidate is not None
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if value is not None:
|
||||||
|
triton = _mapping(raw.get("triton"))
|
||||||
|
metrics = [
|
||||||
|
line for line in _items(triton.get("metrics")) if isinstance(line, str)
|
||||||
|
]
|
||||||
|
metrics.append(f"{measurement} {value}")
|
||||||
|
triton["metrics"] = metrics
|
||||||
|
raw["triton"] = triton
|
||||||
|
elif isinstance(measurement, str) and measurement.startswith("docker_container_"):
|
||||||
|
container_name = tags.get("container_name") or tags.get("container")
|
||||||
|
if not isinstance(container_name, str) or not container_name:
|
||||||
|
continue
|
||||||
|
stats = docker_stats.setdefault(container_name, {"Name": container_name})
|
||||||
|
container_states.setdefault(
|
||||||
|
container_name,
|
||||||
|
{
|
||||||
|
"state": {"Status": "running"},
|
||||||
|
"image": tags.get("container_image"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if measurement == "docker_container_cpu":
|
||||||
|
usage = _number(fields.get("usage_percent"))
|
||||||
|
if usage is not None:
|
||||||
|
stats["CPUPerc"] = f"{usage:.3f}%"
|
||||||
|
elif measurement == "docker_container_mem":
|
||||||
|
usage = _number(fields.get("usage"))
|
||||||
|
limit = _number(fields.get("limit"))
|
||||||
|
usage_percent = _number(fields.get("usage_percent"))
|
||||||
|
stats["MemUsage"] = f"{_format_bytes(usage)} / {_format_bytes(limit)}"
|
||||||
|
if usage_percent is not None:
|
||||||
|
stats["MemPerc"] = f"{usage_percent:.3f}%"
|
||||||
|
elif measurement == "docker_container_net":
|
||||||
|
received = _number(fields.get("rx_bytes"))
|
||||||
|
sent = _number(fields.get("tx_bytes"))
|
||||||
|
stats["NetIO"] = f"{_format_bytes(received)} / {_format_bytes(sent)}"
|
||||||
|
elif measurement == "docker_container_blkio":
|
||||||
|
total = _number(fields.get("io_service_bytes_recursive_total"))
|
||||||
|
stats["BlockIO"] = _format_bytes(total)
|
||||||
|
elif measurement == "win_system":
|
||||||
|
uptime = _number(fields.get("System_Up_Time"))
|
||||||
|
if not raw["os"]:
|
||||||
|
raw["os"] = {
|
||||||
|
"caption": "Windows",
|
||||||
|
"version": None,
|
||||||
|
"uptime_seconds": uptime,
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def run_worker_agent_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
|
||||||
|
started = time.perf_counter()
|
||||||
|
base_url = os.environ.get(
|
||||||
|
"MISSIONCORE_TELEMETRY_QUERY_URL",
|
||||||
|
DEFAULT_TELEMETRY_QUERY_URL,
|
||||||
|
).rstrip("/")
|
||||||
|
contour_id = os.environ.get("MISSIONCORE_TELEMETRY_CONTOUR_ID", "worker-006")
|
||||||
|
agent_id = os.environ.get("MISSIONCORE_TELEMETRY_AGENT_ID", "worker-006")
|
||||||
|
url = (
|
||||||
|
f"{base_url}/v1/contours/{contour_id}/agents/{agent_id}/latest"
|
||||||
|
"?max_age_seconds=30"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(url, timeout=3) as response: # noqa: S310
|
||||||
|
document = json.loads(response.read())
|
||||||
|
except (
|
||||||
|
OSError,
|
||||||
|
urllib.error.URLError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
TypeError,
|
||||||
|
ValueError,
|
||||||
|
):
|
||||||
|
return _failed_probe(
|
||||||
|
profile,
|
||||||
|
"telemetry-agent-unavailable",
|
||||||
|
started,
|
||||||
|
source="agent-mqtt",
|
||||||
|
)
|
||||||
|
if not isinstance(document, dict) or document.get("schema_version") != (
|
||||||
|
TELEMETRY_QUERY_SCHEMA
|
||||||
|
):
|
||||||
|
return _failed_probe(
|
||||||
|
profile,
|
||||||
|
"invalid-telemetry-response",
|
||||||
|
started,
|
||||||
|
source="agent-mqtt",
|
||||||
|
)
|
||||||
|
raw = _agent_raw_document(document)
|
||||||
|
node_id = raw.get("node_id")
|
||||||
|
if not isinstance(node_id, str):
|
||||||
|
return _failed_probe(
|
||||||
|
profile,
|
||||||
|
"telemetry-agent-stale",
|
||||||
|
started,
|
||||||
|
source="agent-mqtt",
|
||||||
|
)
|
||||||
|
identity_matches = node_id == profile.expected_node_id
|
||||||
|
return {
|
||||||
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"source": "agent-mqtt",
|
||||||
|
"reachable": True,
|
||||||
|
"identity_matches": identity_matches,
|
||||||
|
"node_id": node_id,
|
||||||
|
"latency_ms": (time.perf_counter() - started) * 1000,
|
||||||
|
"observed_at_utc": raw.get("observed_at_utc") or _utc_now(),
|
||||||
|
"error_code": None if identity_matches else "worker-identity-mismatch",
|
||||||
|
"raw": raw if identity_matches else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _failed_probe(
|
def _failed_probe(
|
||||||
profile: WorkerConnectionProfile,
|
profile: WorkerConnectionProfile,
|
||||||
error_code: str,
|
error_code: str,
|
||||||
started: float,
|
started: float,
|
||||||
|
*,
|
||||||
|
source: str = "legacy-ssh",
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"schema_version": PROBE_SCHEMA,
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"source": source,
|
||||||
"reachable": False,
|
"reachable": False,
|
||||||
"identity_matches": False,
|
"identity_matches": False,
|
||||||
"node_id": None,
|
"node_id": None,
|
||||||
@@ -405,23 +706,30 @@ def _percent(used: float | None, total: float | None) -> float | None:
|
|||||||
|
|
||||||
|
|
||||||
def _container_document(
|
def _container_document(
|
||||||
name: str,
|
canonical_name: str,
|
||||||
|
aliases: tuple[str, ...],
|
||||||
|
role: str,
|
||||||
|
owner: str,
|
||||||
|
external: bool,
|
||||||
raw_stats: dict[str, Any],
|
raw_stats: dict[str, Any],
|
||||||
raw_states: dict[str, Any],
|
raw_states: dict[str, Any],
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
name = next(
|
||||||
|
(
|
||||||
|
alias
|
||||||
|
for alias in aliases
|
||||||
|
if isinstance(raw_states.get(alias), dict)
|
||||||
|
or isinstance(raw_stats.get(alias), dict)
|
||||||
|
),
|
||||||
|
canonical_name,
|
||||||
|
)
|
||||||
stats = _mapping(raw_stats.get(name))
|
stats = _mapping(raw_stats.get(name))
|
||||||
descriptor = _mapping(raw_states.get(name))
|
descriptor = _mapping(raw_states.get(name))
|
||||||
state = _mapping(descriptor.get("state"))
|
state = _mapping(descriptor.get("state"))
|
||||||
health = _mapping(state.get("Health"))
|
health = _mapping(state.get("Health"))
|
||||||
roles = {
|
|
||||||
"mission-core-triton": ("Inference Runtime", "mission-core", False),
|
|
||||||
"mission-core-perception-worker": ("Perception Pipeline", "mission-core", False),
|
|
||||||
"sentinel-frigate": ("Sentinel Frigate", "external", True),
|
|
||||||
"sentinel-ollama": ("Sentinel Ollama", "external", True),
|
|
||||||
}
|
|
||||||
role, owner, external = roles[name]
|
|
||||||
return {
|
return {
|
||||||
"name": name,
|
"name": name,
|
||||||
|
"canonical_name": canonical_name,
|
||||||
"role": role,
|
"role": role,
|
||||||
"owner": owner,
|
"owner": owner,
|
||||||
"external": external,
|
"external": external,
|
||||||
@@ -593,10 +901,12 @@ class WorkerTelemetryService:
|
|||||||
store: WorkerProfileStore,
|
store: WorkerProfileStore,
|
||||||
probe_runner: ProbeRunner = run_worker_probe,
|
probe_runner: ProbeRunner = run_worker_probe,
|
||||||
*,
|
*,
|
||||||
|
telemetry_probe_runner: ProbeRunner | None = None,
|
||||||
cache_seconds: float = 1.5,
|
cache_seconds: float = 1.5,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.store = store
|
self.store = store
|
||||||
self.probe_runner = probe_runner
|
self.probe_runner = probe_runner
|
||||||
|
self.telemetry_probe_runner = telemetry_probe_runner or probe_runner
|
||||||
self.cache_seconds = cache_seconds
|
self.cache_seconds = cache_seconds
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._cached_at = 0.0
|
self._cached_at = 0.0
|
||||||
@@ -654,7 +964,7 @@ class WorkerTelemetryService:
|
|||||||
"history": list(self._history)[-history_limit:],
|
"history": list(self._history)[-history_limit:],
|
||||||
}
|
}
|
||||||
profile = self.store.read()
|
profile = self.store.read()
|
||||||
probe = self.probe_runner(profile)
|
probe = self.telemetry_probe_runner(profile)
|
||||||
document = self._telemetry_document(profile, probe, now)
|
document = self._telemetry_document(profile, probe, now)
|
||||||
self._cached = document
|
self._cached = document
|
||||||
self._cached_at = now
|
self._cached_at = now
|
||||||
@@ -719,7 +1029,16 @@ class WorkerTelemetryService:
|
|||||||
raw_stats = _mapping(raw.get("docker_stats"))
|
raw_stats = _mapping(raw.get("docker_stats"))
|
||||||
raw_states = _mapping(raw.get("container_states"))
|
raw_states = _mapping(raw.get("container_states"))
|
||||||
runtimes = [
|
runtimes = [
|
||||||
_container_document(name, raw_stats, raw_states) for name in CONTAINER_NAMES
|
_container_document(
|
||||||
|
canonical_name,
|
||||||
|
aliases,
|
||||||
|
role,
|
||||||
|
owner,
|
||||||
|
external,
|
||||||
|
raw_stats,
|
||||||
|
raw_states,
|
||||||
|
)
|
||||||
|
for canonical_name, aliases, role, owner, external in CONTAINER_GROUPS
|
||||||
]
|
]
|
||||||
triton = _mapping(raw.get("triton"))
|
triton = _mapping(raw.get("triton"))
|
||||||
triton_document = {
|
triton_document = {
|
||||||
@@ -801,6 +1120,11 @@ class WorkerTelemetryService:
|
|||||||
def _public_probe(probe: dict[str, Any]) -> dict[str, Any]:
|
def _public_probe(probe: dict[str, Any]) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"schema_version": PROBE_SCHEMA,
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"source": (
|
||||||
|
probe.get("source")
|
||||||
|
if probe.get("source") in {"agent-mqtt", "legacy-ssh"}
|
||||||
|
else "unknown"
|
||||||
|
),
|
||||||
"reachable": probe.get("reachable") is True,
|
"reachable": probe.get("reachable") is True,
|
||||||
"identity_matches": probe.get("identity_matches") is True,
|
"identity_matches": probe.get("identity_matches") is True,
|
||||||
"node_id": probe.get("node_id") if isinstance(probe.get("node_id"), str) else None,
|
"node_id": probe.get("node_id") if isinstance(probe.get("node_id"), str) else None,
|
||||||
@@ -822,9 +1146,14 @@ def build_system_telemetry_router(
|
|||||||
*,
|
*,
|
||||||
root_provider: RootProvider,
|
root_provider: RootProvider,
|
||||||
probe_runner: ProbeRunner = run_worker_probe,
|
probe_runner: ProbeRunner = run_worker_probe,
|
||||||
|
telemetry_probe_runner: ProbeRunner = run_worker_agent_probe,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
store = WorkerProfileStore(root_provider())
|
store = WorkerProfileStore(root_provider())
|
||||||
service = WorkerTelemetryService(store, probe_runner)
|
service = WorkerTelemetryService(
|
||||||
|
store,
|
||||||
|
probe_runner,
|
||||||
|
telemetry_probe_runner=telemetry_probe_runner,
|
||||||
|
)
|
||||||
router = APIRouter(prefix="/api/v1/system", tags=["system"])
|
router = APIRouter(prefix="/api/v1/system", tags=["system"])
|
||||||
|
|
||||||
@router.get("/worker-profile")
|
@router.get("/worker-profile")
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import stat
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from fastapi.routing import APIRoute
|
||||||
|
|
||||||
|
from k1link.web.compute_contour_api import (
|
||||||
|
CATALOG_SCHEMA,
|
||||||
|
ComputeContourCreate,
|
||||||
|
ComputeContourPut,
|
||||||
|
ComputeContourStore,
|
||||||
|
build_compute_contour_router,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
||||||
|
for route in router.routes:
|
||||||
|
if (
|
||||||
|
isinstance(route, APIRoute)
|
||||||
|
and route.path == path
|
||||||
|
and route.methods is not None
|
||||||
|
and method in route.methods
|
||||||
|
):
|
||||||
|
return route.endpoint
|
||||||
|
raise AssertionError(f"{method} {path} route is missing")
|
||||||
|
|
||||||
|
|
||||||
|
def test_contour_store_migrates_worker_006_as_first_configuration(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
store = ComputeContourStore(tmp_path / "system")
|
||||||
|
contours = store.list_contours()
|
||||||
|
|
||||||
|
assert len(contours) == 1
|
||||||
|
assert contours[0].contour_id == "worker-006"
|
||||||
|
assert contours[0].telemetry_mode == "agent-mqtt"
|
||||||
|
assert not store.path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_contour_store_creates_and_updates_private_catalog(tmp_path: Path) -> None:
|
||||||
|
store = ComputeContourStore(tmp_path / "system")
|
||||||
|
created = store.create(
|
||||||
|
ComputeContourCreate(
|
||||||
|
display_name="Field Worker",
|
||||||
|
expected_node_id="FIELD-01",
|
||||||
|
platform="linux",
|
||||||
|
address="192.0.2.25",
|
||||||
|
mqtt_host="192.0.2.5",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
updated = store.update(
|
||||||
|
created.contour_id,
|
||||||
|
ComputeContourPut(
|
||||||
|
revision=created.revision,
|
||||||
|
display_name="Field Worker 01",
|
||||||
|
expected_node_id="FIELD-01",
|
||||||
|
platform="linux",
|
||||||
|
telemetry_mode="agent-mqtt",
|
||||||
|
address="192.0.2.25",
|
||||||
|
ssh_port=22,
|
||||||
|
mqtt_host="192.0.2.5",
|
||||||
|
mqtt_port=1883,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert updated.display_name == "Field Worker 01"
|
||||||
|
assert updated.revision == 1
|
||||||
|
assert stat.S_IMODE(store.path.stat().st_mode) == 0o600
|
||||||
|
assert len(store.list_contours()) == 2
|
||||||
|
with pytest.raises(RuntimeError, match="revision changed"):
|
||||||
|
store.update(
|
||||||
|
created.contour_id,
|
||||||
|
ComputeContourPut(
|
||||||
|
revision=0,
|
||||||
|
display_name="stale",
|
||||||
|
expected_node_id="FIELD-01",
|
||||||
|
platform="linux",
|
||||||
|
telemetry_mode="agent-mqtt",
|
||||||
|
address="",
|
||||||
|
ssh_port=22,
|
||||||
|
mqtt_host="127.0.0.1",
|
||||||
|
mqtt_port=1883,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_contour_router_exposes_catalog_and_safe_install_contract(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
router = build_compute_contour_router(
|
||||||
|
root_provider=lambda: tmp_path / "system"
|
||||||
|
)
|
||||||
|
list_contours = _endpoint(router, "/api/v1/system/contours", "GET")
|
||||||
|
install = _endpoint(
|
||||||
|
router,
|
||||||
|
"/api/v1/system/contours/{contour_id}/agent-install",
|
||||||
|
"GET",
|
||||||
|
)
|
||||||
|
|
||||||
|
catalog = list_contours()
|
||||||
|
assert catalog["schema_version"] == CATALOG_SCHEMA
|
||||||
|
assert catalog["contours"][0]["contour_id"] == "worker-006"
|
||||||
|
document = install("worker-006")
|
||||||
|
assert document["agent"]["distribution"] == "Telegraf"
|
||||||
|
assert "MQTT password" in document["command"]
|
||||||
|
assert "password" not in document["agent"]["environment"]
|
||||||
|
assert document["ready"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_contour_router_returns_404_for_unknown_contour(tmp_path: Path) -> None:
|
||||||
|
router = build_compute_contour_router(
|
||||||
|
root_provider=lambda: tmp_path / "system"
|
||||||
|
)
|
||||||
|
install = _endpoint(
|
||||||
|
router,
|
||||||
|
"/api/v1/system/contours/{contour_id}/agent-install",
|
||||||
|
"GET",
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
install("missing")
|
||||||
|
assert error.value.status_code == 404
|
||||||
@@ -16,6 +16,7 @@ from k1link.web.system_telemetry_api import (
|
|||||||
WorkerConnectionProfilePut,
|
WorkerConnectionProfilePut,
|
||||||
WorkerProfileStore,
|
WorkerProfileStore,
|
||||||
WorkerTelemetryService,
|
WorkerTelemetryService,
|
||||||
|
_agent_raw_document,
|
||||||
_ssh_arguments,
|
_ssh_arguments,
|
||||||
build_system_telemetry_router,
|
build_system_telemetry_router,
|
||||||
)
|
)
|
||||||
@@ -42,6 +43,7 @@ def _probe(
|
|||||||
return {
|
return {
|
||||||
"reachable": True,
|
"reachable": True,
|
||||||
"identity_matches": node_id == EXPECTED_NODE_ID,
|
"identity_matches": node_id == EXPECTED_NODE_ID,
|
||||||
|
"source": "agent-mqtt",
|
||||||
"node_id": node_id,
|
"node_id": node_id,
|
||||||
"latency_ms": 12.5,
|
"latency_ms": 12.5,
|
||||||
"observed_at_utc": "2026-07-27T12:00:00Z",
|
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||||
@@ -208,6 +210,7 @@ def test_worker_telemetry_separates_mission_core_and_external_load(
|
|||||||
runtimes = {runtime["name"]: runtime for runtime in document["runtimes"]}
|
runtimes = {runtime["name"]: runtime for runtime in document["runtimes"]}
|
||||||
|
|
||||||
assert document["connection"]["identity_matches"] is True
|
assert document["connection"]["identity_matches"] is True
|
||||||
|
assert document["connection"]["source"] == "agent-mqtt"
|
||||||
assert document["node"]["memory"]["used_percent"] == 60
|
assert document["node"]["memory"]["used_percent"] == 60
|
||||||
assert document["node"]["gpu"]["memory_used_percent"] == 50
|
assert document["node"]["gpu"]["memory_used_percent"] == 50
|
||||||
assert document["node"]["triton"]["requests_succeeded"] == 12
|
assert document["node"]["triton"]["requests_succeeded"] == 12
|
||||||
@@ -237,6 +240,106 @@ def test_worker_telemetry_separates_mission_core_and_external_load(
|
|||||||
)["share_percent"] is None
|
)["share_percent"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_telemetry_prefers_ndc_container_names_during_migration(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
probe = _probe()
|
||||||
|
raw = probe["raw"]
|
||||||
|
triton_stats = raw["docker_stats"].pop("mission-core-triton")
|
||||||
|
triton_stats["Name"] = "ndc-mission-core-triton"
|
||||||
|
raw["docker_stats"]["ndc-mission-core-triton"] = triton_stats
|
||||||
|
raw["container_states"]["ndc-mission-core-triton"] = raw[
|
||||||
|
"container_states"
|
||||||
|
].pop("mission-core-triton")
|
||||||
|
|
||||||
|
service = WorkerTelemetryService(
|
||||||
|
WorkerProfileStore(tmp_path / "system"),
|
||||||
|
lambda _: probe,
|
||||||
|
cache_seconds=0,
|
||||||
|
)
|
||||||
|
document = service.snapshot(10)
|
||||||
|
mission_core_runtimes = [
|
||||||
|
runtime for runtime in document["runtimes"] if not runtime["external"]
|
||||||
|
]
|
||||||
|
|
||||||
|
assert len(mission_core_runtimes) == 2
|
||||||
|
triton = next(
|
||||||
|
runtime
|
||||||
|
for runtime in mission_core_runtimes
|
||||||
|
if runtime["role"] == "Inference Runtime"
|
||||||
|
)
|
||||||
|
assert triton["name"] == "ndc-mission-core-triton"
|
||||||
|
assert triton["canonical_name"] == "ndc-mission-core-triton"
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_metrics_are_mapped_to_the_existing_product_contract() -> None:
|
||||||
|
document = _agent_raw_document(
|
||||||
|
{
|
||||||
|
"samples": [
|
||||||
|
{
|
||||||
|
"node_id": EXPECTED_NODE_ID,
|
||||||
|
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||||
|
"kind": "host",
|
||||||
|
"measurement": "cpu",
|
||||||
|
"payload": {
|
||||||
|
"fields": {"usage_active": 12.5},
|
||||||
|
"tags": {"cpu": "cpu-total"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"node_id": EXPECTED_NODE_ID,
|
||||||
|
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||||
|
"kind": "host",
|
||||||
|
"measurement": "mem",
|
||||||
|
"payload": {
|
||||||
|
"fields": {
|
||||||
|
"total": 1_000,
|
||||||
|
"available": 400,
|
||||||
|
},
|
||||||
|
"tags": {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"node_id": EXPECTED_NODE_ID,
|
||||||
|
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||||
|
"kind": "host",
|
||||||
|
"measurement": "net",
|
||||||
|
"payload": {
|
||||||
|
"fields": {
|
||||||
|
"bytes_recv": 2_000,
|
||||||
|
"bytes_sent": 1_000,
|
||||||
|
},
|
||||||
|
"tags": {"interface": "Ethernet"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"node_id": EXPECTED_NODE_ID,
|
||||||
|
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||||
|
"kind": "host",
|
||||||
|
"measurement": "docker_container_cpu",
|
||||||
|
"payload": {
|
||||||
|
"fields": {"usage_percent": 2.5},
|
||||||
|
"tags": {
|
||||||
|
"container_name": "ndc-mission-core-triton",
|
||||||
|
"container_image": "triton@sha256:accepted",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert document["node_id"] == EXPECTED_NODE_ID
|
||||||
|
assert document["cpu"]["load_percent"] == 12.5
|
||||||
|
assert document["memory"] == {"total_bytes": 1_000.0, "free_bytes": 400.0}
|
||||||
|
assert document["network"][0]["name"] == "Ethernet"
|
||||||
|
assert document["docker_stats"]["ndc-mission-core-triton"]["CPUPerc"] == "2.500%"
|
||||||
|
assert (
|
||||||
|
document["container_states"]["ndc-mission-core-triton"]["image"]
|
||||||
|
== "triton@sha256:accepted"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user