feat(system): add Worker 006 telemetry and network profile
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,237 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import stat
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.routing import APIRoute
|
||||
from pydantic import ValidationError
|
||||
|
||||
from k1link.web.system_telemetry_api import (
|
||||
EXPECTED_NODE_ID,
|
||||
WorkerConnectionProfile,
|
||||
WorkerConnectionProfilePut,
|
||||
WorkerProfileStore,
|
||||
WorkerTelemetryService,
|
||||
_ssh_arguments,
|
||||
build_system_telemetry_router,
|
||||
)
|
||||
|
||||
|
||||
def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
||||
for route in router.routes:
|
||||
if (
|
||||
isinstance(route, APIRoute)
|
||||
and route.path == path
|
||||
and route.methods is not None
|
||||
and method in route.methods
|
||||
):
|
||||
return route.endpoint
|
||||
raise AssertionError(f"{method} {path} route is missing")
|
||||
|
||||
|
||||
def _probe(
|
||||
*,
|
||||
node_id: str = EXPECTED_NODE_ID,
|
||||
received_bytes: float = 1_000,
|
||||
sent_bytes: float = 500,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"reachable": True,
|
||||
"identity_matches": node_id == EXPECTED_NODE_ID,
|
||||
"node_id": node_id,
|
||||
"latency_ms": 12.5,
|
||||
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||
"error_code": None,
|
||||
"raw": {
|
||||
"node_id": node_id,
|
||||
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||
"os": {
|
||||
"caption": "Windows 11 Pro",
|
||||
"version": "10.0",
|
||||
"uptime_seconds": 100,
|
||||
},
|
||||
"cpu": {
|
||||
"name": "CPU",
|
||||
"logical_processors": 32,
|
||||
"load_percent": 25,
|
||||
},
|
||||
"memory": {
|
||||
"total_bytes": 1000,
|
||||
"free_bytes": 400,
|
||||
},
|
||||
"disks": [],
|
||||
"gpu": {
|
||||
"name": "GPU",
|
||||
"utilization_percent": 30,
|
||||
"memory_used_mib": 100,
|
||||
"memory_total_mib": 200,
|
||||
},
|
||||
"network": [
|
||||
{
|
||||
"name": "LAN",
|
||||
"description": "Ethernet",
|
||||
"status": "Up",
|
||||
"link_speed_bps": 1_000_000_000,
|
||||
"addresses": ["192.0.2.10"],
|
||||
"received_bytes": received_bytes,
|
||||
"sent_bytes": sent_bytes,
|
||||
}
|
||||
],
|
||||
"docker_stats": {
|
||||
"mission-core-triton": {
|
||||
"Name": "mission-core-triton",
|
||||
"CPUPerc": "2.5%",
|
||||
"MemPerc": "3.5%",
|
||||
"MemUsage": "1GiB / 32GiB",
|
||||
"NetIO": "1MB / 2MB",
|
||||
"BlockIO": "0B / 0B",
|
||||
"PIDs": "12",
|
||||
},
|
||||
"sentinel-frigate": {
|
||||
"Name": "sentinel-frigate",
|
||||
"CPUPerc": "150%",
|
||||
"MemPerc": "20%",
|
||||
"MemUsage": "6GiB / 32GiB",
|
||||
"NetIO": "3GB / 1GB",
|
||||
"BlockIO": "1GB / 1GB",
|
||||
"PIDs": "200",
|
||||
},
|
||||
},
|
||||
"container_states": {
|
||||
"mission-core-triton": {
|
||||
"state": {"Status": "running", "Health": {"Status": "healthy"}},
|
||||
"image": "triton@sha256:accepted",
|
||||
},
|
||||
"sentinel-frigate": {
|
||||
"state": {"Status": "running", "Health": {"Status": "healthy"}},
|
||||
"image": "frigate@sha256:external",
|
||||
},
|
||||
},
|
||||
"triton": {
|
||||
"ready": True,
|
||||
"metrics": [
|
||||
'nv_inference_request_success{model="detector"} 12',
|
||||
'nv_inference_request_failure{model="detector"} 0',
|
||||
],
|
||||
},
|
||||
"perception": {
|
||||
"state": "busy",
|
||||
"current_stage": "detector",
|
||||
"active_request_id": "run-001",
|
||||
"active_frame_index": 42,
|
||||
"completed_runs": 3,
|
||||
"failed_runs": 0,
|
||||
"model_load_seconds": 10.5,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_worker_profile_rejects_ssh_option_injection() -> None:
|
||||
for unsafe in (
|
||||
"-oProxyCommand=touch /tmp/unsafe",
|
||||
"worker.local -p 2200",
|
||||
"worker.local/../../unsafe",
|
||||
"worker.local\nHost evil",
|
||||
):
|
||||
with pytest.raises(ValidationError):
|
||||
WorkerConnectionProfile(address=unsafe)
|
||||
|
||||
assert WorkerConnectionProfile(address="192.0.2.15").address == "192.0.2.15"
|
||||
assert WorkerConnectionProfile(address="worker-006.local").address == "worker-006.local"
|
||||
|
||||
|
||||
def test_worker_profile_is_atomic_versioned_and_private(tmp_path: Path) -> None:
|
||||
store = WorkerProfileStore(tmp_path / "system")
|
||||
initial = store.read()
|
||||
assert initial.revision == 0
|
||||
saved = store.save(
|
||||
WorkerConnectionProfilePut(
|
||||
revision=0,
|
||||
address="192.0.2.15",
|
||||
port=2200,
|
||||
)
|
||||
)
|
||||
|
||||
assert saved.revision == 1
|
||||
assert store.read() == saved
|
||||
assert stat.S_IMODE(store.path.stat().st_mode) == 0o600
|
||||
with pytest.raises(RuntimeError, match="revision changed"):
|
||||
store.save(
|
||||
WorkerConnectionProfilePut(
|
||||
revision=0,
|
||||
address="192.0.2.16",
|
||||
port=22,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_ssh_command_keeps_identity_pinned_and_values_as_arguments() -> None:
|
||||
arguments = _ssh_arguments(
|
||||
WorkerConnectionProfile(address="192.0.2.15", port=2200)
|
||||
)
|
||||
assert "BatchMode=yes" in arguments
|
||||
assert "StrictHostKeyChecking=yes" in arguments
|
||||
assert "HostKeyAlias=mission-gpu" in arguments
|
||||
assert "HostName=192.0.2.15" in arguments
|
||||
assert arguments[arguments.index("-p") + 1] == "2200"
|
||||
assert "mission-gpu" in arguments
|
||||
assert "192.0.2.15; touch unsafe" not in arguments
|
||||
|
||||
|
||||
def test_worker_telemetry_separates_mission_core_and_external_load(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service = WorkerTelemetryService(
|
||||
WorkerProfileStore(tmp_path / "system"),
|
||||
lambda _: _probe(),
|
||||
cache_seconds=0,
|
||||
)
|
||||
document = service.snapshot(10)
|
||||
runtimes = {runtime["name"]: runtime for runtime in document["runtimes"]}
|
||||
|
||||
assert document["connection"]["identity_matches"] is True
|
||||
assert document["node"]["memory"]["used_percent"] == 60
|
||||
assert document["node"]["gpu"]["memory_used_percent"] == 50
|
||||
assert document["node"]["triton"]["requests_succeeded"] == 12
|
||||
assert runtimes["mission-core-triton"]["external"] is False
|
||||
assert runtimes["mission-core-triton"]["cpu_percent"] == 2.5
|
||||
assert runtimes["sentinel-frigate"]["external"] is True
|
||||
assert runtimes["sentinel-frigate"]["cpu_percent"] == 150
|
||||
assert document["pipeline"]["active_request_id"] == "run-001"
|
||||
assert next(
|
||||
stage
|
||||
for stage in document["pipeline"]["stages"]
|
||||
if stage["id"] == "detector"
|
||||
)["state"] == "active"
|
||||
assert next(
|
||||
stage
|
||||
for stage in document["pipeline"]["stages"]
|
||||
if stage["id"] == "semantic-model"
|
||||
)["state"] == "active"
|
||||
|
||||
|
||||
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
router = build_system_telemetry_router(
|
||||
root_provider=lambda: tmp_path / "system",
|
||||
probe_runner=lambda _: _probe(node_id="OTHER-NODE"),
|
||||
)
|
||||
apply_profile = _endpoint(router, "/api/v1/system/worker-profile", "PUT")
|
||||
|
||||
with pytest.raises(HTTPException) as error:
|
||||
apply_profile(
|
||||
WorkerConnectionProfilePut(
|
||||
revision=0,
|
||||
address="192.0.2.20",
|
||||
port=22,
|
||||
)
|
||||
)
|
||||
assert error.value.status_code == 409
|
||||
assert WorkerProfileStore(tmp_path / "system").read().revision == 0
|
||||
assert not WorkerProfileStore(tmp_path / "system").path.exists()
|
||||
Reference in New Issue
Block a user