from __future__ import annotations import stat from collections.abc import Callable from pathlib import Path from typing import Any import pytest from fastapi import APIRouter, HTTPException from fastapi.routing import APIRoute from pydantic import ValidationError from k1link.web.system_telemetry_api import ( EXPECTED_NODE_ID, WorkerConnectionProfile, WorkerConnectionProfilePut, WorkerProfileStore, WorkerTelemetryService, _ssh_arguments, build_system_telemetry_router, ) def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]: for route in router.routes: if ( isinstance(route, APIRoute) and route.path == path and route.methods is not None and method in route.methods ): return route.endpoint raise AssertionError(f"{method} {path} route is missing") def _probe( *, node_id: str = EXPECTED_NODE_ID, received_bytes: float = 1_000, sent_bytes: float = 500, ) -> dict[str, Any]: return { "reachable": True, "identity_matches": node_id == EXPECTED_NODE_ID, "node_id": node_id, "latency_ms": 12.5, "observed_at_utc": "2026-07-27T12:00:00Z", "error_code": None, "raw": { "node_id": node_id, "observed_at_utc": "2026-07-27T12:00:00Z", "os": { "caption": "Windows 11 Pro", "version": "10.0", "uptime_seconds": 100, }, "cpu": { "name": "CPU", "logical_processors": 32, "load_percent": 25, }, "memory": { "total_bytes": 1000, "free_bytes": 400, }, "disks": [], "gpu": { "name": "GPU", "utilization_percent": 30, "memory_used_mib": 100, "memory_total_mib": 200, }, "network": [ { "name": "LAN", "description": "Ethernet", "status": "Up", "link_speed_bps": 1_000_000_000, "addresses": ["192.0.2.10"], "received_bytes": received_bytes, "sent_bytes": sent_bytes, } ], "docker_stats": { "mission-core-triton": { "Name": "mission-core-triton", "CPUPerc": "2.5%", "MemPerc": "3.5%", "MemUsage": "1GiB / 32GiB", "NetIO": "1MB / 2MB", "BlockIO": "0B / 0B", "PIDs": "12", }, "sentinel-frigate": { "Name": "sentinel-frigate", "CPUPerc": "150%", "MemPerc": "20%", "MemUsage": "6GiB / 32GiB", "NetIO": "3GB / 1GB", "BlockIO": "1GB / 1GB", "PIDs": "200", }, }, "container_states": { "mission-core-triton": { "state": {"Status": "running", "Health": {"Status": "healthy"}}, "image": "triton@sha256:accepted", }, "sentinel-frigate": { "state": {"Status": "running", "Health": {"Status": "healthy"}}, "image": "frigate@sha256:external", }, }, "triton": { "ready": True, "metrics": [ 'nv_inference_request_success{model="detector"} 12', 'nv_inference_request_failure{model="detector"} 0', ], }, "perception": { "state": "busy", "current_stage": "detector", "active_stages": ["detector", "semantic-model"], "active_request_id": "run-001", "active_frame_index": 42, "completed_runs": 3, "failed_runs": 0, "model_load_seconds": 10.5, "stage_metrics": { "detector": { "elapsed_seconds": 2.5, "activations": 42, "share_percent": 62.5, }, "semantic-model": { "elapsed_seconds": 1.5, "activations": 11, "share_percent": 37.5, }, }, }, }, } def test_worker_profile_rejects_ssh_option_injection() -> None: for unsafe in ( "-oProxyCommand=touch /tmp/unsafe", "worker.local -p 2200", "worker.local/../../unsafe", "worker.local\nHost evil", ): with pytest.raises(ValidationError): WorkerConnectionProfile(address=unsafe) assert WorkerConnectionProfile(address="192.0.2.15").address == "192.0.2.15" assert WorkerConnectionProfile(address="worker-006.local").address == "worker-006.local" def test_worker_profile_is_atomic_versioned_and_private(tmp_path: Path) -> None: store = WorkerProfileStore(tmp_path / "system") initial = store.read() assert initial.revision == 0 saved = store.save( WorkerConnectionProfilePut( revision=0, address="192.0.2.15", port=2200, ) ) assert saved.revision == 1 assert store.read() == saved assert stat.S_IMODE(store.path.stat().st_mode) == 0o600 with pytest.raises(RuntimeError, match="revision changed"): store.save( WorkerConnectionProfilePut( revision=0, address="192.0.2.16", port=22, ) ) def test_ssh_command_keeps_identity_pinned_and_values_as_arguments() -> None: arguments = _ssh_arguments( WorkerConnectionProfile(address="192.0.2.15", port=2200) ) assert "BatchMode=yes" in arguments assert "StrictHostKeyChecking=yes" in arguments assert "HostKeyAlias=mission-gpu" in arguments assert "HostName=192.0.2.15" in arguments assert arguments[arguments.index("-p") + 1] == "2200" assert "mission-gpu" in arguments assert "192.0.2.15; touch unsafe" not in arguments def test_worker_telemetry_separates_mission_core_and_external_load( tmp_path: Path, ) -> None: service = WorkerTelemetryService( WorkerProfileStore(tmp_path / "system"), lambda _: _probe(), cache_seconds=0, ) document = service.snapshot(10) runtimes = {runtime["name"]: runtime for runtime in document["runtimes"]} assert document["connection"]["identity_matches"] is True assert document["node"]["memory"]["used_percent"] == 60 assert document["node"]["gpu"]["memory_used_percent"] == 50 assert document["node"]["triton"]["requests_succeeded"] == 12 assert runtimes["mission-core-triton"]["external"] is False assert runtimes["mission-core-triton"]["cpu_percent"] == 2.5 assert runtimes["sentinel-frigate"]["external"] is True assert runtimes["sentinel-frigate"]["cpu_percent"] == 150 assert document["pipeline"]["active_request_id"] == "run-001" detector_stage = next( stage for stage in document["pipeline"]["stages"] if stage["id"] == "detector" ) assert detector_stage["state"] == "active" assert detector_stage["elapsed_seconds"] == 2.5 assert detector_stage["activations"] == 42 assert detector_stage["share_percent"] == 62.5 assert next( stage for stage in document["pipeline"]["stages"] if stage["id"] == "semantic-model" )["state"] == "active" assert next( stage for stage in document["pipeline"]["stages"] if stage["id"] == "preprocessing" )["share_percent"] is None def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile( tmp_path: Path, ) -> None: router = build_system_telemetry_router( root_provider=lambda: tmp_path / "system", probe_runner=lambda _: _probe(node_id="OTHER-NODE"), ) apply_profile = _endpoint(router, "/api/v1/system/worker-profile", "PUT") with pytest.raises(HTTPException) as error: apply_profile( WorkerConnectionProfilePut( revision=0, address="192.0.2.20", port=22, ) ) assert error.value.status_code == 409 assert WorkerProfileStore(tmp_path / "system").read().revision == 0 assert not WorkerProfileStore(tmp_path / "system").path.exists()