feat(system): add Worker 006 telemetry and network profile
This commit is contained in:
DCCONSTRUCTIONS
@@ -795,6 +795,10 @@ export default function App() {
|
||||
<StatusBadge tone="neutral">Offline evaluation</StatusBadge>
|
||||
) : activeDefinition.kind === "lab-archive" ? (
|
||||
null
|
||||
) : activeDefinition.kind === "compute-modules" ? (
|
||||
<StatusBadge tone="neutral">Живая телеметрия</StatusBadge>
|
||||
) : activeDefinition.kind === "network-monitor" ? (
|
||||
<StatusBadge tone="neutral">Живая сеть</StatusBadge>
|
||||
) : (
|
||||
<StatusBadge tone="warning">Интерфейс готов</StatusBadge>
|
||||
)
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
interface TelemetrySeriesProps {
|
||||
label: string;
|
||||
values: Array<number | null>;
|
||||
value: string;
|
||||
ceiling?: number;
|
||||
}
|
||||
|
||||
function linePoints(values: Array<number | null>, ceiling?: number): string {
|
||||
const finite = values.filter((value): value is number =>
|
||||
value !== null && Number.isFinite(value),
|
||||
);
|
||||
if (finite.length === 0) return "";
|
||||
const maximum = Math.max(ceiling ?? 0, ...finite, 1);
|
||||
const denominator = Math.max(1, values.length - 1);
|
||||
return values.map((value, index) => {
|
||||
const x = index / denominator * 100;
|
||||
const normalized = value === null ? 0 : Math.max(0, Math.min(maximum, value));
|
||||
const y = 36 - normalized / maximum * 34;
|
||||
return `${x.toFixed(2)},${y.toFixed(2)}`;
|
||||
}).join(" ");
|
||||
}
|
||||
|
||||
export function TelemetrySeries({
|
||||
label,
|
||||
values,
|
||||
value,
|
||||
ceiling,
|
||||
}: TelemetrySeriesProps) {
|
||||
const points = linePoints(values, ceiling);
|
||||
return (
|
||||
<div className="system-telemetry-series">
|
||||
<div>
|
||||
<span>{label}</span>
|
||||
<strong>{value}</strong>
|
||||
</div>
|
||||
<svg
|
||||
viewBox="0 0 100 38"
|
||||
preserveAspectRatio="none"
|
||||
role="img"
|
||||
aria-label={`${label}: ${value}`}
|
||||
>
|
||||
<path d="M0 37 H100" />
|
||||
{points ? <polyline points={points} /> : null}
|
||||
</svg>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { GlassSurface, StatusBadge } from "@nodedc/ui-react";
|
||||
|
||||
import type { WorkerRuntime } from "../../core/system/workerTelemetry";
|
||||
import {
|
||||
formatOptionalPercent,
|
||||
runtimeStateLabel,
|
||||
runtimeTone,
|
||||
} from "./systemFormat";
|
||||
|
||||
export function WorkerRuntimeCard({ runtime }: { runtime: WorkerRuntime }) {
|
||||
return (
|
||||
<GlassSurface
|
||||
className="worker-runtime-card"
|
||||
padding="md"
|
||||
tone={runtime.external ? "soft" : "default"}
|
||||
data-external={runtime.external ? "true" : undefined}
|
||||
>
|
||||
<header>
|
||||
<div>
|
||||
<span className="section-eyebrow">
|
||||
{runtime.external ? "ВНЕШНЯЯ НАГРУЗКА" : "MISSION CORE RUNTIME"}
|
||||
</span>
|
||||
<h3>{runtime.role}</h3>
|
||||
<code>{runtime.name}</code>
|
||||
</div>
|
||||
<StatusBadge tone={runtimeTone(runtime)}>
|
||||
{runtimeStateLabel(runtime)}
|
||||
</StatusBadge>
|
||||
</header>
|
||||
<dl>
|
||||
<div>
|
||||
<dt>CPU</dt>
|
||||
<dd>{formatOptionalPercent(runtime.cpu_percent)}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Память</dt>
|
||||
<dd>{runtime.memory_usage ?? "—"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Сеть I/O</dt>
|
||||
<dd>{runtime.network_io ?? "—"}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Процессы</dt>
|
||||
<dd>{runtime.pids ?? "—"}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</GlassSurface>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import type { StatusTone } from "@nodedc/ui-react";
|
||||
|
||||
import type { WorkerRuntime } from "../../core/system/workerTelemetry";
|
||||
|
||||
export function formatOptionalPercent(value: number | null | undefined): string {
|
||||
return typeof value === "number" && Number.isFinite(value)
|
||||
? `${new Intl.NumberFormat("ru-RU", { maximumFractionDigits: 1 }).format(value)}%`
|
||||
: "—";
|
||||
}
|
||||
|
||||
export function formatBytes(value: number | null | undefined): string {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||
const units = ["Б", "КиБ", "МиБ", "ГиБ", "ТиБ"];
|
||||
let current = Math.max(0, value);
|
||||
let unit = 0;
|
||||
while (current >= 1024 && unit < units.length - 1) {
|
||||
current /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${new Intl.NumberFormat("ru-RU", {
|
||||
maximumFractionDigits: current >= 100 ? 0 : 1,
|
||||
}).format(current)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
export function formatRate(value: number | null | undefined): string {
|
||||
const formatted = formatBytes(value);
|
||||
return formatted === "—" ? formatted : `${formatted}/с`;
|
||||
}
|
||||
|
||||
export function formatBitRate(value: number | null | undefined): string {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||
const units = ["бит/с", "Кбит/с", "Мбит/с", "Гбит/с"];
|
||||
let current = Math.max(0, value);
|
||||
let unit = 0;
|
||||
while (current >= 1000 && unit < units.length - 1) {
|
||||
current /= 1000;
|
||||
unit += 1;
|
||||
}
|
||||
return `${new Intl.NumberFormat("ru-RU", {
|
||||
maximumFractionDigits: current >= 100 ? 0 : 1,
|
||||
}).format(current)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
export function formatDuration(value: number | null | undefined): string {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||
const hours = Math.floor(value / 3600);
|
||||
if (hours < 24) return `${hours} ч`;
|
||||
return `${Math.floor(hours / 24)} д ${hours % 24} ч`;
|
||||
}
|
||||
|
||||
export function formatLatency(value: number | null | undefined): string {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||
return `${Math.round(value)} мс`;
|
||||
}
|
||||
|
||||
export function runtimeTone(runtime: WorkerRuntime): StatusTone {
|
||||
if (runtime.state !== "running") return "danger";
|
||||
if (runtime.health && runtime.health !== "healthy") return "warning";
|
||||
return runtime.external ? "neutral" : "success";
|
||||
}
|
||||
|
||||
export function runtimeStateLabel(runtime: WorkerRuntime): string {
|
||||
if (runtime.state !== "running") return "Недоступен";
|
||||
if (runtime.health === "healthy") return "Работает";
|
||||
if (runtime.health) return runtime.health;
|
||||
return "Запущен";
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
import {
|
||||
fetchWorkerTelemetry,
|
||||
type WorkerTelemetry,
|
||||
} from "./workerTelemetry";
|
||||
|
||||
export interface WorkerTelemetryState {
|
||||
telemetry: WorkerTelemetry | null;
|
||||
loading: boolean;
|
||||
error: string | null;
|
||||
refresh: () => void;
|
||||
}
|
||||
|
||||
export function useWorkerTelemetry(pollMilliseconds = 10_000): WorkerTelemetryState {
|
||||
const [telemetry, setTelemetry] = useState<WorkerTelemetry | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [generation, setGeneration] = useState(0);
|
||||
const refresh = useCallback(() => setGeneration((value) => value + 1), []);
|
||||
|
||||
useEffect(() => {
|
||||
const controller = new AbortController();
|
||||
setLoading(true);
|
||||
void fetchWorkerTelemetry(controller.signal)
|
||||
.then((document) => {
|
||||
if (controller.signal.aborted) return;
|
||||
setTelemetry(document);
|
||||
setError(null);
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
if (controller.signal.aborted) return;
|
||||
setError(reason instanceof Error ? reason.message : "Worker 006 не ответил.");
|
||||
})
|
||||
.finally(() => {
|
||||
if (!controller.signal.aborted) setLoading(false);
|
||||
});
|
||||
return () => controller.abort();
|
||||
}, [generation]);
|
||||
|
||||
useEffect(() => {
|
||||
const timer = window.setInterval(refresh, pollMilliseconds);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [pollMilliseconds, refresh]);
|
||||
|
||||
return { telemetry, loading, error, refresh };
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
export interface WorkerConnectionProfile {
|
||||
schema_version: "missioncore.worker-connection-profile/v1";
|
||||
profile_id: string;
|
||||
display_name: string;
|
||||
expected_node_id: string;
|
||||
ssh_host_alias: string;
|
||||
address: string;
|
||||
port: number;
|
||||
revision: number;
|
||||
updated_at_utc: string | null;
|
||||
}
|
||||
|
||||
export interface WorkerProbe {
|
||||
schema_version: "missioncore.worker-probe/v1";
|
||||
reachable: boolean;
|
||||
identity_matches: boolean;
|
||||
node_id: string | null;
|
||||
latency_ms: number | null;
|
||||
observed_at_utc: string;
|
||||
error_code: string | null;
|
||||
}
|
||||
|
||||
export interface WorkerProfileDocument {
|
||||
schema_version: "missioncore.worker-connection-profile/v1";
|
||||
profile: WorkerConnectionProfile;
|
||||
security: {
|
||||
transport: "ssh";
|
||||
host_key_policy: "strict-pinned";
|
||||
credentials_managed_by_ui: false;
|
||||
ssh_host_alias: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface WorkerRuntime {
|
||||
name: string;
|
||||
role: string;
|
||||
owner: "mission-core" | "external";
|
||||
external: boolean;
|
||||
image: string | null;
|
||||
state: string;
|
||||
health: string | null;
|
||||
cpu_percent: number | null;
|
||||
memory_percent: number | null;
|
||||
memory_usage: string | null;
|
||||
network_io: string | null;
|
||||
block_io: string | null;
|
||||
pids: number | null;
|
||||
}
|
||||
|
||||
export interface WorkerPipelineStage {
|
||||
id: string;
|
||||
label: string;
|
||||
state: "active" | "waiting" | "ready" | "unavailable";
|
||||
}
|
||||
|
||||
export interface WorkerTelemetryHistoryRow {
|
||||
observed_at_utc: string;
|
||||
cpu_percent: number | null;
|
||||
memory_percent: number | null;
|
||||
gpu_percent: number | null;
|
||||
gpu_memory_percent: number | null;
|
||||
network_receive_bytes_per_second: number | null;
|
||||
network_send_bytes_per_second: number | null;
|
||||
}
|
||||
|
||||
export interface WorkerNetworkInterface {
|
||||
name: string;
|
||||
description: string | null;
|
||||
status: string | null;
|
||||
link_speed_bps: number | null;
|
||||
mac_address: string | null;
|
||||
addresses: string[];
|
||||
received_bytes: number | null;
|
||||
sent_bytes: number | null;
|
||||
}
|
||||
|
||||
export interface WorkerTelemetry {
|
||||
schema_version: "missioncore.worker-telemetry/v1";
|
||||
profile: WorkerConnectionProfile;
|
||||
connection: WorkerProbe;
|
||||
node: {
|
||||
node_id: string;
|
||||
observed_at_utc: string;
|
||||
os: {
|
||||
caption?: string;
|
||||
version?: string;
|
||||
uptime_seconds?: number;
|
||||
};
|
||||
cpu: {
|
||||
name?: string;
|
||||
logical_processors?: number;
|
||||
load_percent?: number;
|
||||
};
|
||||
memory: {
|
||||
total_bytes: number | null;
|
||||
used_bytes: number | null;
|
||||
free_bytes: number | null;
|
||||
used_percent: number | null;
|
||||
};
|
||||
disks: Array<{
|
||||
name?: string;
|
||||
size_bytes?: number;
|
||||
free_bytes?: number;
|
||||
}>;
|
||||
gpu: {
|
||||
name?: string;
|
||||
utilization_percent?: number;
|
||||
memory_used_mib?: number;
|
||||
memory_total_mib?: number;
|
||||
power_watts?: number;
|
||||
temperature_celsius?: number;
|
||||
memory_used_percent?: number;
|
||||
} | null;
|
||||
triton: {
|
||||
ready: boolean;
|
||||
requests_succeeded: number | null;
|
||||
requests_failed: number | null;
|
||||
inferences: number | null;
|
||||
};
|
||||
} | null;
|
||||
runtimes: WorkerRuntime[];
|
||||
pipeline: {
|
||||
service_state: string;
|
||||
current_stage: string | null;
|
||||
active_request_id: string | null;
|
||||
active_frame_index: number | null;
|
||||
completed_runs: number | null;
|
||||
failed_runs: number | null;
|
||||
model_load_seconds: number | null;
|
||||
stages: WorkerPipelineStage[];
|
||||
};
|
||||
network: {
|
||||
interfaces: WorkerNetworkInterface[];
|
||||
aggregate: {
|
||||
received_bytes: number;
|
||||
sent_bytes: number;
|
||||
receive_bytes_per_second: number | null;
|
||||
send_bytes_per_second: number | null;
|
||||
} | null;
|
||||
};
|
||||
history: WorkerTelemetryHistoryRow[];
|
||||
}
|
||||
|
||||
export interface WorkerProfileMutation {
|
||||
revision: number;
|
||||
address: string;
|
||||
port: number;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function requireJsonRecord(value: unknown, schema: string): Record<string, unknown> {
|
||||
if (!isRecord(value) || value.schema_version !== schema) {
|
||||
throw new Error("Ответ системной телеметрии не соответствует контракту.");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function requestJson(
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
): Promise<unknown> {
|
||||
const response = await fetch(url, {
|
||||
...init,
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
...(init.body ? { "Content-Type": "application/json" } : {}),
|
||||
...init.headers,
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Системный API вернул HTTP ${response.status}.`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function fetchWorkerTelemetry(
|
||||
signal?: AbortSignal,
|
||||
): Promise<WorkerTelemetry> {
|
||||
const document = requireJsonRecord(
|
||||
await requestJson("/api/v1/system/worker-telemetry?history=90", {
|
||||
method: "GET",
|
||||
signal,
|
||||
}),
|
||||
"missioncore.worker-telemetry/v1",
|
||||
);
|
||||
if (
|
||||
!isRecord(document.connection)
|
||||
|| !isRecord(document.profile)
|
||||
|| !Array.isArray(document.runtimes)
|
||||
|| !isRecord(document.pipeline)
|
||||
|| !isRecord(document.network)
|
||||
|| !Array.isArray(document.history)
|
||||
) {
|
||||
throw new Error("Срез Worker 006 неполон.");
|
||||
}
|
||||
return document as unknown as WorkerTelemetry;
|
||||
}
|
||||
|
||||
export async function fetchWorkerProfile(
|
||||
signal?: AbortSignal,
|
||||
): Promise<WorkerProfileDocument> {
|
||||
const document = requireJsonRecord(
|
||||
await requestJson("/api/v1/system/worker-profile", {
|
||||
method: "GET",
|
||||
signal,
|
||||
}),
|
||||
"missioncore.worker-connection-profile/v1",
|
||||
);
|
||||
if (!isRecord(document.profile) || !isRecord(document.security)) {
|
||||
throw new Error("Профиль Worker 006 неполон.");
|
||||
}
|
||||
return document as unknown as WorkerProfileDocument;
|
||||
}
|
||||
|
||||
export async function testWorkerProfile(
|
||||
request: WorkerProfileMutation,
|
||||
signal?: AbortSignal,
|
||||
): Promise<WorkerProbe> {
|
||||
const document = requireJsonRecord(
|
||||
await requestJson("/api/v1/system/worker-profile/test", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(request),
|
||||
signal,
|
||||
}),
|
||||
"missioncore.worker-probe/v1",
|
||||
);
|
||||
return document as unknown as WorkerProbe;
|
||||
}
|
||||
|
||||
export async function saveWorkerProfile(
|
||||
request: WorkerProfileMutation,
|
||||
signal?: AbortSignal,
|
||||
): Promise<WorkerProfileDocument & { verification: WorkerProbe }> {
|
||||
const document = requireJsonRecord(
|
||||
await requestJson("/api/v1/system/worker-profile", {
|
||||
method: "PUT",
|
||||
body: JSON.stringify(request),
|
||||
signal,
|
||||
}),
|
||||
"missioncore.worker-connection-profile/v1",
|
||||
);
|
||||
if (!isRecord(document.profile) || !isRecord(document.verification)) {
|
||||
throw new Error("Сохранённый профиль Worker 006 неполон.");
|
||||
}
|
||||
return document as unknown as WorkerProfileDocument & { verification: WorkerProbe };
|
||||
}
|
||||
@@ -18,6 +18,8 @@ export type WorkspaceKind =
|
||||
| "missions"
|
||||
| "catalog"
|
||||
| "contour-health"
|
||||
| "compute-modules"
|
||||
| "network-monitor"
|
||||
| "datasets"
|
||||
| "lab-archive";
|
||||
|
||||
@@ -589,16 +591,17 @@ export const workspaces: WorkspaceDefinition[] = [
|
||||
label: "Вычислительные модули",
|
||||
title: "Вычислительные модули",
|
||||
eyebrow: "СИСТЕМА / МОДУЛИ",
|
||||
description: "Контракты будущих вычислительных модулей без их запуска на текущем интерфейсном этапе.",
|
||||
description: "Живая аппаратная и процессинговая телеметрия выделенного Worker 006.",
|
||||
icon: "apps",
|
||||
kind: "catalog",
|
||||
kind: "compute-modules",
|
||||
groups: [
|
||||
{
|
||||
title: "Вычислительные роли",
|
||||
description: "Каждый модуль публикует стандартные выходы, интерфейс только отображает их.",
|
||||
capabilities: [
|
||||
active("Адаптер устройства", "Закрытый протокол текущего устройства уже изолирован."),
|
||||
contract("Восприятие", "Рамки, маски, ключевые точки, траектории и семантические подписи."),
|
||||
active("Восприятие", "Persistent perception worker и его текущие стадии обработки."),
|
||||
active("Inference runtime", "Triton health, inference-счётчики и потребление ресурсов."),
|
||||
contract("Картирование", "Высота, занятость, сетки и проходимость."),
|
||||
contract("Локализация", "Поза, одометрия, TF и неопределённость."),
|
||||
contract("Исполнитель миссий", "Состояние графа миссии и действия верхнего уровня."),
|
||||
@@ -637,13 +640,14 @@ export const workspaces: WorkspaceDefinition[] = [
|
||||
eyebrow: "СИСТЕМА / СЕТЬ",
|
||||
description: "Локальный контур, бортовая сеть, ячеистая сеть/LTE и серверные маршруты.",
|
||||
icon: "globe",
|
||||
kind: "catalog",
|
||||
kind: "network-monitor",
|
||||
groups: [
|
||||
{
|
||||
title: "Сетевые зоны",
|
||||
description: "Потоки управления и высокой пропускной способности разделяются архитектурно.",
|
||||
capabilities: [
|
||||
active("Локальная сеть", "Устройство и рабочая станция в одной доступной сети."),
|
||||
active("Переносимый профиль", "Адрес и порт Worker 006 проверяются до применения."),
|
||||
active("Сетевая телеметрия", "Интерфейсы, счётчики, throughput и фактический маршрут."),
|
||||
active("Локальное управление", "Точка приёма учётных данных и браузерный контур управления ограничены локальным хостом."),
|
||||
contract("Бортовая сеть", "Сенсоры, вычислитель и маршрутизатор внутри аппарата."),
|
||||
contract("Дальний транспорт", "Ячеистая сеть или LTE передаёт предпросмотр, позу, события и изменения карты."),
|
||||
|
||||
@@ -11,3 +11,4 @@
|
||||
@import "./styles/responsive.css";
|
||||
@import "./styles/observation.css";
|
||||
@import "./styles/environment-settings.css";
|
||||
@import "./styles/system-telemetry.css";
|
||||
|
||||
@@ -0,0 +1,544 @@
|
||||
.system-workspace {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 1rem;
|
||||
padding-bottom: 1rem;
|
||||
}
|
||||
|
||||
.system-workspace__lead,
|
||||
.system-section-heading {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 1.2rem;
|
||||
}
|
||||
|
||||
.system-workspace__lead {
|
||||
padding: 0.25rem 0 0.35rem;
|
||||
}
|
||||
|
||||
.system-workspace__lead h2,
|
||||
.system-section-heading h3 {
|
||||
margin: 0.4rem 0 0;
|
||||
color: var(--nodedc-text-primary);
|
||||
letter-spacing: -0.035em;
|
||||
}
|
||||
|
||||
.system-workspace__lead h2 {
|
||||
font-size: 1.42rem;
|
||||
}
|
||||
|
||||
.system-section-heading h3 {
|
||||
font-size: 1.02rem;
|
||||
}
|
||||
|
||||
.system-workspace__lead p,
|
||||
.system-section-heading p {
|
||||
max-width: 48rem;
|
||||
margin: 0.42rem 0 0;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.68rem;
|
||||
line-height: 1.48;
|
||||
}
|
||||
|
||||
.system-workspace__actions {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
align-items: center;
|
||||
gap: 0.8rem;
|
||||
}
|
||||
|
||||
.system-workspace__notice {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.system-telemetry-grid,
|
||||
.network-overview-grid {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 0.65rem;
|
||||
}
|
||||
|
||||
.system-telemetry-series,
|
||||
.network-stat-card {
|
||||
min-width: 0;
|
||||
border: 1px solid var(--station-hairline);
|
||||
border-radius: var(--nodedc-radius-card);
|
||||
background: var(--station-panel-soft);
|
||||
}
|
||||
|
||||
.system-telemetry-series {
|
||||
overflow: hidden;
|
||||
padding: 0.82rem 0.82rem 0.32rem;
|
||||
}
|
||||
|
||||
.system-telemetry-series > div {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.system-telemetry-series span,
|
||||
.network-stat-card span {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.62rem;
|
||||
}
|
||||
|
||||
.system-telemetry-series strong,
|
||||
.network-stat-card strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 1.14rem;
|
||||
font-weight: 720;
|
||||
letter-spacing: -0.035em;
|
||||
}
|
||||
|
||||
.system-telemetry-series svg {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 2.45rem;
|
||||
margin-top: 0.45rem;
|
||||
}
|
||||
|
||||
.system-telemetry-series path {
|
||||
fill: none;
|
||||
stroke: var(--station-hairline);
|
||||
stroke-width: 0.8;
|
||||
vector-effect: non-scaling-stroke;
|
||||
}
|
||||
|
||||
.system-telemetry-series polyline {
|
||||
fill: none;
|
||||
stroke: var(--nodedc-text-primary);
|
||||
stroke-linecap: round;
|
||||
stroke-linejoin: round;
|
||||
stroke-width: 1.15;
|
||||
vector-effect: non-scaling-stroke;
|
||||
}
|
||||
|
||||
.worker-hardware,
|
||||
.worker-pipeline,
|
||||
.network-profile,
|
||||
.network-topology {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.worker-hardware__facts {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 0.7rem;
|
||||
}
|
||||
|
||||
.worker-hardware__facts dl,
|
||||
.worker-runtime-card dl,
|
||||
.network-interface-card dl {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
margin: 0;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.worker-hardware__facts dl {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
border: 1px solid var(--station-hairline);
|
||||
border-radius: var(--nodedc-radius-control);
|
||||
}
|
||||
|
||||
.worker-hardware__facts dl > div,
|
||||
.worker-runtime-card dl > div,
|
||||
.network-interface-card dl > div {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.22rem;
|
||||
}
|
||||
|
||||
.worker-hardware__facts dl > div {
|
||||
padding: 0.75rem;
|
||||
}
|
||||
|
||||
.worker-hardware__facts dt,
|
||||
.worker-runtime-card dt,
|
||||
.network-interface-card dt,
|
||||
.network-profile__security dt {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.58rem;
|
||||
}
|
||||
|
||||
.worker-hardware__facts dd,
|
||||
.worker-runtime-card dd,
|
||||
.network-interface-card dd,
|
||||
.network-profile__security dd {
|
||||
min-width: 0;
|
||||
margin: 0;
|
||||
overflow: hidden;
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.67rem;
|
||||
font-weight: 650;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.worker-disk-list {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
|
||||
gap: 0.55rem;
|
||||
}
|
||||
|
||||
.worker-disk-list > div {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 0.8rem;
|
||||
padding: 0.62rem 0.72rem;
|
||||
border-radius: var(--nodedc-radius-control);
|
||||
background: var(--station-accent-soft);
|
||||
}
|
||||
|
||||
.worker-disk-list span {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.61rem;
|
||||
}
|
||||
|
||||
.worker-disk-list strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.64rem;
|
||||
}
|
||||
|
||||
.system-runtime-section {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.system-runtime-section--external {
|
||||
padding-top: 0.25rem;
|
||||
border-top: 1px solid var(--station-hairline);
|
||||
}
|
||||
|
||||
.worker-runtime-grid {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 0.7rem;
|
||||
}
|
||||
|
||||
.worker-runtime-card {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.9rem;
|
||||
}
|
||||
|
||||
.worker-runtime-card[data-external="true"] {
|
||||
opacity: 0.78;
|
||||
}
|
||||
|
||||
.worker-runtime-card > header,
|
||||
.network-interface-card > header {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 0.8rem;
|
||||
}
|
||||
|
||||
.worker-runtime-card h3 {
|
||||
margin: 0.38rem 0 0.18rem;
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.9rem;
|
||||
letter-spacing: -0.025em;
|
||||
}
|
||||
|
||||
.worker-runtime-card code {
|
||||
display: block;
|
||||
max-width: 23rem;
|
||||
overflow: hidden;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.56rem;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.worker-runtime-card dl {
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.worker-runtime-card dl > div {
|
||||
padding-right: 0.65rem;
|
||||
}
|
||||
|
||||
.worker-pipeline__summary {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 0.55rem;
|
||||
}
|
||||
|
||||
.worker-pipeline__summary > div {
|
||||
display: grid;
|
||||
gap: 0.32rem;
|
||||
padding: 0.7rem;
|
||||
border-radius: var(--nodedc-radius-control);
|
||||
background: var(--station-panel-soft);
|
||||
}
|
||||
|
||||
.worker-pipeline__summary span {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.57rem;
|
||||
}
|
||||
|
||||
.worker-pipeline__summary strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.worker-stage-list {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
gap: 0.48rem;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.worker-stage-list li {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: 0.55rem;
|
||||
padding: 0.66rem;
|
||||
border: 1px solid var(--station-hairline);
|
||||
border-radius: var(--nodedc-radius-control);
|
||||
background: var(--station-panel-soft);
|
||||
}
|
||||
|
||||
.worker-stage-list li[data-state="active"] {
|
||||
border-color: var(--station-hairline-strong);
|
||||
background: var(--nodedc-focus-surface);
|
||||
}
|
||||
|
||||
.worker-stage-list li > span,
|
||||
.worker-stage-list li > small {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.53rem;
|
||||
}
|
||||
|
||||
.worker-stage-list li > strong {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.62rem;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.network-stat-card {
|
||||
display: grid;
|
||||
align-content: center;
|
||||
gap: 0.3rem;
|
||||
padding: 0.82rem;
|
||||
}
|
||||
|
||||
.network-stat-card small {
|
||||
overflow: hidden;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.55rem;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.network-profile__form {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: minmax(12rem, 1.5fr) minmax(8rem, 0.5fr) auto;
|
||||
align-items: end;
|
||||
gap: 0.7rem;
|
||||
}
|
||||
|
||||
.network-profile__buttons {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding-bottom: 0.02rem;
|
||||
}
|
||||
|
||||
.network-profile__security {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
margin: 0;
|
||||
gap: 0.55rem;
|
||||
}
|
||||
|
||||
.network-profile__security > div {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.24rem;
|
||||
padding: 0.64rem 0.7rem;
|
||||
border-radius: var(--nodedc-radius-control);
|
||||
background: var(--station-accent-soft);
|
||||
}
|
||||
|
||||
.network-route {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: repeat(3, minmax(8rem, 1fr) 2.2rem) minmax(8rem, 1fr);
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.network-route > div {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.3rem;
|
||||
padding: 0.78rem;
|
||||
border: 1px solid var(--station-hairline);
|
||||
border-radius: var(--nodedc-radius-control);
|
||||
background: var(--station-panel-soft);
|
||||
}
|
||||
|
||||
.network-route > div[data-state="offline"] {
|
||||
opacity: 0.48;
|
||||
}
|
||||
|
||||
.network-route > div span,
|
||||
.network-route > div small {
|
||||
overflow: hidden;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.55rem;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.network-route > div strong {
|
||||
overflow: hidden;
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.68rem;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.network-route > i {
|
||||
height: 1px;
|
||||
background: var(--station-hairline-strong);
|
||||
}
|
||||
|
||||
.network-route > i[data-state="offline"] {
|
||||
background: var(--station-hairline);
|
||||
opacity: 0.4;
|
||||
}
|
||||
|
||||
.network-interface-section {
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.network-interface-list {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 0.65rem;
|
||||
}
|
||||
|
||||
.network-interface-card {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.85rem;
|
||||
}
|
||||
|
||||
.network-interface-card header strong,
|
||||
.network-interface-card header small {
|
||||
display: block;
|
||||
max-width: 17rem;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.network-interface-card header strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.71rem;
|
||||
}
|
||||
|
||||
.network-interface-card header small {
|
||||
margin-top: 0.25rem;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.55rem;
|
||||
}
|
||||
|
||||
.network-interface-card dl {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 0.65rem;
|
||||
}
|
||||
|
||||
@media (max-width: 1100px) {
|
||||
.system-telemetry-grid,
|
||||
.network-overview-grid,
|
||||
.worker-pipeline__summary,
|
||||
.network-profile__security {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.worker-stage-list,
|
||||
.network-interface-list {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.network-profile__form {
|
||||
grid-template-columns: minmax(12rem, 1fr) 8rem;
|
||||
}
|
||||
|
||||
.network-profile__buttons {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.network-route {
|
||||
grid-template-columns: 1fr;
|
||||
gap: 0.4rem;
|
||||
}
|
||||
|
||||
.network-route > i {
|
||||
width: 1px;
|
||||
height: 1.2rem;
|
||||
margin-left: 1.1rem;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.system-workspace__lead,
|
||||
.system-section-heading {
|
||||
display: grid;
|
||||
}
|
||||
|
||||
.system-workspace__actions {
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.system-telemetry-grid,
|
||||
.network-overview-grid,
|
||||
.worker-hardware__facts,
|
||||
.worker-runtime-grid,
|
||||
.worker-stage-list,
|
||||
.network-interface-list,
|
||||
.network-profile__security {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.worker-runtime-card dl,
|
||||
.worker-pipeline__summary {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.network-profile__form {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.network-profile__buttons {
|
||||
grid-column: auto;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,8 @@ import type { WorkspaceRendererProps } from "./contracts";
|
||||
import { DatasetGatewayWorkspace } from "./DatasetGatewayWorkspace";
|
||||
import { ContourHealthWorkspace } from "./ContourHealthWorkspace";
|
||||
import { LaboratoryArchiveWorkspace } from "./laboratory/LaboratoryArchiveWorkspace";
|
||||
import { ComputeModulesWorkspace } from "./system/ComputeModulesWorkspace";
|
||||
import { NetworkWorkspace } from "./system/NetworkWorkspace";
|
||||
|
||||
function statusTone(status: CapabilityStatus): "success" | "accent" | "warning" | "neutral" {
|
||||
if (status === "active") return "success";
|
||||
@@ -1177,6 +1179,10 @@ export function WorkspaceRenderer(props: WorkspaceRendererProps) {
|
||||
state={props.state}
|
||||
/>
|
||||
);
|
||||
case "compute-modules":
|
||||
return <ComputeModulesWorkspace />;
|
||||
case "network-monitor":
|
||||
return <NetworkWorkspace />;
|
||||
case "datasets":
|
||||
return <DatasetGatewayWorkspace />;
|
||||
case "lab-archive":
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import {
|
||||
Button,
|
||||
GlassSurface,
|
||||
Icon,
|
||||
StatusBadge,
|
||||
} from "@nodedc/ui-react";
|
||||
|
||||
import { TelemetrySeries } from "../../components/system/TelemetrySeries";
|
||||
import { WorkerRuntimeCard } from "../../components/system/WorkerRuntimeCard";
|
||||
import {
|
||||
formatBytes,
|
||||
formatDuration,
|
||||
formatOptionalPercent,
|
||||
} from "../../components/system/systemFormat";
|
||||
import { useWorkerTelemetry } from "../../core/system/useWorkerTelemetry";
|
||||
|
||||
function pipelineStateLabel(state: string): string {
|
||||
if (state === "busy") return "Выполняет задачу";
|
||||
if (state === "ready") return "Готов к задаче";
|
||||
return "Нет live-состояния";
|
||||
}
|
||||
|
||||
export function ComputeModulesWorkspace() {
|
||||
const { telemetry, loading, error, refresh } = useWorkerTelemetry();
|
||||
const node = telemetry?.node ?? null;
|
||||
const missionCoreRuntimes = telemetry?.runtimes.filter((runtime) => !runtime.external) ?? [];
|
||||
const externalRuntimes = telemetry?.runtimes.filter((runtime) => runtime.external) ?? [];
|
||||
const connected = Boolean(
|
||||
telemetry?.connection.reachable && telemetry.connection.identity_matches && node,
|
||||
);
|
||||
const history = telemetry?.history ?? [];
|
||||
const cpuPercent = node?.cpu.load_percent;
|
||||
const memoryPercent = node?.memory.used_percent;
|
||||
const gpuPercent = node?.gpu?.utilization_percent;
|
||||
const gpuMemoryPercent = node?.gpu?.memory_used_percent;
|
||||
|
||||
return (
|
||||
<div className="system-workspace compute-modules-workspace">
|
||||
<section className="system-workspace__lead">
|
||||
<div>
|
||||
<span className="section-eyebrow">СИСТЕМА / ВЫЧИСЛИТЕЛЬНЫЕ МОДУЛИ</span>
|
||||
<h2>Worker 006</h2>
|
||||
<p>
|
||||
Живой аппаратный и процессинговый срез выделенного узла. Нагрузка внешних
|
||||
сервисов отделена от Mission Core и не входит в оценку наших runtime.
|
||||
</p>
|
||||
</div>
|
||||
<div className="system-workspace__actions">
|
||||
<StatusBadge tone={connected ? "success" : "danger"}>
|
||||
{connected ? "Узел доступен" : "Нет связи с узлом"}
|
||||
</StatusBadge>
|
||||
<Button
|
||||
size="compact"
|
||||
variant="secondary"
|
||||
disabled={loading}
|
||||
onClick={refresh}
|
||||
icon={<Icon name="refresh" size={14} />}
|
||||
>
|
||||
{loading ? "Читаем" : "Обновить"}
|
||||
</Button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{error ? (
|
||||
<GlassSurface className="system-workspace__notice" padding="md" tone="soft">
|
||||
<StatusBadge tone="warning">{error}</StatusBadge>
|
||||
</GlassSurface>
|
||||
) : null}
|
||||
|
||||
<section className="system-telemetry-grid" aria-label="Аппаратная телеметрия">
|
||||
<TelemetrySeries
|
||||
label="CPU"
|
||||
value={formatOptionalPercent(cpuPercent)}
|
||||
ceiling={100}
|
||||
values={history.map((item) => item.cpu_percent)}
|
||||
/>
|
||||
<TelemetrySeries
|
||||
label="RAM"
|
||||
value={formatOptionalPercent(memoryPercent)}
|
||||
ceiling={100}
|
||||
values={history.map((item) => item.memory_percent)}
|
||||
/>
|
||||
<TelemetrySeries
|
||||
label="GPU"
|
||||
value={formatOptionalPercent(gpuPercent)}
|
||||
ceiling={100}
|
||||
values={history.map((item) => item.gpu_percent)}
|
||||
/>
|
||||
<TelemetrySeries
|
||||
label="VRAM"
|
||||
value={formatOptionalPercent(gpuMemoryPercent)}
|
||||
ceiling={100}
|
||||
values={history.map((item) => item.gpu_memory_percent)}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<GlassSurface className="worker-hardware" padding="lg">
|
||||
<header className="system-section-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">HARDWARE</span>
|
||||
<h3>{node?.node_id ?? telemetry?.profile.expected_node_id ?? "Worker 006"}</h3>
|
||||
<p>{node?.os.caption ?? "Аппаратный профиль недоступен"}</p>
|
||||
</div>
|
||||
<StatusBadge tone={node?.node_id ? "success" : "danger"}>
|
||||
{node?.node_id ? telemetry?.profile.display_name : "Нет данных"}
|
||||
</StatusBadge>
|
||||
</header>
|
||||
<div className="worker-hardware__facts">
|
||||
<dl>
|
||||
<div><dt>Процессор</dt><dd>{node?.cpu.name ?? "—"}</dd></div>
|
||||
<div><dt>Логические ядра</dt><dd>{node?.cpu.logical_processors ?? "—"}</dd></div>
|
||||
<div><dt>Память занята</dt><dd>{formatBytes(node?.memory.used_bytes)}</dd></div>
|
||||
<div><dt>Uptime</dt><dd>{formatDuration(node?.os.uptime_seconds)}</dd></div>
|
||||
</dl>
|
||||
<dl>
|
||||
<div><dt>GPU</dt><dd>{node?.gpu?.name ?? "—"}</dd></div>
|
||||
<div><dt>VRAM занята</dt><dd>{formatBytes(
|
||||
typeof node?.gpu?.memory_used_mib === "number"
|
||||
? node.gpu.memory_used_mib * 1024 * 1024
|
||||
: null,
|
||||
)}</dd></div>
|
||||
<div><dt>Температура</dt><dd>{
|
||||
typeof node?.gpu?.temperature_celsius === "number"
|
||||
? `${node.gpu.temperature_celsius} °C`
|
||||
: "—"
|
||||
}</dd></div>
|
||||
<div><dt>Мощность</dt><dd>{
|
||||
typeof node?.gpu?.power_watts === "number"
|
||||
? `${node.gpu.power_watts.toFixed(1)} Вт`
|
||||
: "—"
|
||||
}</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
<div className="worker-disk-list">
|
||||
{(node?.disks ?? []).map((disk) => {
|
||||
const size = typeof disk.size_bytes === "number" ? disk.size_bytes : null;
|
||||
const free = typeof disk.free_bytes === "number" ? disk.free_bytes : null;
|
||||
const used = size !== null && free !== null ? size - free : null;
|
||||
return (
|
||||
<div key={disk.name ?? "disk"}>
|
||||
<span>Диск {disk.name ?? "—"}</span>
|
||||
<strong>{formatBytes(used)} / {formatBytes(size)}</strong>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</GlassSurface>
|
||||
|
||||
<section className="system-runtime-section">
|
||||
<header className="system-section-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">PROCESSING RUNTIME</span>
|
||||
<h3>Контейнеры Mission Core</h3>
|
||||
<p>Два ограниченных runtime: inference server и прикладной perception pipeline.</p>
|
||||
</div>
|
||||
<StatusBadge tone={node?.triton.ready ? "success" : "danger"}>
|
||||
{node?.triton.ready ? "Triton ready" : "Triton недоступен"}
|
||||
</StatusBadge>
|
||||
</header>
|
||||
<div className="worker-runtime-grid">
|
||||
{missionCoreRuntimes.map((runtime) => (
|
||||
<WorkerRuntimeCard key={runtime.name} runtime={runtime} />
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<GlassSurface className="worker-pipeline" padding="lg">
|
||||
<header className="system-section-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">ТЕКУЩАЯ ЗАДАЧА</span>
|
||||
<h3>{pipelineStateLabel(telemetry?.pipeline.service_state ?? "unavailable")}</h3>
|
||||
<p>
|
||||
{telemetry?.pipeline.active_request_id
|
||||
? `Run ${telemetry.pipeline.active_request_id}`
|
||||
: "Очередь свободна; модели остаются загруженными в persistent worker."}
|
||||
</p>
|
||||
</div>
|
||||
<StatusBadge tone={
|
||||
telemetry?.pipeline.service_state === "busy" ? "warning"
|
||||
: telemetry?.pipeline.service_state === "ready" ? "success"
|
||||
: "danger"
|
||||
}>
|
||||
{telemetry?.pipeline.service_state ?? "unavailable"}
|
||||
</StatusBadge>
|
||||
</header>
|
||||
<div className="worker-pipeline__summary">
|
||||
<div><span>Завершено запусков</span><strong>{telemetry?.pipeline.completed_runs ?? "—"}</strong></div>
|
||||
<div><span>Ошибок запусков</span><strong>{telemetry?.pipeline.failed_runs ?? "—"}</strong></div>
|
||||
<div><span>Inference success</span><strong>{node?.triton.requests_succeeded ?? "—"}</strong></div>
|
||||
<div><span>Inference failed</span><strong>{node?.triton.requests_failed ?? "—"}</strong></div>
|
||||
</div>
|
||||
<ol className="worker-stage-list">
|
||||
{(telemetry?.pipeline.stages ?? []).map((stage, index) => (
|
||||
<li key={stage.id} data-state={stage.state}>
|
||||
<span>{String(index + 1).padStart(2, "0")}</span>
|
||||
<strong>{stage.label}</strong>
|
||||
<small>{
|
||||
stage.state === "active" ? "выполняется"
|
||||
: stage.state === "waiting" ? "в очереди"
|
||||
: stage.state === "ready" ? "готов"
|
||||
: "нет данных"
|
||||
}</small>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
</GlassSurface>
|
||||
|
||||
<section className="system-runtime-section system-runtime-section--external">
|
||||
<header className="system-section-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">CO-TENANTS / НЕ MISSION CORE</span>
|
||||
<h3>Внешняя нагрузка узла</h3>
|
||||
<p>
|
||||
Эти процессы влияют на общую плату и VRAM, но не считаются расходом Mission Core.
|
||||
</p>
|
||||
</div>
|
||||
</header>
|
||||
<div className="worker-runtime-grid">
|
||||
{externalRuntimes.map((runtime) => (
|
||||
<WorkerRuntimeCard key={runtime.name} runtime={runtime} />
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import {
|
||||
Button,
|
||||
GlassSurface,
|
||||
Icon,
|
||||
StatusBadge,
|
||||
TextField,
|
||||
} from "@nodedc/ui-react";
|
||||
|
||||
import { TelemetrySeries } from "../../components/system/TelemetrySeries";
|
||||
import {
|
||||
formatBitRate,
|
||||
formatBytes,
|
||||
formatLatency,
|
||||
formatRate,
|
||||
} from "../../components/system/systemFormat";
|
||||
import {
|
||||
fetchWorkerProfile,
|
||||
saveWorkerProfile,
|
||||
testWorkerProfile,
|
||||
type WorkerConnectionProfile,
|
||||
type WorkerProbe,
|
||||
} from "../../core/system/workerTelemetry";
|
||||
import { useWorkerTelemetry } from "../../core/system/useWorkerTelemetry";
|
||||
|
||||
type ProfileAction = "idle" | "testing" | "saving";
|
||||
|
||||
function probeLabel(probe: WorkerProbe | null): string {
|
||||
if (!probe) return "Изменения не проверены";
|
||||
if (!probe.reachable) return "Адрес не отвечает";
|
||||
if (!probe.identity_matches) return "Ответил другой узел";
|
||||
return "Worker 006 подтверждён";
|
||||
}
|
||||
|
||||
export function NetworkWorkspace() {
|
||||
const { telemetry, loading, error, refresh } = useWorkerTelemetry();
|
||||
const [profile, setProfile] = useState<WorkerConnectionProfile | null>(null);
|
||||
const [address, setAddress] = useState("");
|
||||
const [port, setPort] = useState("22");
|
||||
const [profileAction, setProfileAction] = useState<ProfileAction>("idle");
|
||||
const [probe, setProbe] = useState<WorkerProbe | null>(null);
|
||||
const [profileError, setProfileError] = useState<string | null>(null);
|
||||
const aggregate = telemetry?.network.aggregate ?? null;
|
||||
const connected = Boolean(
|
||||
telemetry?.connection.reachable && telemetry.connection.identity_matches,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const controller = new AbortController();
|
||||
void fetchWorkerProfile(controller.signal)
|
||||
.then((document) => {
|
||||
if (controller.signal.aborted) return;
|
||||
setProfile(document.profile);
|
||||
setAddress(document.profile.address);
|
||||
setPort(String(document.profile.port));
|
||||
setProfileError(null);
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
if (controller.signal.aborted) return;
|
||||
setProfileError(reason instanceof Error ? reason.message : "Профиль не прочитан.");
|
||||
});
|
||||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const mutation = useMemo(() => {
|
||||
const numericPort = Number(port);
|
||||
if (!profile || !Number.isInteger(numericPort) || numericPort < 1 || numericPort > 65535) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
revision: profile.revision,
|
||||
address: address.trim(),
|
||||
port: numericPort,
|
||||
};
|
||||
}, [address, port, profile]);
|
||||
|
||||
const runTest = () => {
|
||||
if (!mutation) return;
|
||||
const controller = new AbortController();
|
||||
setProfileAction("testing");
|
||||
setProfileError(null);
|
||||
setProbe(null);
|
||||
void testWorkerProfile(mutation, controller.signal)
|
||||
.then(setProbe)
|
||||
.catch((reason: unknown) => {
|
||||
setProfileError(reason instanceof Error ? reason.message : "Проверка не выполнена.");
|
||||
})
|
||||
.finally(() => setProfileAction("idle"));
|
||||
};
|
||||
|
||||
const saveProfile = () => {
|
||||
if (!mutation) return;
|
||||
const controller = new AbortController();
|
||||
setProfileAction("saving");
|
||||
setProfileError(null);
|
||||
void saveWorkerProfile(mutation, controller.signal)
|
||||
.then((document) => {
|
||||
setProfile(document.profile);
|
||||
setAddress(document.profile.address);
|
||||
setPort(String(document.profile.port));
|
||||
setProbe(document.verification);
|
||||
refresh();
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
setProfileError(reason instanceof Error ? reason.message : "Профиль не сохранён.");
|
||||
})
|
||||
.finally(() => setProfileAction("idle"));
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="system-workspace network-workspace">
|
||||
<section className="system-workspace__lead">
|
||||
<div>
|
||||
<span className="section-eyebrow">СИСТЕМА / СЕТЬ</span>
|
||||
<h2>Локальный вычислительный контур</h2>
|
||||
<p>
|
||||
Переносимый профиль связи между Mission Core и Worker 006. Адрес можно заменить
|
||||
при переходе в другую локальную сеть; идентичность узла проверяется до сохранения.
|
||||
</p>
|
||||
</div>
|
||||
<div className="system-workspace__actions">
|
||||
<StatusBadge tone={connected ? "success" : "danger"}>
|
||||
{connected ? "Маршрут доступен" : "Маршрут недоступен"}
|
||||
</StatusBadge>
|
||||
<Button
|
||||
size="compact"
|
||||
variant="secondary"
|
||||
disabled={loading}
|
||||
onClick={refresh}
|
||||
icon={<Icon name="refresh" size={14} />}
|
||||
>
|
||||
{loading ? "Читаем" : "Обновить"}
|
||||
</Button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{(error || profileError) ? (
|
||||
<GlassSurface className="system-workspace__notice" padding="md" tone="soft">
|
||||
<StatusBadge tone="warning">{profileError ?? error}</StatusBadge>
|
||||
</GlassSurface>
|
||||
) : null}
|
||||
|
||||
<section className="network-overview-grid" aria-label="Сетевая телеметрия">
|
||||
<TelemetrySeries
|
||||
label="Приём"
|
||||
value={formatRate(aggregate?.receive_bytes_per_second)}
|
||||
values={(telemetry?.history ?? []).map(
|
||||
(item) => item.network_receive_bytes_per_second,
|
||||
)}
|
||||
/>
|
||||
<TelemetrySeries
|
||||
label="Передача"
|
||||
value={formatRate(aggregate?.send_bytes_per_second)}
|
||||
values={(telemetry?.history ?? []).map(
|
||||
(item) => item.network_send_bytes_per_second,
|
||||
)}
|
||||
/>
|
||||
<div className="network-stat-card">
|
||||
<span>Сбор телеметрии</span>
|
||||
<strong>{formatLatency(telemetry?.connection.latency_ms)}</strong>
|
||||
<small>полный SSH probe Worker 006</small>
|
||||
</div>
|
||||
<div className="network-stat-card">
|
||||
<span>Активные интерфейсы</span>
|
||||
<strong>{telemetry?.network.interfaces.length ?? "—"}</strong>
|
||||
<small>по Windows network counters</small>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<GlassSurface className="network-profile" padding="lg">
|
||||
<header className="system-section-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">ПРОФИЛЬ ПОДКЛЮЧЕНИЯ</span>
|
||||
<h3>Worker 006</h3>
|
||||
<p>
|
||||
Реальный Node ID: <code>{profile?.expected_node_id ?? "DESKTOP-OPJ8J04"}</code>.
|
||||
Пустой адрес использует закреплённый локальный SSH-профиль.
|
||||
</p>
|
||||
</div>
|
||||
<StatusBadge tone={
|
||||
probe?.reachable && probe.identity_matches ? "success"
|
||||
: probe ? "danger"
|
||||
: connected ? "success" : "neutral"
|
||||
}>
|
||||
{probe ? probeLabel(probe) : connected ? "Текущий профиль работает" : "Не проверено"}
|
||||
</StatusBadge>
|
||||
</header>
|
||||
<div className="network-profile__form">
|
||||
<TextField
|
||||
label="Адрес Worker 006"
|
||||
hint="IP или hostname"
|
||||
value={address}
|
||||
placeholder="из SSH-профиля mission-gpu"
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
onChange={(event) => {
|
||||
setAddress(event.target.value);
|
||||
setProbe(null);
|
||||
}}
|
||||
/>
|
||||
<TextField
|
||||
label="SSH-порт"
|
||||
hint="1–65535"
|
||||
type="number"
|
||||
min={1}
|
||||
max={65535}
|
||||
value={port}
|
||||
onChange={(event) => {
|
||||
setPort(event.target.value);
|
||||
setProbe(null);
|
||||
}}
|
||||
/>
|
||||
<div className="network-profile__buttons">
|
||||
<Button
|
||||
variant="secondary"
|
||||
disabled={!mutation || profileAction !== "idle"}
|
||||
onClick={runTest}
|
||||
>
|
||||
{profileAction === "testing" ? "Проверяем" : "Проверить"}
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
disabled={!mutation || profileAction !== "idle"}
|
||||
onClick={saveProfile}
|
||||
>
|
||||
{profileAction === "saving" ? "Сохраняем" : "Проверить и применить"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<dl className="network-profile__security">
|
||||
<div><dt>Транспорт</dt><dd>SSH · key-only</dd></div>
|
||||
<div><dt>Host key</dt><dd>strict · pinned</dd></div>
|
||||
<div><dt>Профиль</dt><dd>{profile?.ssh_host_alias ?? "mission-gpu"}</dd></div>
|
||||
<div><dt>Учётные данные</dt><dd>не доступны интерфейсу</dd></div>
|
||||
</dl>
|
||||
</GlassSurface>
|
||||
|
||||
<GlassSurface className="network-topology" padding="lg">
|
||||
<header className="system-section-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">ФАКТИЧЕСКИЙ МАРШРУТ</span>
|
||||
<h3>Поток управления и обработки</h3>
|
||||
<p>Схема собрана из текущего профиля и live health runtime, без demo-узлов.</p>
|
||||
</div>
|
||||
</header>
|
||||
<div className="network-route">
|
||||
<div data-state="online">
|
||||
<span>Операторский UI</span>
|
||||
<strong>Browser</strong>
|
||||
<small>127.0.0.1:8000</small>
|
||||
</div>
|
||||
<i aria-hidden="true" data-state="online" />
|
||||
<div data-state="online">
|
||||
<span>Control Plane</span>
|
||||
<strong>Mission Core</strong>
|
||||
<small>локальный API</small>
|
||||
</div>
|
||||
<i aria-hidden="true" data-state={connected ? "online" : "offline"} />
|
||||
<div data-state={connected ? "online" : "offline"}>
|
||||
<span>Вычислительный узел</span>
|
||||
<strong>Worker 006</strong>
|
||||
<small>{profile?.address || profile?.ssh_host_alias || "mission-gpu"}:{profile?.port ?? 22}</small>
|
||||
</div>
|
||||
<i aria-hidden="true" data-state={
|
||||
telemetry?.runtimes.some((runtime) => !runtime.external && runtime.state === "running")
|
||||
? "online" : "offline"
|
||||
} />
|
||||
<div data-state={
|
||||
telemetry?.runtimes.some((runtime) => !runtime.external && runtime.state === "running")
|
||||
? "online" : "offline"
|
||||
}>
|
||||
<span>Runtime</span>
|
||||
<strong>Triton + Pipeline</strong>
|
||||
<small>внутренний Docker-контур</small>
|
||||
</div>
|
||||
</div>
|
||||
</GlassSurface>
|
||||
|
||||
<section className="network-interface-section">
|
||||
<header className="system-section-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">ИНТЕРФЕЙСЫ WORKER 006</span>
|
||||
<h3>Адаптеры и счётчики</h3>
|
||||
<p>Только активные интерфейсы, которые вернул сам узел.</p>
|
||||
</div>
|
||||
</header>
|
||||
<div className="network-interface-list">
|
||||
{(telemetry?.network.interfaces ?? []).map((adapter) => (
|
||||
<GlassSurface className="network-interface-card" padding="md" key={adapter.name}>
|
||||
<header>
|
||||
<div>
|
||||
<strong>{adapter.name}</strong>
|
||||
<small>{adapter.description ?? "Описание недоступно"}</small>
|
||||
</div>
|
||||
<StatusBadge tone={adapter.status === "Up" ? "success" : "warning"}>
|
||||
{adapter.status ?? "unknown"}
|
||||
</StatusBadge>
|
||||
</header>
|
||||
<dl>
|
||||
<div><dt>Адреса</dt><dd>{adapter.addresses.join(", ") || "—"}</dd></div>
|
||||
<div><dt>Link</dt><dd>{formatBitRate(adapter.link_speed_bps)}</dd></div>
|
||||
<div><dt>Получено</dt><dd>{formatBytes(adapter.received_bytes)}</dd></div>
|
||||
<div><dt>Отправлено</dt><dd>{formatBytes(adapter.sent_bytes)}</dd></div>
|
||||
</dl>
|
||||
</GlassSurface>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { test } from "node:test";
|
||||
|
||||
const sourceRoot = new URL("../src/", import.meta.url);
|
||||
|
||||
async function read(relativePath) {
|
||||
return readFile(new URL(relativePath, sourceRoot), "utf8");
|
||||
}
|
||||
|
||||
test("Worker 006 telemetry remains a bounded system feature slice", async () => {
|
||||
const [
|
||||
productModel,
|
||||
workspaceHub,
|
||||
core,
|
||||
computeWorkspace,
|
||||
networkWorkspace,
|
||||
styles,
|
||||
] = await Promise.all([
|
||||
read("productModel.ts"),
|
||||
read("workspaces/Workspaces.tsx"),
|
||||
read("core/system/workerTelemetry.ts"),
|
||||
read("workspaces/system/ComputeModulesWorkspace.tsx"),
|
||||
read("workspaces/system/NetworkWorkspace.tsx"),
|
||||
read("styles.css"),
|
||||
]);
|
||||
|
||||
assert.match(productModel, /kind: "compute-modules"/);
|
||||
assert.match(productModel, /kind: "network-monitor"/);
|
||||
assert.match(workspaceHub, /<ComputeModulesWorkspace \/>/);
|
||||
assert.match(workspaceHub, /<NetworkWorkspace \/>/);
|
||||
assert.doesNotMatch(workspaceHub, /worker-telemetry|worker-profile|DESKTOP-OPJ8J04/);
|
||||
assert.match(core, /\/api\/v1\/system\/worker-telemetry/);
|
||||
assert.match(core, /\/api\/v1\/system\/worker-profile/);
|
||||
assert.doesNotMatch(core, /@nodedc\/ui-react/);
|
||||
assert.match(computeWorkspace, /useWorkerTelemetry/);
|
||||
assert.match(networkWorkspace, /127\.0\.0\.1:8000/);
|
||||
assert.doesNotMatch(networkWorkspace, /8765/);
|
||||
assert.match(styles, /system-telemetry\.css/);
|
||||
});
|
||||
@@ -1138,7 +1138,11 @@ def _load_models(args: argparse.Namespace, common: dict[str, Any]) -> _LoadedMod
|
||||
)
|
||||
|
||||
|
||||
def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
def run(
|
||||
args: argparse.Namespace,
|
||||
loaded: _LoadedModels | None = None,
|
||||
runtime_state: dict[str, Any] | None = None,
|
||||
) -> int:
|
||||
import av
|
||||
import torch
|
||||
import transformers
|
||||
@@ -1149,6 +1153,18 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
token = sys.stdin.readline().strip() if args.token_stdin else args.token
|
||||
if not token or len(token) < 40:
|
||||
raise RuntimeError("LAB E15 shadow token is missing")
|
||||
|
||||
def report_stage(stage: str, frame_index: int | None = None) -> None:
|
||||
if runtime_state is None:
|
||||
return
|
||||
runtime_state["current_stage"] = stage
|
||||
runtime_state["stage_observed_at_utc"] = (
|
||||
datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
)
|
||||
if frame_index is not None:
|
||||
runtime_state["active_frame_index"] = frame_index
|
||||
|
||||
report_stage("preprocessing")
|
||||
common = _common(args) if loaded is None else loaded.common
|
||||
live = common["live"]
|
||||
e14 = common["e14"]
|
||||
@@ -1215,6 +1231,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
def on_decoded(frame: DecodedCameraFrame) -> None:
|
||||
nonlocal decoded_frame_count
|
||||
try:
|
||||
report_stage("camera-decode", frame.frame_index)
|
||||
if not first_camera_epoch_ns:
|
||||
first_camera_epoch_ns.append(frame.metadata.captured_at_epoch_ns)
|
||||
last_camera_epoch_ns[:] = [frame.metadata.captured_at_epoch_ns]
|
||||
@@ -1383,6 +1400,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
)
|
||||
semantic_thread.start()
|
||||
decoder.start()
|
||||
report_stage("source-ingress")
|
||||
receiver_thread = threading.Thread(
|
||||
target=_receiver,
|
||||
kwargs={
|
||||
@@ -1429,10 +1447,13 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
latency["decode_age_ms"].append(float(envelope.decode_ms))
|
||||
latency["queue_wait_ms"].append(max(0.0, (started - envelope.decoded_monotonic) * 1000))
|
||||
try:
|
||||
report_stage("preprocessing", envelope.frame_index)
|
||||
detector_started = time.perf_counter()
|
||||
tensor = _preprocess(envelope.image, valid_mask, detector)
|
||||
report_stage("detector", envelope.frame_index)
|
||||
output_tensor, _request_ms = _infer(args.triton_url, detector["model"], tensor)
|
||||
detections, _rejected = _detections(output_tensor, detector, valid_mask)
|
||||
report_stage("tracking", envelope.frame_index)
|
||||
tracks = tracker.update(detections, envelope.frame_index)
|
||||
latency["detector_ms"].append((time.perf_counter() - detector_started) * 1000)
|
||||
|
||||
@@ -1463,6 +1484,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
).reshape((-1, 3))
|
||||
position = binding.pose.position_xyz
|
||||
quaternion = binding.pose.orientation_xyzw
|
||||
report_stage("sensor-fusion", envelope.frame_index)
|
||||
projection_started = time.perf_counter()
|
||||
pixels, depths, source_indices, points_lidar = project_points(
|
||||
points_map,
|
||||
@@ -1533,6 +1555,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
if temporal_stabilizer is None:
|
||||
fusion_objects = raw_fusion_objects
|
||||
else:
|
||||
report_stage("temporal-state", envelope.frame_index)
|
||||
temporal_started = time.perf_counter()
|
||||
fusion_objects = temporal_stabilizer.update(
|
||||
frame_index=envelope.frame_index,
|
||||
@@ -1619,6 +1642,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
objects=fusion_objects,
|
||||
delivery=world["delivery"],
|
||||
)
|
||||
report_stage("result-publication", envelope.frame_index)
|
||||
try:
|
||||
result_queue.put_nowait(live_result)
|
||||
except queue.Full:
|
||||
@@ -1627,6 +1651,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
||||
result_queue.task_done()
|
||||
transport.results_dropped += 1
|
||||
result_queue.put_nowait(live_result)
|
||||
report_stage("source-ingress", envelope.frame_index)
|
||||
except Exception:
|
||||
detector_failures += 1
|
||||
raise
|
||||
@@ -2063,7 +2088,15 @@ def serve(args: argparse.Namespace) -> int:
|
||||
loaded = _load_models(args, common)
|
||||
model_load_seconds = time.perf_counter() - load_started
|
||||
run_lock = threading.Lock()
|
||||
state = {"busy": False, "completed_runs": 0, "failed_runs": 0}
|
||||
state: dict[str, Any] = {
|
||||
"busy": False,
|
||||
"completed_runs": 0,
|
||||
"failed_runs": 0,
|
||||
"active_request_id": None,
|
||||
"active_frame_index": None,
|
||||
"current_stage": None,
|
||||
"stage_observed_at_utc": None,
|
||||
}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "MissionCorePersistentPerception/1"
|
||||
@@ -2100,6 +2133,10 @@ def serve(args: argparse.Namespace) -> int:
|
||||
"model_load_seconds": model_load_seconds,
|
||||
"completed_runs": state["completed_runs"],
|
||||
"failed_runs": state["failed_runs"],
|
||||
"active_request_id": state["active_request_id"],
|
||||
"active_frame_index": state["active_frame_index"],
|
||||
"current_stage": state["current_stage"],
|
||||
"stage_observed_at_utc": state["stage_observed_at_utc"],
|
||||
"authority": common["live"]["authority"],
|
||||
"gpu": torch.cuda.get_device_name(),
|
||||
},
|
||||
@@ -2129,7 +2166,8 @@ def serve(args: argparse.Namespace) -> int:
|
||||
request_id = str(document.get("request_id", "invalid"))
|
||||
run_args = _persistent_run_arguments(args, document)
|
||||
document["token"] = None
|
||||
exit_code = run(run_args, loaded)
|
||||
state["active_request_id"] = request_id
|
||||
exit_code = run(run_args, loaded, state)
|
||||
state["completed_runs"] += 1
|
||||
self._send(
|
||||
200,
|
||||
@@ -2164,6 +2202,10 @@ def serve(args: argparse.Namespace) -> int:
|
||||
)
|
||||
finally:
|
||||
state["busy"] = False
|
||||
state["active_request_id"] = None
|
||||
state["active_frame_index"] = None
|
||||
state["current_stage"] = None
|
||||
state["stage_observed_at_utc"] = None
|
||||
run_lock.release()
|
||||
|
||||
server = ThreadingHTTPServer((args.listen_host, args.listen_port), Handler)
|
||||
|
||||
@@ -52,6 +52,7 @@ from k1link.web.plugin_runtime import (
|
||||
)
|
||||
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
||||
from k1link.web.session_api import build_session_router
|
||||
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
||||
@@ -618,6 +619,11 @@ app.include_router(
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_system_telemetry_router(
|
||||
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||
|
||||
@@ -0,0 +1,834 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import ipaddress
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
PROFILE_SCHEMA: Final = "missioncore.worker-connection-profile/v1"
|
||||
TELEMETRY_SCHEMA: Final = "missioncore.worker-telemetry/v1"
|
||||
PROBE_SCHEMA: Final = "missioncore.worker-probe/v1"
|
||||
DEFAULT_PROFILE_ID: Final = "worker-006"
|
||||
DEFAULT_DISPLAY_NAME: Final = "Worker 006"
|
||||
EXPECTED_NODE_ID: Final = "DESKTOP-OPJ8J04"
|
||||
SSH_HOST_ALIAS: Final = "mission-gpu"
|
||||
PROFILE_FILE_NAME: Final = "worker-006.json"
|
||||
CONTAINER_NAMES: Final = (
|
||||
"mission-core-triton",
|
||||
"mission-core-perception-worker",
|
||||
"sentinel-frigate",
|
||||
"sentinel-ollama",
|
||||
)
|
||||
SAFE_HOSTNAME = re.compile(
|
||||
r"^(?=.{1,253}\.?$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*"
|
||||
r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.?$"
|
||||
)
|
||||
PROMETHEUS_SAMPLE = re.compile(
|
||||
r"^(?P<name>nv_inference_(?:request_success|request_failure|count))"
|
||||
r"(?:\{[^}]*\})?\s+(?P<value>[0-9.eE+-]+)$"
|
||||
)
|
||||
|
||||
RootProvider = Callable[[], Path]
|
||||
ProbeRunner = Callable[["WorkerConnectionProfile"], dict[str, Any]]
|
||||
|
||||
|
||||
class WorkerConnectionProfile(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
schema_version: str = PROFILE_SCHEMA
|
||||
profile_id: str = DEFAULT_PROFILE_ID
|
||||
display_name: str = DEFAULT_DISPLAY_NAME
|
||||
expected_node_id: str = EXPECTED_NODE_ID
|
||||
ssh_host_alias: str = SSH_HOST_ALIAS
|
||||
address: str = ""
|
||||
port: int = Field(default=22, ge=1, le=65535)
|
||||
revision: int = Field(default=0, ge=0)
|
||||
updated_at_utc: str | None = None
|
||||
|
||||
@field_validator("address")
|
||||
@classmethod
|
||||
def validate_address(cls, value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not normalized:
|
||||
return ""
|
||||
if any(character.isspace() or ord(character) < 32 for character in normalized):
|
||||
raise ValueError("worker address contains whitespace or control characters")
|
||||
try:
|
||||
ipaddress.ip_address(normalized)
|
||||
except ValueError:
|
||||
if SAFE_HOSTNAME.fullmatch(normalized) is None:
|
||||
raise ValueError("worker address is not a valid IP address or hostname") from None
|
||||
return normalized.rstrip(".")
|
||||
|
||||
|
||||
class WorkerConnectionProfilePut(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
revision: int = Field(ge=0)
|
||||
address: str = Field(default="", max_length=253)
|
||||
port: int = Field(default=22, ge=1, le=65535)
|
||||
|
||||
@field_validator("address")
|
||||
@classmethod
|
||||
def validate_address(cls, value: str) -> str:
|
||||
return WorkerConnectionProfile.validate_address(value)
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
class WorkerProfileStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.path = root / PROFILE_FILE_NAME
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def read(self) -> WorkerConnectionProfile:
|
||||
with self._lock:
|
||||
if not self.path.is_file():
|
||||
return WorkerConnectionProfile()
|
||||
try:
|
||||
document = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
return WorkerConnectionProfile.model_validate(document)
|
||||
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
||||
return WorkerConnectionProfile()
|
||||
|
||||
def save(self, request: WorkerConnectionProfilePut) -> WorkerConnectionProfile:
|
||||
with self._lock:
|
||||
current = self._read_unlocked()
|
||||
if current.revision != request.revision:
|
||||
raise RuntimeError("worker profile revision changed")
|
||||
profile = current.model_copy(
|
||||
update={
|
||||
"address": request.address,
|
||||
"port": request.port,
|
||||
"revision": current.revision + 1,
|
||||
"updated_at_utc": _utc_now(),
|
||||
}
|
||||
)
|
||||
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=".worker-006.",
|
||||
suffix=".tmp",
|
||||
dir=self.root,
|
||||
)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
|
||||
json.dump(
|
||||
profile.model_dump(mode="json"),
|
||||
stream,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.chmod(temporary, 0o600)
|
||||
os.replace(temporary, self.path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return profile
|
||||
|
||||
def _read_unlocked(self) -> WorkerConnectionProfile:
|
||||
if not self.path.is_file():
|
||||
return WorkerConnectionProfile()
|
||||
try:
|
||||
return WorkerConnectionProfile.model_validate_json(
|
||||
self.path.read_text(encoding="utf-8")
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
return WorkerConnectionProfile()
|
||||
|
||||
|
||||
WORKER_PROBE_POWERSHELL: Final = r"""
|
||||
$ErrorActionPreference = "Stop"
|
||||
$containers = @(
|
||||
"mission-core-triton",
|
||||
"mission-core-perception-worker",
|
||||
"sentinel-frigate",
|
||||
"sentinel-ollama"
|
||||
)
|
||||
$os = Get-CimInstance Win32_OperatingSystem
|
||||
$cpu = Get-CimInstance Win32_Processor
|
||||
$computer = Get-CimInstance Win32_ComputerSystem
|
||||
$dockerStats = @{}
|
||||
try {
|
||||
$statsLines = @(docker stats --no-stream --format "{{json .}}" $containers)
|
||||
foreach ($line in $statsLines) {
|
||||
if ($line) {
|
||||
$row = $line | ConvertFrom-Json
|
||||
$dockerStats[$row.Name] = $row
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
$containerStates = @{}
|
||||
foreach ($name in $containers) {
|
||||
try {
|
||||
$state = (docker inspect --format "{{json .State}}" $name) | ConvertFrom-Json
|
||||
$image = docker inspect --format "{{.Config.Image}}" $name
|
||||
$containerStates[$name] = [ordered]@{state=$state; image=$image}
|
||||
} catch {
|
||||
$containerStates[$name] = $null
|
||||
}
|
||||
}
|
||||
$gpu = $null
|
||||
try {
|
||||
$gpuQuery = "name,utilization.gpu,memory.used,memory.total,power.draw,temperature.gpu"
|
||||
$gpuLine = nvidia-smi "--query-gpu=$gpuQuery" --format=csv,noheader,nounits |
|
||||
Select-Object -First 1
|
||||
$gpuParts = @($gpuLine -split ",\s*")
|
||||
if ($gpuParts.Count -ge 6) {
|
||||
$gpu = [ordered]@{
|
||||
name=$gpuParts[0]
|
||||
utilization_percent=[double]$gpuParts[1]
|
||||
memory_used_mib=[double]$gpuParts[2]
|
||||
memory_total_mib=[double]$gpuParts[3]
|
||||
power_watts=[double]$gpuParts[4]
|
||||
temperature_celsius=[double]$gpuParts[5]
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
$network = @()
|
||||
foreach ($adapter in @(Get-NetAdapter | Where-Object Status -eq "Up")) {
|
||||
try {
|
||||
$statistics = Get-NetAdapterStatistics -Name $adapter.Name
|
||||
$addresses = @(
|
||||
Get-NetIPAddress -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 `
|
||||
-ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.IPAddress -notlike "169.254.*" } |
|
||||
Select-Object -ExpandProperty IPAddress
|
||||
)
|
||||
$network += [ordered]@{
|
||||
name=$adapter.Name
|
||||
description=$adapter.InterfaceDescription
|
||||
status=$adapter.Status
|
||||
link_speed_bps=[double]$adapter.Speed
|
||||
mac_address=$adapter.MacAddress
|
||||
addresses=$addresses
|
||||
received_bytes=[double]$statistics.ReceivedBytes
|
||||
sent_bytes=[double]$statistics.SentBytes
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
$tritonReady = $false
|
||||
try {
|
||||
$tritonResponse = Invoke-WebRequest -UseBasicParsing `
|
||||
-Uri "http://127.0.0.1:8000/v2/health/ready" -TimeoutSec 2
|
||||
$tritonReady = $tritonResponse.StatusCode -eq 200
|
||||
} catch {}
|
||||
$tritonMetrics = @()
|
||||
try {
|
||||
$metricsResponse = Invoke-WebRequest -UseBasicParsing `
|
||||
-Uri "http://127.0.0.1:8002/metrics" -TimeoutSec 2
|
||||
$tritonMetrics = @(
|
||||
$metricsResponse.Content -split "`n" |
|
||||
Where-Object { $_ -match "^nv_inference_(request_success|request_failure|count)" }
|
||||
)
|
||||
} catch {}
|
||||
$perceptionHealth = $null
|
||||
try {
|
||||
$healthJson = docker exec mission-core-perception-worker python3 -c "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
|
||||
$perceptionHealth = $healthJson | ConvertFrom-Json
|
||||
} catch {}
|
||||
$disks = @(
|
||||
Get-CimInstance Win32_LogicalDisk -Filter "DriveType=3" |
|
||||
ForEach-Object {
|
||||
[ordered]@{
|
||||
name=$_.DeviceID
|
||||
size_bytes=[double]$_.Size
|
||||
free_bytes=[double]$_.FreeSpace
|
||||
}
|
||||
}
|
||||
)
|
||||
[ordered]@{
|
||||
schema_version="missioncore.worker-probe-raw/v1"
|
||||
observed_at_utc=[DateTime]::UtcNow.ToString("o")
|
||||
node_id=$env:COMPUTERNAME
|
||||
os=[ordered]@{
|
||||
caption=$os.Caption
|
||||
version=$os.Version
|
||||
uptime_seconds=([DateTime]::UtcNow - $os.LastBootUpTime.ToUniversalTime()).TotalSeconds
|
||||
}
|
||||
cpu=[ordered]@{
|
||||
name=(@($cpu | Select-Object -ExpandProperty Name) -join " + ")
|
||||
logical_processors=[int]$computer.NumberOfLogicalProcessors
|
||||
load_percent=[double](($cpu | Measure-Object LoadPercentage -Average).Average)
|
||||
}
|
||||
memory=[ordered]@{
|
||||
total_bytes=[double]$os.TotalVisibleMemorySize * 1024
|
||||
free_bytes=[double]$os.FreePhysicalMemory * 1024
|
||||
}
|
||||
disks=$disks
|
||||
gpu=$gpu
|
||||
network=$network
|
||||
docker_stats=$dockerStats
|
||||
container_states=$containerStates
|
||||
triton=[ordered]@{ready=$tritonReady; metrics=$tritonMetrics}
|
||||
perception=$perceptionHealth
|
||||
} | ConvertTo-Json -Depth 12 -Compress
|
||||
""".strip()
|
||||
|
||||
|
||||
def _ssh_arguments(profile: WorkerConnectionProfile) -> list[str]:
|
||||
arguments = [
|
||||
"ssh",
|
||||
"-o",
|
||||
"BatchMode=yes",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=yes",
|
||||
"-o",
|
||||
f"HostKeyAlias={profile.ssh_host_alias}",
|
||||
"-o",
|
||||
"ConnectTimeout=5",
|
||||
]
|
||||
if profile.address:
|
||||
arguments.extend(["-o", f"HostName={profile.address}"])
|
||||
arguments.extend(
|
||||
[
|
||||
"-p",
|
||||
str(profile.port),
|
||||
profile.ssh_host_alias,
|
||||
"powershell.exe",
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
(
|
||||
"$encoded=[Console]::In.ReadToEnd();"
|
||||
"Invoke-Expression "
|
||||
"([Text.Encoding]::Unicode.GetString([Convert]::FromBase64String($encoded)))"
|
||||
),
|
||||
]
|
||||
)
|
||||
return arguments
|
||||
|
||||
|
||||
def run_worker_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
_ssh_arguments(profile),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
input=base64.b64encode(WORKER_PROBE_POWERSHELL.encode("utf-16le")),
|
||||
timeout=12,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return _failed_probe(profile, "worker-unreachable", started)
|
||||
if completed.returncode != 0:
|
||||
return _failed_probe(profile, "worker-unreachable", started)
|
||||
try:
|
||||
raw = completed.stdout.decode("cp866")
|
||||
document = json.loads(raw.strip())
|
||||
except (UnicodeDecodeError, json.JSONDecodeError, TypeError):
|
||||
return _failed_probe(profile, "invalid-worker-response", started)
|
||||
if not isinstance(document, dict):
|
||||
return _failed_probe(profile, "invalid-worker-response", started)
|
||||
node_id = document.get("node_id")
|
||||
if node_id != profile.expected_node_id:
|
||||
return {
|
||||
"schema_version": PROBE_SCHEMA,
|
||||
"reachable": True,
|
||||
"identity_matches": False,
|
||||
"node_id": node_id if isinstance(node_id, str) else None,
|
||||
"latency_ms": (time.perf_counter() - started) * 1000,
|
||||
"observed_at_utc": _utc_now(),
|
||||
"error_code": "worker-identity-mismatch",
|
||||
"raw": None,
|
||||
}
|
||||
return {
|
||||
"schema_version": PROBE_SCHEMA,
|
||||
"reachable": True,
|
||||
"identity_matches": True,
|
||||
"node_id": node_id,
|
||||
"latency_ms": (time.perf_counter() - started) * 1000,
|
||||
"observed_at_utc": _utc_now(),
|
||||
"error_code": None,
|
||||
"raw": document,
|
||||
}
|
||||
|
||||
|
||||
def _failed_probe(
|
||||
profile: WorkerConnectionProfile,
|
||||
error_code: str,
|
||||
started: float,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": PROBE_SCHEMA,
|
||||
"reachable": False,
|
||||
"identity_matches": False,
|
||||
"node_id": None,
|
||||
"expected_node_id": profile.expected_node_id,
|
||||
"latency_ms": (time.perf_counter() - started) * 1000,
|
||||
"observed_at_utc": _utc_now(),
|
||||
"error_code": error_code,
|
||||
"raw": None,
|
||||
}
|
||||
|
||||
|
||||
def _number(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||
return None
|
||||
result = float(value)
|
||||
return result if math.isfinite(result) else None
|
||||
|
||||
|
||||
def _mapping(value: object) -> dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _items(value: object) -> list[Any]:
|
||||
return value if isinstance(value, list) else []
|
||||
|
||||
|
||||
def _percent(used: float | None, total: float | None) -> float | None:
|
||||
if used is None or total is None or total <= 0:
|
||||
return None
|
||||
return max(0.0, min(100.0, used / total * 100))
|
||||
|
||||
|
||||
def _container_document(
|
||||
name: str,
|
||||
raw_stats: dict[str, Any],
|
||||
raw_states: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
stats = _mapping(raw_stats.get(name))
|
||||
descriptor = _mapping(raw_states.get(name))
|
||||
state = _mapping(descriptor.get("state"))
|
||||
health = _mapping(state.get("Health"))
|
||||
roles = {
|
||||
"mission-core-triton": ("Inference Runtime", "mission-core", False),
|
||||
"mission-core-perception-worker": ("Perception Pipeline", "mission-core", False),
|
||||
"sentinel-frigate": ("Sentinel Frigate", "external", True),
|
||||
"sentinel-ollama": ("Sentinel Ollama", "external", True),
|
||||
}
|
||||
role, owner, external = roles[name]
|
||||
return {
|
||||
"name": name,
|
||||
"role": role,
|
||||
"owner": owner,
|
||||
"external": external,
|
||||
"image": descriptor.get("image") if isinstance(descriptor.get("image"), str) else None,
|
||||
"state": state.get("Status") if isinstance(state.get("Status"), str) else "unavailable",
|
||||
"health": health.get("Status") if isinstance(health.get("Status"), str) else None,
|
||||
"cpu_percent": _parse_percent(stats.get("CPUPerc")),
|
||||
"memory_percent": _parse_percent(stats.get("MemPerc")),
|
||||
"memory_usage": stats.get("MemUsage") if isinstance(stats.get("MemUsage"), str) else None,
|
||||
"network_io": stats.get("NetIO") if isinstance(stats.get("NetIO"), str) else None,
|
||||
"block_io": stats.get("BlockIO") if isinstance(stats.get("BlockIO"), str) else None,
|
||||
"pids": _parse_integer(stats.get("PIDs")),
|
||||
}
|
||||
|
||||
|
||||
def _parse_percent(value: object) -> float | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
number = float(value.rstrip("%"))
|
||||
except ValueError:
|
||||
return None
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
|
||||
def _parse_integer(value: object) -> int | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _triton_metrics(lines: object) -> dict[str, float | None]:
|
||||
totals: dict[str, float] = {
|
||||
"nv_inference_request_success": 0,
|
||||
"nv_inference_request_failure": 0,
|
||||
"nv_inference_count": 0,
|
||||
}
|
||||
matched: set[str] = set()
|
||||
for line in _items(lines):
|
||||
if not isinstance(line, str):
|
||||
continue
|
||||
match = PROMETHEUS_SAMPLE.fullmatch(line.strip())
|
||||
if match is None:
|
||||
continue
|
||||
try:
|
||||
value = float(match.group("value"))
|
||||
except ValueError:
|
||||
continue
|
||||
if math.isfinite(value):
|
||||
name = match.group("name")
|
||||
totals[name] += value
|
||||
matched.add(name)
|
||||
return {
|
||||
"requests_succeeded": (
|
||||
totals["nv_inference_request_success"]
|
||||
if "nv_inference_request_success" in matched
|
||||
else None
|
||||
),
|
||||
"requests_failed": (
|
||||
totals["nv_inference_request_failure"]
|
||||
if "nv_inference_request_failure" in matched
|
||||
else None
|
||||
),
|
||||
"inferences": (
|
||||
totals["nv_inference_count"] if "nv_inference_count" in matched else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
perception = _mapping(raw.get("perception"))
|
||||
current_stage = perception.get("current_stage")
|
||||
if not isinstance(current_stage, str):
|
||||
current_stage = None
|
||||
busy = perception.get("state") == "busy"
|
||||
stages = (
|
||||
("source-ingress", "Приём сенсорного потока"),
|
||||
("camera-decode", "Декодирование камеры"),
|
||||
("preprocessing", "Предобработка"),
|
||||
("detector", "Детектор объектов"),
|
||||
("semantic-model", "Семантическая модель"),
|
||||
("sensor-fusion", "Camera ↔ LiDAR fusion"),
|
||||
("tracking", "Трекинг"),
|
||||
("temporal-state", "Временное состояние"),
|
||||
("result-publication", "Публикация результата"),
|
||||
)
|
||||
|
||||
def stage_state(stage_id: str) -> str:
|
||||
if not perception:
|
||||
return "unavailable"
|
||||
if not busy:
|
||||
return "ready"
|
||||
if stage_id == current_stage or stage_id in {
|
||||
"source-ingress",
|
||||
"camera-decode",
|
||||
"semantic-model",
|
||||
}:
|
||||
return "active"
|
||||
return "waiting"
|
||||
|
||||
return {
|
||||
"service_state": (
|
||||
perception.get("state") if isinstance(perception.get("state"), str) else "unavailable"
|
||||
),
|
||||
"current_stage": current_stage,
|
||||
"active_request_id": (
|
||||
perception.get("active_request_id")
|
||||
if isinstance(perception.get("active_request_id"), str)
|
||||
else None
|
||||
),
|
||||
"active_frame_index": (
|
||||
int(perception["active_frame_index"])
|
||||
if isinstance(perception.get("active_frame_index"), int)
|
||||
else None
|
||||
),
|
||||
"completed_runs": (
|
||||
int(perception["completed_runs"])
|
||||
if isinstance(perception.get("completed_runs"), int)
|
||||
else None
|
||||
),
|
||||
"failed_runs": (
|
||||
int(perception["failed_runs"])
|
||||
if isinstance(perception.get("failed_runs"), int)
|
||||
else None
|
||||
),
|
||||
"model_load_seconds": _number(perception.get("model_load_seconds")),
|
||||
"stages": [
|
||||
{
|
||||
"id": stage_id,
|
||||
"label": label,
|
||||
"state": stage_state(stage_id),
|
||||
}
|
||||
for stage_id, label in stages
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class WorkerTelemetryService:
|
||||
def __init__(
|
||||
self,
|
||||
store: WorkerProfileStore,
|
||||
probe_runner: ProbeRunner = run_worker_probe,
|
||||
*,
|
||||
cache_seconds: float = 1.5,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.probe_runner = probe_runner
|
||||
self.cache_seconds = cache_seconds
|
||||
self._lock = threading.Lock()
|
||||
self._cached_at = 0.0
|
||||
self._cached: dict[str, Any] | None = None
|
||||
self._previous_network: tuple[float, float, float] | None = None
|
||||
self._history: deque[dict[str, Any]] = deque(maxlen=300)
|
||||
|
||||
def profile_document(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": PROFILE_SCHEMA,
|
||||
"profile": self.store.read().model_dump(mode="json"),
|
||||
"security": {
|
||||
"transport": "ssh",
|
||||
"host_key_policy": "strict-pinned",
|
||||
"credentials_managed_by_ui": False,
|
||||
"ssh_host_alias": SSH_HOST_ALIAS,
|
||||
},
|
||||
}
|
||||
|
||||
def test_profile(self, request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||
current = self.store.read()
|
||||
candidate = current.model_copy(
|
||||
update={"address": request.address, "port": request.port}
|
||||
)
|
||||
return self._public_probe(self.probe_runner(candidate))
|
||||
|
||||
def apply_profile(self, request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||
current = self.store.read()
|
||||
if request.revision != current.revision:
|
||||
raise RuntimeError("worker profile revision changed")
|
||||
candidate = current.model_copy(
|
||||
update={"address": request.address, "port": request.port}
|
||||
)
|
||||
probe = self.probe_runner(candidate)
|
||||
if not probe.get("reachable") or not probe.get("identity_matches"):
|
||||
raise ConnectionError(str(probe.get("error_code") or "worker-unreachable"))
|
||||
saved = self.store.save(request)
|
||||
with self._lock:
|
||||
self._cached = None
|
||||
self._cached_at = 0
|
||||
self._previous_network = None
|
||||
self._history.clear()
|
||||
return {
|
||||
**self.profile_document(),
|
||||
"verification": self._public_probe(probe),
|
||||
"profile": saved.model_dump(mode="json"),
|
||||
}
|
||||
|
||||
def snapshot(self, history_limit: int) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
if self._cached is not None and now - self._cached_at < self.cache_seconds:
|
||||
return {
|
||||
**self._cached,
|
||||
"history": list(self._history)[-history_limit:],
|
||||
}
|
||||
profile = self.store.read()
|
||||
probe = self.probe_runner(profile)
|
||||
document = self._telemetry_document(profile, probe, now)
|
||||
self._cached = document
|
||||
self._cached_at = now
|
||||
return {
|
||||
**document,
|
||||
"history": list(self._history)[-history_limit:],
|
||||
}
|
||||
|
||||
def _telemetry_document(
|
||||
self,
|
||||
profile: WorkerConnectionProfile,
|
||||
probe: dict[str, Any],
|
||||
monotonic_now: float,
|
||||
) -> dict[str, Any]:
|
||||
public_probe = self._public_probe(probe)
|
||||
if not probe.get("reachable") or not probe.get("identity_matches"):
|
||||
return {
|
||||
"schema_version": TELEMETRY_SCHEMA,
|
||||
"profile": profile.model_dump(mode="json"),
|
||||
"connection": public_probe,
|
||||
"node": None,
|
||||
"runtimes": [],
|
||||
"pipeline": _pipeline_document({}),
|
||||
"network": {"interfaces": [], "aggregate": None},
|
||||
}
|
||||
raw = _mapping(probe.get("raw"))
|
||||
memory = _mapping(raw.get("memory"))
|
||||
memory_total = _number(memory.get("total_bytes"))
|
||||
memory_free = _number(memory.get("free_bytes"))
|
||||
memory_used = (
|
||||
memory_total - memory_free
|
||||
if memory_total is not None and memory_free is not None
|
||||
else None
|
||||
)
|
||||
gpu = _mapping(raw.get("gpu"))
|
||||
gpu_used = _number(gpu.get("memory_used_mib"))
|
||||
gpu_total = _number(gpu.get("memory_total_mib"))
|
||||
interfaces = [
|
||||
self._network_interface(item)
|
||||
for item in _items(raw.get("network"))
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
received = sum(
|
||||
value
|
||||
for item in interfaces
|
||||
if (value := _number(item.get("received_bytes"))) is not None
|
||||
)
|
||||
sent = sum(
|
||||
value
|
||||
for item in interfaces
|
||||
if (value := _number(item.get("sent_bytes"))) is not None
|
||||
)
|
||||
receive_rate: float | None = None
|
||||
send_rate: float | None = None
|
||||
if self._previous_network is not None:
|
||||
previous_at, previous_received, previous_sent = self._previous_network
|
||||
elapsed = monotonic_now - previous_at
|
||||
if elapsed > 0 and received >= previous_received and sent >= previous_sent:
|
||||
receive_rate = (received - previous_received) / elapsed
|
||||
send_rate = (sent - previous_sent) / elapsed
|
||||
self._previous_network = (monotonic_now, received, sent)
|
||||
raw_stats = _mapping(raw.get("docker_stats"))
|
||||
raw_states = _mapping(raw.get("container_states"))
|
||||
runtimes = [
|
||||
_container_document(name, raw_stats, raw_states) for name in CONTAINER_NAMES
|
||||
]
|
||||
triton = _mapping(raw.get("triton"))
|
||||
triton_document = {
|
||||
"ready": triton.get("ready") is True,
|
||||
**_triton_metrics(triton.get("metrics")),
|
||||
}
|
||||
memory_used_percent = _percent(memory_used, memory_total)
|
||||
gpu_memory_used_percent = _percent(gpu_used, gpu_total)
|
||||
node: dict[str, Any] = {
|
||||
"node_id": raw.get("node_id"),
|
||||
"observed_at_utc": raw.get("observed_at_utc"),
|
||||
"os": _mapping(raw.get("os")),
|
||||
"cpu": _mapping(raw.get("cpu")),
|
||||
"memory": {
|
||||
"total_bytes": memory_total,
|
||||
"used_bytes": memory_used,
|
||||
"free_bytes": memory_free,
|
||||
"used_percent": memory_used_percent,
|
||||
},
|
||||
"disks": _items(raw.get("disks")),
|
||||
"gpu": {
|
||||
**gpu,
|
||||
"memory_used_percent": gpu_memory_used_percent,
|
||||
}
|
||||
if gpu
|
||||
else None,
|
||||
"triton": triton_document,
|
||||
}
|
||||
network = {
|
||||
"interfaces": interfaces,
|
||||
"aggregate": {
|
||||
"received_bytes": received,
|
||||
"sent_bytes": sent,
|
||||
"receive_bytes_per_second": receive_rate,
|
||||
"send_bytes_per_second": send_rate,
|
||||
},
|
||||
}
|
||||
history_row = {
|
||||
"observed_at_utc": raw.get("observed_at_utc") or _utc_now(),
|
||||
"cpu_percent": _number(_mapping(raw.get("cpu")).get("load_percent")),
|
||||
"memory_percent": memory_used_percent,
|
||||
"gpu_percent": _number(gpu.get("utilization_percent")),
|
||||
"gpu_memory_percent": gpu_memory_used_percent,
|
||||
"network_receive_bytes_per_second": receive_rate,
|
||||
"network_send_bytes_per_second": send_rate,
|
||||
}
|
||||
self._history.append(history_row)
|
||||
return {
|
||||
"schema_version": TELEMETRY_SCHEMA,
|
||||
"profile": profile.model_dump(mode="json"),
|
||||
"connection": public_probe,
|
||||
"node": node,
|
||||
"runtimes": runtimes,
|
||||
"pipeline": _pipeline_document(raw),
|
||||
"network": network,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _network_interface(item: dict[str, Any]) -> dict[str, Any]:
|
||||
addresses = [
|
||||
address for address in _items(item.get("addresses")) if isinstance(address, str)
|
||||
]
|
||||
return {
|
||||
"name": item.get("name") if isinstance(item.get("name"), str) else "unknown",
|
||||
"description": (
|
||||
item.get("description") if isinstance(item.get("description"), str) else None
|
||||
),
|
||||
"status": item.get("status") if isinstance(item.get("status"), str) else None,
|
||||
"link_speed_bps": _number(item.get("link_speed_bps")),
|
||||
"mac_address": (
|
||||
item.get("mac_address") if isinstance(item.get("mac_address"), str) else None
|
||||
),
|
||||
"addresses": addresses,
|
||||
"received_bytes": _number(item.get("received_bytes")),
|
||||
"sent_bytes": _number(item.get("sent_bytes")),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _public_probe(probe: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": PROBE_SCHEMA,
|
||||
"reachable": probe.get("reachable") is True,
|
||||
"identity_matches": probe.get("identity_matches") is True,
|
||||
"node_id": probe.get("node_id") if isinstance(probe.get("node_id"), str) else None,
|
||||
"latency_ms": _number(probe.get("latency_ms")),
|
||||
"observed_at_utc": (
|
||||
probe.get("observed_at_utc")
|
||||
if isinstance(probe.get("observed_at_utc"), str)
|
||||
else _utc_now()
|
||||
),
|
||||
"error_code": (
|
||||
probe.get("error_code")
|
||||
if isinstance(probe.get("error_code"), str)
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def build_system_telemetry_router(
|
||||
*,
|
||||
root_provider: RootProvider,
|
||||
probe_runner: ProbeRunner = run_worker_probe,
|
||||
) -> APIRouter:
|
||||
store = WorkerProfileStore(root_provider())
|
||||
service = WorkerTelemetryService(store, probe_runner)
|
||||
router = APIRouter(prefix="/api/v1/system", tags=["system"])
|
||||
|
||||
@router.get("/worker-profile")
|
||||
def get_worker_profile() -> dict[str, Any]:
|
||||
return service.profile_document()
|
||||
|
||||
@router.post("/worker-profile/test")
|
||||
def test_worker_profile(request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||
return service.test_profile(request)
|
||||
|
||||
@router.put("/worker-profile")
|
||||
def put_worker_profile(request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||
try:
|
||||
return service.apply_profile(request)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Профиль Worker 006 был изменён в другой сессии.",
|
||||
) from exc
|
||||
except ConnectionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Новый адрес не прошёл проверку узла Worker 006.",
|
||||
) from exc
|
||||
|
||||
@router.get("/worker-telemetry")
|
||||
def get_worker_telemetry(
|
||||
history: int = Query(default=90, ge=1, le=300),
|
||||
) -> dict[str, Any]:
|
||||
return service.snapshot(history)
|
||||
|
||||
return router
|
||||
@@ -0,0 +1,237 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import stat
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.routing import APIRoute
|
||||
from pydantic import ValidationError
|
||||
|
||||
from k1link.web.system_telemetry_api import (
|
||||
EXPECTED_NODE_ID,
|
||||
WorkerConnectionProfile,
|
||||
WorkerConnectionProfilePut,
|
||||
WorkerProfileStore,
|
||||
WorkerTelemetryService,
|
||||
_ssh_arguments,
|
||||
build_system_telemetry_router,
|
||||
)
|
||||
|
||||
|
||||
def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
||||
for route in router.routes:
|
||||
if (
|
||||
isinstance(route, APIRoute)
|
||||
and route.path == path
|
||||
and route.methods is not None
|
||||
and method in route.methods
|
||||
):
|
||||
return route.endpoint
|
||||
raise AssertionError(f"{method} {path} route is missing")
|
||||
|
||||
|
||||
def _probe(
|
||||
*,
|
||||
node_id: str = EXPECTED_NODE_ID,
|
||||
received_bytes: float = 1_000,
|
||||
sent_bytes: float = 500,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"reachable": True,
|
||||
"identity_matches": node_id == EXPECTED_NODE_ID,
|
||||
"node_id": node_id,
|
||||
"latency_ms": 12.5,
|
||||
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||
"error_code": None,
|
||||
"raw": {
|
||||
"node_id": node_id,
|
||||
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||
"os": {
|
||||
"caption": "Windows 11 Pro",
|
||||
"version": "10.0",
|
||||
"uptime_seconds": 100,
|
||||
},
|
||||
"cpu": {
|
||||
"name": "CPU",
|
||||
"logical_processors": 32,
|
||||
"load_percent": 25,
|
||||
},
|
||||
"memory": {
|
||||
"total_bytes": 1000,
|
||||
"free_bytes": 400,
|
||||
},
|
||||
"disks": [],
|
||||
"gpu": {
|
||||
"name": "GPU",
|
||||
"utilization_percent": 30,
|
||||
"memory_used_mib": 100,
|
||||
"memory_total_mib": 200,
|
||||
},
|
||||
"network": [
|
||||
{
|
||||
"name": "LAN",
|
||||
"description": "Ethernet",
|
||||
"status": "Up",
|
||||
"link_speed_bps": 1_000_000_000,
|
||||
"addresses": ["192.0.2.10"],
|
||||
"received_bytes": received_bytes,
|
||||
"sent_bytes": sent_bytes,
|
||||
}
|
||||
],
|
||||
"docker_stats": {
|
||||
"mission-core-triton": {
|
||||
"Name": "mission-core-triton",
|
||||
"CPUPerc": "2.5%",
|
||||
"MemPerc": "3.5%",
|
||||
"MemUsage": "1GiB / 32GiB",
|
||||
"NetIO": "1MB / 2MB",
|
||||
"BlockIO": "0B / 0B",
|
||||
"PIDs": "12",
|
||||
},
|
||||
"sentinel-frigate": {
|
||||
"Name": "sentinel-frigate",
|
||||
"CPUPerc": "150%",
|
||||
"MemPerc": "20%",
|
||||
"MemUsage": "6GiB / 32GiB",
|
||||
"NetIO": "3GB / 1GB",
|
||||
"BlockIO": "1GB / 1GB",
|
||||
"PIDs": "200",
|
||||
},
|
||||
},
|
||||
"container_states": {
|
||||
"mission-core-triton": {
|
||||
"state": {"Status": "running", "Health": {"Status": "healthy"}},
|
||||
"image": "triton@sha256:accepted",
|
||||
},
|
||||
"sentinel-frigate": {
|
||||
"state": {"Status": "running", "Health": {"Status": "healthy"}},
|
||||
"image": "frigate@sha256:external",
|
||||
},
|
||||
},
|
||||
"triton": {
|
||||
"ready": True,
|
||||
"metrics": [
|
||||
'nv_inference_request_success{model="detector"} 12',
|
||||
'nv_inference_request_failure{model="detector"} 0',
|
||||
],
|
||||
},
|
||||
"perception": {
|
||||
"state": "busy",
|
||||
"current_stage": "detector",
|
||||
"active_request_id": "run-001",
|
||||
"active_frame_index": 42,
|
||||
"completed_runs": 3,
|
||||
"failed_runs": 0,
|
||||
"model_load_seconds": 10.5,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_worker_profile_rejects_ssh_option_injection() -> None:
|
||||
for unsafe in (
|
||||
"-oProxyCommand=touch /tmp/unsafe",
|
||||
"worker.local -p 2200",
|
||||
"worker.local/../../unsafe",
|
||||
"worker.local\nHost evil",
|
||||
):
|
||||
with pytest.raises(ValidationError):
|
||||
WorkerConnectionProfile(address=unsafe)
|
||||
|
||||
assert WorkerConnectionProfile(address="192.0.2.15").address == "192.0.2.15"
|
||||
assert WorkerConnectionProfile(address="worker-006.local").address == "worker-006.local"
|
||||
|
||||
|
||||
def test_worker_profile_is_atomic_versioned_and_private(tmp_path: Path) -> None:
|
||||
store = WorkerProfileStore(tmp_path / "system")
|
||||
initial = store.read()
|
||||
assert initial.revision == 0
|
||||
saved = store.save(
|
||||
WorkerConnectionProfilePut(
|
||||
revision=0,
|
||||
address="192.0.2.15",
|
||||
port=2200,
|
||||
)
|
||||
)
|
||||
|
||||
assert saved.revision == 1
|
||||
assert store.read() == saved
|
||||
assert stat.S_IMODE(store.path.stat().st_mode) == 0o600
|
||||
with pytest.raises(RuntimeError, match="revision changed"):
|
||||
store.save(
|
||||
WorkerConnectionProfilePut(
|
||||
revision=0,
|
||||
address="192.0.2.16",
|
||||
port=22,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_ssh_command_keeps_identity_pinned_and_values_as_arguments() -> None:
|
||||
arguments = _ssh_arguments(
|
||||
WorkerConnectionProfile(address="192.0.2.15", port=2200)
|
||||
)
|
||||
assert "BatchMode=yes" in arguments
|
||||
assert "StrictHostKeyChecking=yes" in arguments
|
||||
assert "HostKeyAlias=mission-gpu" in arguments
|
||||
assert "HostName=192.0.2.15" in arguments
|
||||
assert arguments[arguments.index("-p") + 1] == "2200"
|
||||
assert "mission-gpu" in arguments
|
||||
assert "192.0.2.15; touch unsafe" not in arguments
|
||||
|
||||
|
||||
def test_worker_telemetry_separates_mission_core_and_external_load(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service = WorkerTelemetryService(
|
||||
WorkerProfileStore(tmp_path / "system"),
|
||||
lambda _: _probe(),
|
||||
cache_seconds=0,
|
||||
)
|
||||
document = service.snapshot(10)
|
||||
runtimes = {runtime["name"]: runtime for runtime in document["runtimes"]}
|
||||
|
||||
assert document["connection"]["identity_matches"] is True
|
||||
assert document["node"]["memory"]["used_percent"] == 60
|
||||
assert document["node"]["gpu"]["memory_used_percent"] == 50
|
||||
assert document["node"]["triton"]["requests_succeeded"] == 12
|
||||
assert runtimes["mission-core-triton"]["external"] is False
|
||||
assert runtimes["mission-core-triton"]["cpu_percent"] == 2.5
|
||||
assert runtimes["sentinel-frigate"]["external"] is True
|
||||
assert runtimes["sentinel-frigate"]["cpu_percent"] == 150
|
||||
assert document["pipeline"]["active_request_id"] == "run-001"
|
||||
assert next(
|
||||
stage
|
||||
for stage in document["pipeline"]["stages"]
|
||||
if stage["id"] == "detector"
|
||||
)["state"] == "active"
|
||||
assert next(
|
||||
stage
|
||||
for stage in document["pipeline"]["stages"]
|
||||
if stage["id"] == "semantic-model"
|
||||
)["state"] == "active"
|
||||
|
||||
|
||||
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
router = build_system_telemetry_router(
|
||||
root_provider=lambda: tmp_path / "system",
|
||||
probe_runner=lambda _: _probe(node_id="OTHER-NODE"),
|
||||
)
|
||||
apply_profile = _endpoint(router, "/api/v1/system/worker-profile", "PUT")
|
||||
|
||||
with pytest.raises(HTTPException) as error:
|
||||
apply_profile(
|
||||
WorkerConnectionProfilePut(
|
||||
revision=0,
|
||||
address="192.0.2.20",
|
||||
port=22,
|
||||
)
|
||||
)
|
||||
assert error.value.status_code == 409
|
||||
assert WorkerProfileStore(tmp_path / "system").read().revision == 0
|
||||
assert not WorkerProfileStore(tmp_path / "system").path.exists()
|
||||
Reference in New Issue
Block a user