feat(system): add Worker 006 telemetry and network profile
This commit is contained in:
@@ -795,6 +795,10 @@ export default function App() {
|
|||||||
<StatusBadge tone="neutral">Offline evaluation</StatusBadge>
|
<StatusBadge tone="neutral">Offline evaluation</StatusBadge>
|
||||||
) : activeDefinition.kind === "lab-archive" ? (
|
) : activeDefinition.kind === "lab-archive" ? (
|
||||||
null
|
null
|
||||||
|
) : activeDefinition.kind === "compute-modules" ? (
|
||||||
|
<StatusBadge tone="neutral">Живая телеметрия</StatusBadge>
|
||||||
|
) : activeDefinition.kind === "network-monitor" ? (
|
||||||
|
<StatusBadge tone="neutral">Живая сеть</StatusBadge>
|
||||||
) : (
|
) : (
|
||||||
<StatusBadge tone="warning">Интерфейс готов</StatusBadge>
|
<StatusBadge tone="warning">Интерфейс готов</StatusBadge>
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
interface TelemetrySeriesProps {
|
||||||
|
label: string;
|
||||||
|
values: Array<number | null>;
|
||||||
|
value: string;
|
||||||
|
ceiling?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function linePoints(values: Array<number | null>, ceiling?: number): string {
|
||||||
|
const finite = values.filter((value): value is number =>
|
||||||
|
value !== null && Number.isFinite(value),
|
||||||
|
);
|
||||||
|
if (finite.length === 0) return "";
|
||||||
|
const maximum = Math.max(ceiling ?? 0, ...finite, 1);
|
||||||
|
const denominator = Math.max(1, values.length - 1);
|
||||||
|
return values.map((value, index) => {
|
||||||
|
const x = index / denominator * 100;
|
||||||
|
const normalized = value === null ? 0 : Math.max(0, Math.min(maximum, value));
|
||||||
|
const y = 36 - normalized / maximum * 34;
|
||||||
|
return `${x.toFixed(2)},${y.toFixed(2)}`;
|
||||||
|
}).join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function TelemetrySeries({
|
||||||
|
label,
|
||||||
|
values,
|
||||||
|
value,
|
||||||
|
ceiling,
|
||||||
|
}: TelemetrySeriesProps) {
|
||||||
|
const points = linePoints(values, ceiling);
|
||||||
|
return (
|
||||||
|
<div className="system-telemetry-series">
|
||||||
|
<div>
|
||||||
|
<span>{label}</span>
|
||||||
|
<strong>{value}</strong>
|
||||||
|
</div>
|
||||||
|
<svg
|
||||||
|
viewBox="0 0 100 38"
|
||||||
|
preserveAspectRatio="none"
|
||||||
|
role="img"
|
||||||
|
aria-label={`${label}: ${value}`}
|
||||||
|
>
|
||||||
|
<path d="M0 37 H100" />
|
||||||
|
{points ? <polyline points={points} /> : null}
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import { GlassSurface, StatusBadge } from "@nodedc/ui-react";
|
||||||
|
|
||||||
|
import type { WorkerRuntime } from "../../core/system/workerTelemetry";
|
||||||
|
import {
|
||||||
|
formatOptionalPercent,
|
||||||
|
runtimeStateLabel,
|
||||||
|
runtimeTone,
|
||||||
|
} from "./systemFormat";
|
||||||
|
|
||||||
|
export function WorkerRuntimeCard({ runtime }: { runtime: WorkerRuntime }) {
|
||||||
|
return (
|
||||||
|
<GlassSurface
|
||||||
|
className="worker-runtime-card"
|
||||||
|
padding="md"
|
||||||
|
tone={runtime.external ? "soft" : "default"}
|
||||||
|
data-external={runtime.external ? "true" : undefined}
|
||||||
|
>
|
||||||
|
<header>
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">
|
||||||
|
{runtime.external ? "ВНЕШНЯЯ НАГРУЗКА" : "MISSION CORE RUNTIME"}
|
||||||
|
</span>
|
||||||
|
<h3>{runtime.role}</h3>
|
||||||
|
<code>{runtime.name}</code>
|
||||||
|
</div>
|
||||||
|
<StatusBadge tone={runtimeTone(runtime)}>
|
||||||
|
{runtimeStateLabel(runtime)}
|
||||||
|
</StatusBadge>
|
||||||
|
</header>
|
||||||
|
<dl>
|
||||||
|
<div>
|
||||||
|
<dt>CPU</dt>
|
||||||
|
<dd>{formatOptionalPercent(runtime.cpu_percent)}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Память</dt>
|
||||||
|
<dd>{runtime.memory_usage ?? "—"}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Сеть I/O</dt>
|
||||||
|
<dd>{runtime.network_io ?? "—"}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Процессы</dt>
|
||||||
|
<dd>{runtime.pids ?? "—"}</dd>
|
||||||
|
</div>
|
||||||
|
</dl>
|
||||||
|
</GlassSurface>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
import type { StatusTone } from "@nodedc/ui-react";
|
||||||
|
|
||||||
|
import type { WorkerRuntime } from "../../core/system/workerTelemetry";
|
||||||
|
|
||||||
|
export function formatOptionalPercent(value: number | null | undefined): string {
|
||||||
|
return typeof value === "number" && Number.isFinite(value)
|
||||||
|
? `${new Intl.NumberFormat("ru-RU", { maximumFractionDigits: 1 }).format(value)}%`
|
||||||
|
: "—";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatBytes(value: number | null | undefined): string {
|
||||||
|
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||||
|
const units = ["Б", "КиБ", "МиБ", "ГиБ", "ТиБ"];
|
||||||
|
let current = Math.max(0, value);
|
||||||
|
let unit = 0;
|
||||||
|
while (current >= 1024 && unit < units.length - 1) {
|
||||||
|
current /= 1024;
|
||||||
|
unit += 1;
|
||||||
|
}
|
||||||
|
return `${new Intl.NumberFormat("ru-RU", {
|
||||||
|
maximumFractionDigits: current >= 100 ? 0 : 1,
|
||||||
|
}).format(current)} ${units[unit]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatRate(value: number | null | undefined): string {
|
||||||
|
const formatted = formatBytes(value);
|
||||||
|
return formatted === "—" ? formatted : `${formatted}/с`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatBitRate(value: number | null | undefined): string {
|
||||||
|
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||||
|
const units = ["бит/с", "Кбит/с", "Мбит/с", "Гбит/с"];
|
||||||
|
let current = Math.max(0, value);
|
||||||
|
let unit = 0;
|
||||||
|
while (current >= 1000 && unit < units.length - 1) {
|
||||||
|
current /= 1000;
|
||||||
|
unit += 1;
|
||||||
|
}
|
||||||
|
return `${new Intl.NumberFormat("ru-RU", {
|
||||||
|
maximumFractionDigits: current >= 100 ? 0 : 1,
|
||||||
|
}).format(current)} ${units[unit]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatDuration(value: number | null | undefined): string {
|
||||||
|
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||||
|
const hours = Math.floor(value / 3600);
|
||||||
|
if (hours < 24) return `${hours} ч`;
|
||||||
|
return `${Math.floor(hours / 24)} д ${hours % 24} ч`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatLatency(value: number | null | undefined): string {
|
||||||
|
if (typeof value !== "number" || !Number.isFinite(value)) return "—";
|
||||||
|
return `${Math.round(value)} мс`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runtimeTone(runtime: WorkerRuntime): StatusTone {
|
||||||
|
if (runtime.state !== "running") return "danger";
|
||||||
|
if (runtime.health && runtime.health !== "healthy") return "warning";
|
||||||
|
return runtime.external ? "neutral" : "success";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runtimeStateLabel(runtime: WorkerRuntime): string {
|
||||||
|
if (runtime.state !== "running") return "Недоступен";
|
||||||
|
if (runtime.health === "healthy") return "Работает";
|
||||||
|
if (runtime.health) return runtime.health;
|
||||||
|
return "Запущен";
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
import { useCallback, useEffect, useState } from "react";
|
||||||
|
|
||||||
|
import {
|
||||||
|
fetchWorkerTelemetry,
|
||||||
|
type WorkerTelemetry,
|
||||||
|
} from "./workerTelemetry";
|
||||||
|
|
||||||
|
export interface WorkerTelemetryState {
|
||||||
|
telemetry: WorkerTelemetry | null;
|
||||||
|
loading: boolean;
|
||||||
|
error: string | null;
|
||||||
|
refresh: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useWorkerTelemetry(pollMilliseconds = 10_000): WorkerTelemetryState {
|
||||||
|
const [telemetry, setTelemetry] = useState<WorkerTelemetry | null>(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [generation, setGeneration] = useState(0);
|
||||||
|
const refresh = useCallback(() => setGeneration((value) => value + 1), []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
setLoading(true);
|
||||||
|
void fetchWorkerTelemetry(controller.signal)
|
||||||
|
.then((document) => {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
setTelemetry(document);
|
||||||
|
setError(null);
|
||||||
|
})
|
||||||
|
.catch((reason: unknown) => {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
setError(reason instanceof Error ? reason.message : "Worker 006 не ответил.");
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!controller.signal.aborted) setLoading(false);
|
||||||
|
});
|
||||||
|
return () => controller.abort();
|
||||||
|
}, [generation]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const timer = window.setInterval(refresh, pollMilliseconds);
|
||||||
|
return () => window.clearInterval(timer);
|
||||||
|
}, [pollMilliseconds, refresh]);
|
||||||
|
|
||||||
|
return { telemetry, loading, error, refresh };
|
||||||
|
}
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
export interface WorkerConnectionProfile {
|
||||||
|
schema_version: "missioncore.worker-connection-profile/v1";
|
||||||
|
profile_id: string;
|
||||||
|
display_name: string;
|
||||||
|
expected_node_id: string;
|
||||||
|
ssh_host_alias: string;
|
||||||
|
address: string;
|
||||||
|
port: number;
|
||||||
|
revision: number;
|
||||||
|
updated_at_utc: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerProbe {
|
||||||
|
schema_version: "missioncore.worker-probe/v1";
|
||||||
|
reachable: boolean;
|
||||||
|
identity_matches: boolean;
|
||||||
|
node_id: string | null;
|
||||||
|
latency_ms: number | null;
|
||||||
|
observed_at_utc: string;
|
||||||
|
error_code: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerProfileDocument {
|
||||||
|
schema_version: "missioncore.worker-connection-profile/v1";
|
||||||
|
profile: WorkerConnectionProfile;
|
||||||
|
security: {
|
||||||
|
transport: "ssh";
|
||||||
|
host_key_policy: "strict-pinned";
|
||||||
|
credentials_managed_by_ui: false;
|
||||||
|
ssh_host_alias: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerRuntime {
|
||||||
|
name: string;
|
||||||
|
role: string;
|
||||||
|
owner: "mission-core" | "external";
|
||||||
|
external: boolean;
|
||||||
|
image: string | null;
|
||||||
|
state: string;
|
||||||
|
health: string | null;
|
||||||
|
cpu_percent: number | null;
|
||||||
|
memory_percent: number | null;
|
||||||
|
memory_usage: string | null;
|
||||||
|
network_io: string | null;
|
||||||
|
block_io: string | null;
|
||||||
|
pids: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerPipelineStage {
|
||||||
|
id: string;
|
||||||
|
label: string;
|
||||||
|
state: "active" | "waiting" | "ready" | "unavailable";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerTelemetryHistoryRow {
|
||||||
|
observed_at_utc: string;
|
||||||
|
cpu_percent: number | null;
|
||||||
|
memory_percent: number | null;
|
||||||
|
gpu_percent: number | null;
|
||||||
|
gpu_memory_percent: number | null;
|
||||||
|
network_receive_bytes_per_second: number | null;
|
||||||
|
network_send_bytes_per_second: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerNetworkInterface {
|
||||||
|
name: string;
|
||||||
|
description: string | null;
|
||||||
|
status: string | null;
|
||||||
|
link_speed_bps: number | null;
|
||||||
|
mac_address: string | null;
|
||||||
|
addresses: string[];
|
||||||
|
received_bytes: number | null;
|
||||||
|
sent_bytes: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerTelemetry {
|
||||||
|
schema_version: "missioncore.worker-telemetry/v1";
|
||||||
|
profile: WorkerConnectionProfile;
|
||||||
|
connection: WorkerProbe;
|
||||||
|
node: {
|
||||||
|
node_id: string;
|
||||||
|
observed_at_utc: string;
|
||||||
|
os: {
|
||||||
|
caption?: string;
|
||||||
|
version?: string;
|
||||||
|
uptime_seconds?: number;
|
||||||
|
};
|
||||||
|
cpu: {
|
||||||
|
name?: string;
|
||||||
|
logical_processors?: number;
|
||||||
|
load_percent?: number;
|
||||||
|
};
|
||||||
|
memory: {
|
||||||
|
total_bytes: number | null;
|
||||||
|
used_bytes: number | null;
|
||||||
|
free_bytes: number | null;
|
||||||
|
used_percent: number | null;
|
||||||
|
};
|
||||||
|
disks: Array<{
|
||||||
|
name?: string;
|
||||||
|
size_bytes?: number;
|
||||||
|
free_bytes?: number;
|
||||||
|
}>;
|
||||||
|
gpu: {
|
||||||
|
name?: string;
|
||||||
|
utilization_percent?: number;
|
||||||
|
memory_used_mib?: number;
|
||||||
|
memory_total_mib?: number;
|
||||||
|
power_watts?: number;
|
||||||
|
temperature_celsius?: number;
|
||||||
|
memory_used_percent?: number;
|
||||||
|
} | null;
|
||||||
|
triton: {
|
||||||
|
ready: boolean;
|
||||||
|
requests_succeeded: number | null;
|
||||||
|
requests_failed: number | null;
|
||||||
|
inferences: number | null;
|
||||||
|
};
|
||||||
|
} | null;
|
||||||
|
runtimes: WorkerRuntime[];
|
||||||
|
pipeline: {
|
||||||
|
service_state: string;
|
||||||
|
current_stage: string | null;
|
||||||
|
active_request_id: string | null;
|
||||||
|
active_frame_index: number | null;
|
||||||
|
completed_runs: number | null;
|
||||||
|
failed_runs: number | null;
|
||||||
|
model_load_seconds: number | null;
|
||||||
|
stages: WorkerPipelineStage[];
|
||||||
|
};
|
||||||
|
network: {
|
||||||
|
interfaces: WorkerNetworkInterface[];
|
||||||
|
aggregate: {
|
||||||
|
received_bytes: number;
|
||||||
|
sent_bytes: number;
|
||||||
|
receive_bytes_per_second: number | null;
|
||||||
|
send_bytes_per_second: number | null;
|
||||||
|
} | null;
|
||||||
|
};
|
||||||
|
history: WorkerTelemetryHistoryRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerProfileMutation {
|
||||||
|
revision: number;
|
||||||
|
address: string;
|
||||||
|
port: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireJsonRecord(value: unknown, schema: string): Record<string, unknown> {
|
||||||
|
if (!isRecord(value) || value.schema_version !== schema) {
|
||||||
|
throw new Error("Ответ системной телеметрии не соответствует контракту.");
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestJson(
|
||||||
|
url: string,
|
||||||
|
init: RequestInit,
|
||||||
|
): Promise<unknown> {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
...init,
|
||||||
|
headers: {
|
||||||
|
Accept: "application/json",
|
||||||
|
...(init.body ? { "Content-Type": "application/json" } : {}),
|
||||||
|
...init.headers,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`Системный API вернул HTTP ${response.status}.`);
|
||||||
|
}
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchWorkerTelemetry(
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<WorkerTelemetry> {
|
||||||
|
const document = requireJsonRecord(
|
||||||
|
await requestJson("/api/v1/system/worker-telemetry?history=90", {
|
||||||
|
method: "GET",
|
||||||
|
signal,
|
||||||
|
}),
|
||||||
|
"missioncore.worker-telemetry/v1",
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
!isRecord(document.connection)
|
||||||
|
|| !isRecord(document.profile)
|
||||||
|
|| !Array.isArray(document.runtimes)
|
||||||
|
|| !isRecord(document.pipeline)
|
||||||
|
|| !isRecord(document.network)
|
||||||
|
|| !Array.isArray(document.history)
|
||||||
|
) {
|
||||||
|
throw new Error("Срез Worker 006 неполон.");
|
||||||
|
}
|
||||||
|
return document as unknown as WorkerTelemetry;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchWorkerProfile(
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<WorkerProfileDocument> {
|
||||||
|
const document = requireJsonRecord(
|
||||||
|
await requestJson("/api/v1/system/worker-profile", {
|
||||||
|
method: "GET",
|
||||||
|
signal,
|
||||||
|
}),
|
||||||
|
"missioncore.worker-connection-profile/v1",
|
||||||
|
);
|
||||||
|
if (!isRecord(document.profile) || !isRecord(document.security)) {
|
||||||
|
throw new Error("Профиль Worker 006 неполон.");
|
||||||
|
}
|
||||||
|
return document as unknown as WorkerProfileDocument;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function testWorkerProfile(
|
||||||
|
request: WorkerProfileMutation,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<WorkerProbe> {
|
||||||
|
const document = requireJsonRecord(
|
||||||
|
await requestJson("/api/v1/system/worker-profile/test", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(request),
|
||||||
|
signal,
|
||||||
|
}),
|
||||||
|
"missioncore.worker-probe/v1",
|
||||||
|
);
|
||||||
|
return document as unknown as WorkerProbe;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function saveWorkerProfile(
|
||||||
|
request: WorkerProfileMutation,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<WorkerProfileDocument & { verification: WorkerProbe }> {
|
||||||
|
const document = requireJsonRecord(
|
||||||
|
await requestJson("/api/v1/system/worker-profile", {
|
||||||
|
method: "PUT",
|
||||||
|
body: JSON.stringify(request),
|
||||||
|
signal,
|
||||||
|
}),
|
||||||
|
"missioncore.worker-connection-profile/v1",
|
||||||
|
);
|
||||||
|
if (!isRecord(document.profile) || !isRecord(document.verification)) {
|
||||||
|
throw new Error("Сохранённый профиль Worker 006 неполон.");
|
||||||
|
}
|
||||||
|
return document as unknown as WorkerProfileDocument & { verification: WorkerProbe };
|
||||||
|
}
|
||||||
@@ -18,6 +18,8 @@ export type WorkspaceKind =
|
|||||||
| "missions"
|
| "missions"
|
||||||
| "catalog"
|
| "catalog"
|
||||||
| "contour-health"
|
| "contour-health"
|
||||||
|
| "compute-modules"
|
||||||
|
| "network-monitor"
|
||||||
| "datasets"
|
| "datasets"
|
||||||
| "lab-archive";
|
| "lab-archive";
|
||||||
|
|
||||||
@@ -589,16 +591,17 @@ export const workspaces: WorkspaceDefinition[] = [
|
|||||||
label: "Вычислительные модули",
|
label: "Вычислительные модули",
|
||||||
title: "Вычислительные модули",
|
title: "Вычислительные модули",
|
||||||
eyebrow: "СИСТЕМА / МОДУЛИ",
|
eyebrow: "СИСТЕМА / МОДУЛИ",
|
||||||
description: "Контракты будущих вычислительных модулей без их запуска на текущем интерфейсном этапе.",
|
description: "Живая аппаратная и процессинговая телеметрия выделенного Worker 006.",
|
||||||
icon: "apps",
|
icon: "apps",
|
||||||
kind: "catalog",
|
kind: "compute-modules",
|
||||||
groups: [
|
groups: [
|
||||||
{
|
{
|
||||||
title: "Вычислительные роли",
|
title: "Вычислительные роли",
|
||||||
description: "Каждый модуль публикует стандартные выходы, интерфейс только отображает их.",
|
description: "Каждый модуль публикует стандартные выходы, интерфейс только отображает их.",
|
||||||
capabilities: [
|
capabilities: [
|
||||||
active("Адаптер устройства", "Закрытый протокол текущего устройства уже изолирован."),
|
active("Адаптер устройства", "Закрытый протокол текущего устройства уже изолирован."),
|
||||||
contract("Восприятие", "Рамки, маски, ключевые точки, траектории и семантические подписи."),
|
active("Восприятие", "Persistent perception worker и его текущие стадии обработки."),
|
||||||
|
active("Inference runtime", "Triton health, inference-счётчики и потребление ресурсов."),
|
||||||
contract("Картирование", "Высота, занятость, сетки и проходимость."),
|
contract("Картирование", "Высота, занятость, сетки и проходимость."),
|
||||||
contract("Локализация", "Поза, одометрия, TF и неопределённость."),
|
contract("Локализация", "Поза, одометрия, TF и неопределённость."),
|
||||||
contract("Исполнитель миссий", "Состояние графа миссии и действия верхнего уровня."),
|
contract("Исполнитель миссий", "Состояние графа миссии и действия верхнего уровня."),
|
||||||
@@ -637,13 +640,14 @@ export const workspaces: WorkspaceDefinition[] = [
|
|||||||
eyebrow: "СИСТЕМА / СЕТЬ",
|
eyebrow: "СИСТЕМА / СЕТЬ",
|
||||||
description: "Локальный контур, бортовая сеть, ячеистая сеть/LTE и серверные маршруты.",
|
description: "Локальный контур, бортовая сеть, ячеистая сеть/LTE и серверные маршруты.",
|
||||||
icon: "globe",
|
icon: "globe",
|
||||||
kind: "catalog",
|
kind: "network-monitor",
|
||||||
groups: [
|
groups: [
|
||||||
{
|
{
|
||||||
title: "Сетевые зоны",
|
title: "Сетевые зоны",
|
||||||
description: "Потоки управления и высокой пропускной способности разделяются архитектурно.",
|
description: "Потоки управления и высокой пропускной способности разделяются архитектурно.",
|
||||||
capabilities: [
|
capabilities: [
|
||||||
active("Локальная сеть", "Устройство и рабочая станция в одной доступной сети."),
|
active("Переносимый профиль", "Адрес и порт Worker 006 проверяются до применения."),
|
||||||
|
active("Сетевая телеметрия", "Интерфейсы, счётчики, throughput и фактический маршрут."),
|
||||||
active("Локальное управление", "Точка приёма учётных данных и браузерный контур управления ограничены локальным хостом."),
|
active("Локальное управление", "Точка приёма учётных данных и браузерный контур управления ограничены локальным хостом."),
|
||||||
contract("Бортовая сеть", "Сенсоры, вычислитель и маршрутизатор внутри аппарата."),
|
contract("Бортовая сеть", "Сенсоры, вычислитель и маршрутизатор внутри аппарата."),
|
||||||
contract("Дальний транспорт", "Ячеистая сеть или LTE передаёт предпросмотр, позу, события и изменения карты."),
|
contract("Дальний транспорт", "Ячеистая сеть или LTE передаёт предпросмотр, позу, события и изменения карты."),
|
||||||
|
|||||||
@@ -11,3 +11,4 @@
|
|||||||
@import "./styles/responsive.css";
|
@import "./styles/responsive.css";
|
||||||
@import "./styles/observation.css";
|
@import "./styles/observation.css";
|
||||||
@import "./styles/environment-settings.css";
|
@import "./styles/environment-settings.css";
|
||||||
|
@import "./styles/system-telemetry.css";
|
||||||
|
|||||||
@@ -0,0 +1,544 @@
|
|||||||
|
.system-workspace {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 1rem;
|
||||||
|
padding-bottom: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__lead,
|
||||||
|
.system-section-heading {
|
||||||
|
display: flex;
|
||||||
|
min-width: 0;
|
||||||
|
align-items: flex-start;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 1.2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__lead {
|
||||||
|
padding: 0.25rem 0 0.35rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__lead h2,
|
||||||
|
.system-section-heading h3 {
|
||||||
|
margin: 0.4rem 0 0;
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
letter-spacing: -0.035em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__lead h2 {
|
||||||
|
font-size: 1.42rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-section-heading h3 {
|
||||||
|
font-size: 1.02rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__lead p,
|
||||||
|
.system-section-heading p {
|
||||||
|
max-width: 48rem;
|
||||||
|
margin: 0.42rem 0 0;
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.68rem;
|
||||||
|
line-height: 1.48;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__actions {
|
||||||
|
display: flex;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.8rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__notice {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-grid,
|
||||||
|
.network-overview-grid {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
|
gap: 0.65rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series,
|
||||||
|
.network-stat-card {
|
||||||
|
min-width: 0;
|
||||||
|
border: 1px solid var(--station-hairline);
|
||||||
|
border-radius: var(--nodedc-radius-card);
|
||||||
|
background: var(--station-panel-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series {
|
||||||
|
overflow: hidden;
|
||||||
|
padding: 0.82rem 0.82rem 0.32rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series > div {
|
||||||
|
display: flex;
|
||||||
|
align-items: baseline;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series span,
|
||||||
|
.network-stat-card span {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.62rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series strong,
|
||||||
|
.network-stat-card strong {
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 1.14rem;
|
||||||
|
font-weight: 720;
|
||||||
|
letter-spacing: -0.035em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series svg {
|
||||||
|
display: block;
|
||||||
|
width: 100%;
|
||||||
|
height: 2.45rem;
|
||||||
|
margin-top: 0.45rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series path {
|
||||||
|
fill: none;
|
||||||
|
stroke: var(--station-hairline);
|
||||||
|
stroke-width: 0.8;
|
||||||
|
vector-effect: non-scaling-stroke;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-series polyline {
|
||||||
|
fill: none;
|
||||||
|
stroke: var(--nodedc-text-primary);
|
||||||
|
stroke-linecap: round;
|
||||||
|
stroke-linejoin: round;
|
||||||
|
stroke-width: 1.15;
|
||||||
|
vector-effect: non-scaling-stroke;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware,
|
||||||
|
.worker-pipeline,
|
||||||
|
.network-profile,
|
||||||
|
.network-topology {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware__facts {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 0.7rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware__facts dl,
|
||||||
|
.worker-runtime-card dl,
|
||||||
|
.network-interface-card dl {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
margin: 0;
|
||||||
|
gap: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware__facts dl {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
border: 1px solid var(--station-hairline);
|
||||||
|
border-radius: var(--nodedc-radius-control);
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware__facts dl > div,
|
||||||
|
.worker-runtime-card dl > div,
|
||||||
|
.network-interface-card dl > div {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.22rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware__facts dl > div {
|
||||||
|
padding: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware__facts dt,
|
||||||
|
.worker-runtime-card dt,
|
||||||
|
.network-interface-card dt,
|
||||||
|
.network-profile__security dt {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.58rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-hardware__facts dd,
|
||||||
|
.worker-runtime-card dd,
|
||||||
|
.network-interface-card dd,
|
||||||
|
.network-profile__security dd {
|
||||||
|
min-width: 0;
|
||||||
|
margin: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.67rem;
|
||||||
|
font-weight: 650;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-disk-list {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
|
||||||
|
gap: 0.55rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-disk-list > div {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 0.8rem;
|
||||||
|
padding: 0.62rem 0.72rem;
|
||||||
|
border-radius: var(--nodedc-radius-control);
|
||||||
|
background: var(--station-accent-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-disk-list span {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.61rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-disk-list strong {
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.64rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-runtime-section {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-runtime-section--external {
|
||||||
|
padding-top: 0.25rem;
|
||||||
|
border-top: 1px solid var(--station-hairline);
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-grid {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 0.7rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card[data-external="true"] {
|
||||||
|
opacity: 0.78;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card > header,
|
||||||
|
.network-interface-card > header {
|
||||||
|
display: flex;
|
||||||
|
min-width: 0;
|
||||||
|
align-items: flex-start;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 0.8rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card h3 {
|
||||||
|
margin: 0.38rem 0 0.18rem;
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.9rem;
|
||||||
|
letter-spacing: -0.025em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card code {
|
||||||
|
display: block;
|
||||||
|
max-width: 23rem;
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.56rem;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card dl {
|
||||||
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card dl > div {
|
||||||
|
padding-right: 0.65rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-pipeline__summary {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
|
gap: 0.55rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-pipeline__summary > div {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.32rem;
|
||||||
|
padding: 0.7rem;
|
||||||
|
border-radius: var(--nodedc-radius-control);
|
||||||
|
background: var(--station-panel-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-pipeline__summary span {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.57rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-pipeline__summary strong {
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.78rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-stage-list {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
gap: 0.48rem;
|
||||||
|
list-style: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-stage-list li {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.55rem;
|
||||||
|
padding: 0.66rem;
|
||||||
|
border: 1px solid var(--station-hairline);
|
||||||
|
border-radius: var(--nodedc-radius-control);
|
||||||
|
background: var(--station-panel-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-stage-list li[data-state="active"] {
|
||||||
|
border-color: var(--station-hairline-strong);
|
||||||
|
background: var(--nodedc-focus-surface);
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-stage-list li > span,
|
||||||
|
.worker-stage-list li > small {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.53rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-stage-list li > strong {
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.62rem;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-stat-card {
|
||||||
|
display: grid;
|
||||||
|
align-content: center;
|
||||||
|
gap: 0.3rem;
|
||||||
|
padding: 0.82rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-stat-card small {
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.55rem;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__form {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: minmax(12rem, 1.5fr) minmax(8rem, 0.5fr) auto;
|
||||||
|
align-items: end;
|
||||||
|
gap: 0.7rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__buttons {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.5rem;
|
||||||
|
padding-bottom: 0.02rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__security {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
|
margin: 0;
|
||||||
|
gap: 0.55rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__security > div {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.24rem;
|
||||||
|
padding: 0.64rem 0.7rem;
|
||||||
|
border-radius: var(--nodedc-radius-control);
|
||||||
|
background: var(--station-accent-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: repeat(3, minmax(8rem, 1fr) 2.2rem) minmax(8rem, 1fr);
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route > div {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.3rem;
|
||||||
|
padding: 0.78rem;
|
||||||
|
border: 1px solid var(--station-hairline);
|
||||||
|
border-radius: var(--nodedc-radius-control);
|
||||||
|
background: var(--station-panel-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route > div[data-state="offline"] {
|
||||||
|
opacity: 0.48;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route > div span,
|
||||||
|
.network-route > div small {
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.55rem;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route > div strong {
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.68rem;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route > i {
|
||||||
|
height: 1px;
|
||||||
|
background: var(--station-hairline-strong);
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route > i[data-state="offline"] {
|
||||||
|
background: var(--station-hairline);
|
||||||
|
opacity: 0.4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-interface-section {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-interface-list {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
gap: 0.65rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-interface-card {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-interface-card header strong,
|
||||||
|
.network-interface-card header small {
|
||||||
|
display: block;
|
||||||
|
max-width: 17rem;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-interface-card header strong {
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.71rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-interface-card header small {
|
||||||
|
margin-top: 0.25rem;
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.55rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-interface-card dl {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
gap: 0.65rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 1100px) {
|
||||||
|
.system-telemetry-grid,
|
||||||
|
.network-overview-grid,
|
||||||
|
.worker-pipeline__summary,
|
||||||
|
.network-profile__security {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-stage-list,
|
||||||
|
.network-interface-list {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__form {
|
||||||
|
grid-template-columns: minmax(12rem, 1fr) 8rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__buttons {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
gap: 0.4rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-route > i {
|
||||||
|
width: 1px;
|
||||||
|
height: 1.2rem;
|
||||||
|
margin-left: 1.1rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 760px) {
|
||||||
|
.system-workspace__lead,
|
||||||
|
.system-section-heading {
|
||||||
|
display: grid;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-workspace__actions {
|
||||||
|
justify-content: space-between;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-telemetry-grid,
|
||||||
|
.network-overview-grid,
|
||||||
|
.worker-hardware__facts,
|
||||||
|
.worker-runtime-grid,
|
||||||
|
.worker-stage-list,
|
||||||
|
.network-interface-list,
|
||||||
|
.network-profile__security {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
|
||||||
|
.worker-runtime-card dl,
|
||||||
|
.worker-pipeline__summary {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__form {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-profile__buttons {
|
||||||
|
grid-column: auto;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -46,6 +46,8 @@ import type { WorkspaceRendererProps } from "./contracts";
|
|||||||
import { DatasetGatewayWorkspace } from "./DatasetGatewayWorkspace";
|
import { DatasetGatewayWorkspace } from "./DatasetGatewayWorkspace";
|
||||||
import { ContourHealthWorkspace } from "./ContourHealthWorkspace";
|
import { ContourHealthWorkspace } from "./ContourHealthWorkspace";
|
||||||
import { LaboratoryArchiveWorkspace } from "./laboratory/LaboratoryArchiveWorkspace";
|
import { LaboratoryArchiveWorkspace } from "./laboratory/LaboratoryArchiveWorkspace";
|
||||||
|
import { ComputeModulesWorkspace } from "./system/ComputeModulesWorkspace";
|
||||||
|
import { NetworkWorkspace } from "./system/NetworkWorkspace";
|
||||||
|
|
||||||
function statusTone(status: CapabilityStatus): "success" | "accent" | "warning" | "neutral" {
|
function statusTone(status: CapabilityStatus): "success" | "accent" | "warning" | "neutral" {
|
||||||
if (status === "active") return "success";
|
if (status === "active") return "success";
|
||||||
@@ -1177,6 +1179,10 @@ export function WorkspaceRenderer(props: WorkspaceRendererProps) {
|
|||||||
state={props.state}
|
state={props.state}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
|
case "compute-modules":
|
||||||
|
return <ComputeModulesWorkspace />;
|
||||||
|
case "network-monitor":
|
||||||
|
return <NetworkWorkspace />;
|
||||||
case "datasets":
|
case "datasets":
|
||||||
return <DatasetGatewayWorkspace />;
|
return <DatasetGatewayWorkspace />;
|
||||||
case "lab-archive":
|
case "lab-archive":
|
||||||
|
|||||||
@@ -0,0 +1,226 @@
|
|||||||
|
import {
|
||||||
|
Button,
|
||||||
|
GlassSurface,
|
||||||
|
Icon,
|
||||||
|
StatusBadge,
|
||||||
|
} from "@nodedc/ui-react";
|
||||||
|
|
||||||
|
import { TelemetrySeries } from "../../components/system/TelemetrySeries";
|
||||||
|
import { WorkerRuntimeCard } from "../../components/system/WorkerRuntimeCard";
|
||||||
|
import {
|
||||||
|
formatBytes,
|
||||||
|
formatDuration,
|
||||||
|
formatOptionalPercent,
|
||||||
|
} from "../../components/system/systemFormat";
|
||||||
|
import { useWorkerTelemetry } from "../../core/system/useWorkerTelemetry";
|
||||||
|
|
||||||
|
function pipelineStateLabel(state: string): string {
|
||||||
|
if (state === "busy") return "Выполняет задачу";
|
||||||
|
if (state === "ready") return "Готов к задаче";
|
||||||
|
return "Нет live-состояния";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ComputeModulesWorkspace() {
|
||||||
|
const { telemetry, loading, error, refresh } = useWorkerTelemetry();
|
||||||
|
const node = telemetry?.node ?? null;
|
||||||
|
const missionCoreRuntimes = telemetry?.runtimes.filter((runtime) => !runtime.external) ?? [];
|
||||||
|
const externalRuntimes = telemetry?.runtimes.filter((runtime) => runtime.external) ?? [];
|
||||||
|
const connected = Boolean(
|
||||||
|
telemetry?.connection.reachable && telemetry.connection.identity_matches && node,
|
||||||
|
);
|
||||||
|
const history = telemetry?.history ?? [];
|
||||||
|
const cpuPercent = node?.cpu.load_percent;
|
||||||
|
const memoryPercent = node?.memory.used_percent;
|
||||||
|
const gpuPercent = node?.gpu?.utilization_percent;
|
||||||
|
const gpuMemoryPercent = node?.gpu?.memory_used_percent;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="system-workspace compute-modules-workspace">
|
||||||
|
<section className="system-workspace__lead">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">СИСТЕМА / ВЫЧИСЛИТЕЛЬНЫЕ МОДУЛИ</span>
|
||||||
|
<h2>Worker 006</h2>
|
||||||
|
<p>
|
||||||
|
Живой аппаратный и процессинговый срез выделенного узла. Нагрузка внешних
|
||||||
|
сервисов отделена от Mission Core и не входит в оценку наших runtime.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="system-workspace__actions">
|
||||||
|
<StatusBadge tone={connected ? "success" : "danger"}>
|
||||||
|
{connected ? "Узел доступен" : "Нет связи с узлом"}
|
||||||
|
</StatusBadge>
|
||||||
|
<Button
|
||||||
|
size="compact"
|
||||||
|
variant="secondary"
|
||||||
|
disabled={loading}
|
||||||
|
onClick={refresh}
|
||||||
|
icon={<Icon name="refresh" size={14} />}
|
||||||
|
>
|
||||||
|
{loading ? "Читаем" : "Обновить"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{error ? (
|
||||||
|
<GlassSurface className="system-workspace__notice" padding="md" tone="soft">
|
||||||
|
<StatusBadge tone="warning">{error}</StatusBadge>
|
||||||
|
</GlassSurface>
|
||||||
|
) : null}
|
||||||
|
|
||||||
|
<section className="system-telemetry-grid" aria-label="Аппаратная телеметрия">
|
||||||
|
<TelemetrySeries
|
||||||
|
label="CPU"
|
||||||
|
value={formatOptionalPercent(cpuPercent)}
|
||||||
|
ceiling={100}
|
||||||
|
values={history.map((item) => item.cpu_percent)}
|
||||||
|
/>
|
||||||
|
<TelemetrySeries
|
||||||
|
label="RAM"
|
||||||
|
value={formatOptionalPercent(memoryPercent)}
|
||||||
|
ceiling={100}
|
||||||
|
values={history.map((item) => item.memory_percent)}
|
||||||
|
/>
|
||||||
|
<TelemetrySeries
|
||||||
|
label="GPU"
|
||||||
|
value={formatOptionalPercent(gpuPercent)}
|
||||||
|
ceiling={100}
|
||||||
|
values={history.map((item) => item.gpu_percent)}
|
||||||
|
/>
|
||||||
|
<TelemetrySeries
|
||||||
|
label="VRAM"
|
||||||
|
value={formatOptionalPercent(gpuMemoryPercent)}
|
||||||
|
ceiling={100}
|
||||||
|
values={history.map((item) => item.gpu_memory_percent)}
|
||||||
|
/>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<GlassSurface className="worker-hardware" padding="lg">
|
||||||
|
<header className="system-section-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">HARDWARE</span>
|
||||||
|
<h3>{node?.node_id ?? telemetry?.profile.expected_node_id ?? "Worker 006"}</h3>
|
||||||
|
<p>{node?.os.caption ?? "Аппаратный профиль недоступен"}</p>
|
||||||
|
</div>
|
||||||
|
<StatusBadge tone={node?.node_id ? "success" : "danger"}>
|
||||||
|
{node?.node_id ? telemetry?.profile.display_name : "Нет данных"}
|
||||||
|
</StatusBadge>
|
||||||
|
</header>
|
||||||
|
<div className="worker-hardware__facts">
|
||||||
|
<dl>
|
||||||
|
<div><dt>Процессор</dt><dd>{node?.cpu.name ?? "—"}</dd></div>
|
||||||
|
<div><dt>Логические ядра</dt><dd>{node?.cpu.logical_processors ?? "—"}</dd></div>
|
||||||
|
<div><dt>Память занята</dt><dd>{formatBytes(node?.memory.used_bytes)}</dd></div>
|
||||||
|
<div><dt>Uptime</dt><dd>{formatDuration(node?.os.uptime_seconds)}</dd></div>
|
||||||
|
</dl>
|
||||||
|
<dl>
|
||||||
|
<div><dt>GPU</dt><dd>{node?.gpu?.name ?? "—"}</dd></div>
|
||||||
|
<div><dt>VRAM занята</dt><dd>{formatBytes(
|
||||||
|
typeof node?.gpu?.memory_used_mib === "number"
|
||||||
|
? node.gpu.memory_used_mib * 1024 * 1024
|
||||||
|
: null,
|
||||||
|
)}</dd></div>
|
||||||
|
<div><dt>Температура</dt><dd>{
|
||||||
|
typeof node?.gpu?.temperature_celsius === "number"
|
||||||
|
? `${node.gpu.temperature_celsius} °C`
|
||||||
|
: "—"
|
||||||
|
}</dd></div>
|
||||||
|
<div><dt>Мощность</dt><dd>{
|
||||||
|
typeof node?.gpu?.power_watts === "number"
|
||||||
|
? `${node.gpu.power_watts.toFixed(1)} Вт`
|
||||||
|
: "—"
|
||||||
|
}</dd></div>
|
||||||
|
</dl>
|
||||||
|
</div>
|
||||||
|
<div className="worker-disk-list">
|
||||||
|
{(node?.disks ?? []).map((disk) => {
|
||||||
|
const size = typeof disk.size_bytes === "number" ? disk.size_bytes : null;
|
||||||
|
const free = typeof disk.free_bytes === "number" ? disk.free_bytes : null;
|
||||||
|
const used = size !== null && free !== null ? size - free : null;
|
||||||
|
return (
|
||||||
|
<div key={disk.name ?? "disk"}>
|
||||||
|
<span>Диск {disk.name ?? "—"}</span>
|
||||||
|
<strong>{formatBytes(used)} / {formatBytes(size)}</strong>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</GlassSurface>
|
||||||
|
|
||||||
|
<section className="system-runtime-section">
|
||||||
|
<header className="system-section-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">PROCESSING RUNTIME</span>
|
||||||
|
<h3>Контейнеры Mission Core</h3>
|
||||||
|
<p>Два ограниченных runtime: inference server и прикладной perception pipeline.</p>
|
||||||
|
</div>
|
||||||
|
<StatusBadge tone={node?.triton.ready ? "success" : "danger"}>
|
||||||
|
{node?.triton.ready ? "Triton ready" : "Triton недоступен"}
|
||||||
|
</StatusBadge>
|
||||||
|
</header>
|
||||||
|
<div className="worker-runtime-grid">
|
||||||
|
{missionCoreRuntimes.map((runtime) => (
|
||||||
|
<WorkerRuntimeCard key={runtime.name} runtime={runtime} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<GlassSurface className="worker-pipeline" padding="lg">
|
||||||
|
<header className="system-section-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">ТЕКУЩАЯ ЗАДАЧА</span>
|
||||||
|
<h3>{pipelineStateLabel(telemetry?.pipeline.service_state ?? "unavailable")}</h3>
|
||||||
|
<p>
|
||||||
|
{telemetry?.pipeline.active_request_id
|
||||||
|
? `Run ${telemetry.pipeline.active_request_id}`
|
||||||
|
: "Очередь свободна; модели остаются загруженными в persistent worker."}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<StatusBadge tone={
|
||||||
|
telemetry?.pipeline.service_state === "busy" ? "warning"
|
||||||
|
: telemetry?.pipeline.service_state === "ready" ? "success"
|
||||||
|
: "danger"
|
||||||
|
}>
|
||||||
|
{telemetry?.pipeline.service_state ?? "unavailable"}
|
||||||
|
</StatusBadge>
|
||||||
|
</header>
|
||||||
|
<div className="worker-pipeline__summary">
|
||||||
|
<div><span>Завершено запусков</span><strong>{telemetry?.pipeline.completed_runs ?? "—"}</strong></div>
|
||||||
|
<div><span>Ошибок запусков</span><strong>{telemetry?.pipeline.failed_runs ?? "—"}</strong></div>
|
||||||
|
<div><span>Inference success</span><strong>{node?.triton.requests_succeeded ?? "—"}</strong></div>
|
||||||
|
<div><span>Inference failed</span><strong>{node?.triton.requests_failed ?? "—"}</strong></div>
|
||||||
|
</div>
|
||||||
|
<ol className="worker-stage-list">
|
||||||
|
{(telemetry?.pipeline.stages ?? []).map((stage, index) => (
|
||||||
|
<li key={stage.id} data-state={stage.state}>
|
||||||
|
<span>{String(index + 1).padStart(2, "0")}</span>
|
||||||
|
<strong>{stage.label}</strong>
|
||||||
|
<small>{
|
||||||
|
stage.state === "active" ? "выполняется"
|
||||||
|
: stage.state === "waiting" ? "в очереди"
|
||||||
|
: stage.state === "ready" ? "готов"
|
||||||
|
: "нет данных"
|
||||||
|
}</small>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ol>
|
||||||
|
</GlassSurface>
|
||||||
|
|
||||||
|
<section className="system-runtime-section system-runtime-section--external">
|
||||||
|
<header className="system-section-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">CO-TENANTS / НЕ MISSION CORE</span>
|
||||||
|
<h3>Внешняя нагрузка узла</h3>
|
||||||
|
<p>
|
||||||
|
Эти процессы влияют на общую плату и VRAM, но не считаются расходом Mission Core.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div className="worker-runtime-grid">
|
||||||
|
{externalRuntimes.map((runtime) => (
|
||||||
|
<WorkerRuntimeCard key={runtime.name} runtime={runtime} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,311 @@
|
|||||||
|
import { useEffect, useMemo, useState } from "react";
|
||||||
|
import {
|
||||||
|
Button,
|
||||||
|
GlassSurface,
|
||||||
|
Icon,
|
||||||
|
StatusBadge,
|
||||||
|
TextField,
|
||||||
|
} from "@nodedc/ui-react";
|
||||||
|
|
||||||
|
import { TelemetrySeries } from "../../components/system/TelemetrySeries";
|
||||||
|
import {
|
||||||
|
formatBitRate,
|
||||||
|
formatBytes,
|
||||||
|
formatLatency,
|
||||||
|
formatRate,
|
||||||
|
} from "../../components/system/systemFormat";
|
||||||
|
import {
|
||||||
|
fetchWorkerProfile,
|
||||||
|
saveWorkerProfile,
|
||||||
|
testWorkerProfile,
|
||||||
|
type WorkerConnectionProfile,
|
||||||
|
type WorkerProbe,
|
||||||
|
} from "../../core/system/workerTelemetry";
|
||||||
|
import { useWorkerTelemetry } from "../../core/system/useWorkerTelemetry";
|
||||||
|
|
||||||
|
type ProfileAction = "idle" | "testing" | "saving";
|
||||||
|
|
||||||
|
function probeLabel(probe: WorkerProbe | null): string {
|
||||||
|
if (!probe) return "Изменения не проверены";
|
||||||
|
if (!probe.reachable) return "Адрес не отвечает";
|
||||||
|
if (!probe.identity_matches) return "Ответил другой узел";
|
||||||
|
return "Worker 006 подтверждён";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function NetworkWorkspace() {
|
||||||
|
const { telemetry, loading, error, refresh } = useWorkerTelemetry();
|
||||||
|
const [profile, setProfile] = useState<WorkerConnectionProfile | null>(null);
|
||||||
|
const [address, setAddress] = useState("");
|
||||||
|
const [port, setPort] = useState("22");
|
||||||
|
const [profileAction, setProfileAction] = useState<ProfileAction>("idle");
|
||||||
|
const [probe, setProbe] = useState<WorkerProbe | null>(null);
|
||||||
|
const [profileError, setProfileError] = useState<string | null>(null);
|
||||||
|
const aggregate = telemetry?.network.aggregate ?? null;
|
||||||
|
const connected = Boolean(
|
||||||
|
telemetry?.connection.reachable && telemetry.connection.identity_matches,
|
||||||
|
);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
void fetchWorkerProfile(controller.signal)
|
||||||
|
.then((document) => {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
setProfile(document.profile);
|
||||||
|
setAddress(document.profile.address);
|
||||||
|
setPort(String(document.profile.port));
|
||||||
|
setProfileError(null);
|
||||||
|
})
|
||||||
|
.catch((reason: unknown) => {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
setProfileError(reason instanceof Error ? reason.message : "Профиль не прочитан.");
|
||||||
|
});
|
||||||
|
return () => controller.abort();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const mutation = useMemo(() => {
|
||||||
|
const numericPort = Number(port);
|
||||||
|
if (!profile || !Number.isInteger(numericPort) || numericPort < 1 || numericPort > 65535) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
revision: profile.revision,
|
||||||
|
address: address.trim(),
|
||||||
|
port: numericPort,
|
||||||
|
};
|
||||||
|
}, [address, port, profile]);
|
||||||
|
|
||||||
|
const runTest = () => {
|
||||||
|
if (!mutation) return;
|
||||||
|
const controller = new AbortController();
|
||||||
|
setProfileAction("testing");
|
||||||
|
setProfileError(null);
|
||||||
|
setProbe(null);
|
||||||
|
void testWorkerProfile(mutation, controller.signal)
|
||||||
|
.then(setProbe)
|
||||||
|
.catch((reason: unknown) => {
|
||||||
|
setProfileError(reason instanceof Error ? reason.message : "Проверка не выполнена.");
|
||||||
|
})
|
||||||
|
.finally(() => setProfileAction("idle"));
|
||||||
|
};
|
||||||
|
|
||||||
|
const saveProfile = () => {
|
||||||
|
if (!mutation) return;
|
||||||
|
const controller = new AbortController();
|
||||||
|
setProfileAction("saving");
|
||||||
|
setProfileError(null);
|
||||||
|
void saveWorkerProfile(mutation, controller.signal)
|
||||||
|
.then((document) => {
|
||||||
|
setProfile(document.profile);
|
||||||
|
setAddress(document.profile.address);
|
||||||
|
setPort(String(document.profile.port));
|
||||||
|
setProbe(document.verification);
|
||||||
|
refresh();
|
||||||
|
})
|
||||||
|
.catch((reason: unknown) => {
|
||||||
|
setProfileError(reason instanceof Error ? reason.message : "Профиль не сохранён.");
|
||||||
|
})
|
||||||
|
.finally(() => setProfileAction("idle"));
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="system-workspace network-workspace">
|
||||||
|
<section className="system-workspace__lead">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">СИСТЕМА / СЕТЬ</span>
|
||||||
|
<h2>Локальный вычислительный контур</h2>
|
||||||
|
<p>
|
||||||
|
Переносимый профиль связи между Mission Core и Worker 006. Адрес можно заменить
|
||||||
|
при переходе в другую локальную сеть; идентичность узла проверяется до сохранения.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="system-workspace__actions">
|
||||||
|
<StatusBadge tone={connected ? "success" : "danger"}>
|
||||||
|
{connected ? "Маршрут доступен" : "Маршрут недоступен"}
|
||||||
|
</StatusBadge>
|
||||||
|
<Button
|
||||||
|
size="compact"
|
||||||
|
variant="secondary"
|
||||||
|
disabled={loading}
|
||||||
|
onClick={refresh}
|
||||||
|
icon={<Icon name="refresh" size={14} />}
|
||||||
|
>
|
||||||
|
{loading ? "Читаем" : "Обновить"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{(error || profileError) ? (
|
||||||
|
<GlassSurface className="system-workspace__notice" padding="md" tone="soft">
|
||||||
|
<StatusBadge tone="warning">{profileError ?? error}</StatusBadge>
|
||||||
|
</GlassSurface>
|
||||||
|
) : null}
|
||||||
|
|
||||||
|
<section className="network-overview-grid" aria-label="Сетевая телеметрия">
|
||||||
|
<TelemetrySeries
|
||||||
|
label="Приём"
|
||||||
|
value={formatRate(aggregate?.receive_bytes_per_second)}
|
||||||
|
values={(telemetry?.history ?? []).map(
|
||||||
|
(item) => item.network_receive_bytes_per_second,
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
<TelemetrySeries
|
||||||
|
label="Передача"
|
||||||
|
value={formatRate(aggregate?.send_bytes_per_second)}
|
||||||
|
values={(telemetry?.history ?? []).map(
|
||||||
|
(item) => item.network_send_bytes_per_second,
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
<div className="network-stat-card">
|
||||||
|
<span>Сбор телеметрии</span>
|
||||||
|
<strong>{formatLatency(telemetry?.connection.latency_ms)}</strong>
|
||||||
|
<small>полный SSH probe Worker 006</small>
|
||||||
|
</div>
|
||||||
|
<div className="network-stat-card">
|
||||||
|
<span>Активные интерфейсы</span>
|
||||||
|
<strong>{telemetry?.network.interfaces.length ?? "—"}</strong>
|
||||||
|
<small>по Windows network counters</small>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<GlassSurface className="network-profile" padding="lg">
|
||||||
|
<header className="system-section-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">ПРОФИЛЬ ПОДКЛЮЧЕНИЯ</span>
|
||||||
|
<h3>Worker 006</h3>
|
||||||
|
<p>
|
||||||
|
Реальный Node ID: <code>{profile?.expected_node_id ?? "DESKTOP-OPJ8J04"}</code>.
|
||||||
|
Пустой адрес использует закреплённый локальный SSH-профиль.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<StatusBadge tone={
|
||||||
|
probe?.reachable && probe.identity_matches ? "success"
|
||||||
|
: probe ? "danger"
|
||||||
|
: connected ? "success" : "neutral"
|
||||||
|
}>
|
||||||
|
{probe ? probeLabel(probe) : connected ? "Текущий профиль работает" : "Не проверено"}
|
||||||
|
</StatusBadge>
|
||||||
|
</header>
|
||||||
|
<div className="network-profile__form">
|
||||||
|
<TextField
|
||||||
|
label="Адрес Worker 006"
|
||||||
|
hint="IP или hostname"
|
||||||
|
value={address}
|
||||||
|
placeholder="из SSH-профиля mission-gpu"
|
||||||
|
autoComplete="off"
|
||||||
|
spellCheck={false}
|
||||||
|
onChange={(event) => {
|
||||||
|
setAddress(event.target.value);
|
||||||
|
setProbe(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<TextField
|
||||||
|
label="SSH-порт"
|
||||||
|
hint="1–65535"
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
max={65535}
|
||||||
|
value={port}
|
||||||
|
onChange={(event) => {
|
||||||
|
setPort(event.target.value);
|
||||||
|
setProbe(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<div className="network-profile__buttons">
|
||||||
|
<Button
|
||||||
|
variant="secondary"
|
||||||
|
disabled={!mutation || profileAction !== "idle"}
|
||||||
|
onClick={runTest}
|
||||||
|
>
|
||||||
|
{profileAction === "testing" ? "Проверяем" : "Проверить"}
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
disabled={!mutation || profileAction !== "idle"}
|
||||||
|
onClick={saveProfile}
|
||||||
|
>
|
||||||
|
{profileAction === "saving" ? "Сохраняем" : "Проверить и применить"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<dl className="network-profile__security">
|
||||||
|
<div><dt>Транспорт</dt><dd>SSH · key-only</dd></div>
|
||||||
|
<div><dt>Host key</dt><dd>strict · pinned</dd></div>
|
||||||
|
<div><dt>Профиль</dt><dd>{profile?.ssh_host_alias ?? "mission-gpu"}</dd></div>
|
||||||
|
<div><dt>Учётные данные</dt><dd>не доступны интерфейсу</dd></div>
|
||||||
|
</dl>
|
||||||
|
</GlassSurface>
|
||||||
|
|
||||||
|
<GlassSurface className="network-topology" padding="lg">
|
||||||
|
<header className="system-section-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">ФАКТИЧЕСКИЙ МАРШРУТ</span>
|
||||||
|
<h3>Поток управления и обработки</h3>
|
||||||
|
<p>Схема собрана из текущего профиля и live health runtime, без demo-узлов.</p>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div className="network-route">
|
||||||
|
<div data-state="online">
|
||||||
|
<span>Операторский UI</span>
|
||||||
|
<strong>Browser</strong>
|
||||||
|
<small>127.0.0.1:8000</small>
|
||||||
|
</div>
|
||||||
|
<i aria-hidden="true" data-state="online" />
|
||||||
|
<div data-state="online">
|
||||||
|
<span>Control Plane</span>
|
||||||
|
<strong>Mission Core</strong>
|
||||||
|
<small>локальный API</small>
|
||||||
|
</div>
|
||||||
|
<i aria-hidden="true" data-state={connected ? "online" : "offline"} />
|
||||||
|
<div data-state={connected ? "online" : "offline"}>
|
||||||
|
<span>Вычислительный узел</span>
|
||||||
|
<strong>Worker 006</strong>
|
||||||
|
<small>{profile?.address || profile?.ssh_host_alias || "mission-gpu"}:{profile?.port ?? 22}</small>
|
||||||
|
</div>
|
||||||
|
<i aria-hidden="true" data-state={
|
||||||
|
telemetry?.runtimes.some((runtime) => !runtime.external && runtime.state === "running")
|
||||||
|
? "online" : "offline"
|
||||||
|
} />
|
||||||
|
<div data-state={
|
||||||
|
telemetry?.runtimes.some((runtime) => !runtime.external && runtime.state === "running")
|
||||||
|
? "online" : "offline"
|
||||||
|
}>
|
||||||
|
<span>Runtime</span>
|
||||||
|
<strong>Triton + Pipeline</strong>
|
||||||
|
<small>внутренний Docker-контур</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</GlassSurface>
|
||||||
|
|
||||||
|
<section className="network-interface-section">
|
||||||
|
<header className="system-section-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">ИНТЕРФЕЙСЫ WORKER 006</span>
|
||||||
|
<h3>Адаптеры и счётчики</h3>
|
||||||
|
<p>Только активные интерфейсы, которые вернул сам узел.</p>
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div className="network-interface-list">
|
||||||
|
{(telemetry?.network.interfaces ?? []).map((adapter) => (
|
||||||
|
<GlassSurface className="network-interface-card" padding="md" key={adapter.name}>
|
||||||
|
<header>
|
||||||
|
<div>
|
||||||
|
<strong>{adapter.name}</strong>
|
||||||
|
<small>{adapter.description ?? "Описание недоступно"}</small>
|
||||||
|
</div>
|
||||||
|
<StatusBadge tone={adapter.status === "Up" ? "success" : "warning"}>
|
||||||
|
{adapter.status ?? "unknown"}
|
||||||
|
</StatusBadge>
|
||||||
|
</header>
|
||||||
|
<dl>
|
||||||
|
<div><dt>Адреса</dt><dd>{adapter.addresses.join(", ") || "—"}</dd></div>
|
||||||
|
<div><dt>Link</dt><dd>{formatBitRate(adapter.link_speed_bps)}</dd></div>
|
||||||
|
<div><dt>Получено</dt><dd>{formatBytes(adapter.received_bytes)}</dd></div>
|
||||||
|
<div><dt>Отправлено</dt><dd>{formatBytes(adapter.sent_bytes)}</dd></div>
|
||||||
|
</dl>
|
||||||
|
</GlassSurface>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFile } from "node:fs/promises";
|
||||||
|
import { test } from "node:test";
|
||||||
|
|
||||||
|
const sourceRoot = new URL("../src/", import.meta.url);
|
||||||
|
|
||||||
|
async function read(relativePath) {
|
||||||
|
return readFile(new URL(relativePath, sourceRoot), "utf8");
|
||||||
|
}
|
||||||
|
|
||||||
|
test("Worker 006 telemetry remains a bounded system feature slice", async () => {
|
||||||
|
const [
|
||||||
|
productModel,
|
||||||
|
workspaceHub,
|
||||||
|
core,
|
||||||
|
computeWorkspace,
|
||||||
|
networkWorkspace,
|
||||||
|
styles,
|
||||||
|
] = await Promise.all([
|
||||||
|
read("productModel.ts"),
|
||||||
|
read("workspaces/Workspaces.tsx"),
|
||||||
|
read("core/system/workerTelemetry.ts"),
|
||||||
|
read("workspaces/system/ComputeModulesWorkspace.tsx"),
|
||||||
|
read("workspaces/system/NetworkWorkspace.tsx"),
|
||||||
|
read("styles.css"),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.match(productModel, /kind: "compute-modules"/);
|
||||||
|
assert.match(productModel, /kind: "network-monitor"/);
|
||||||
|
assert.match(workspaceHub, /<ComputeModulesWorkspace \/>/);
|
||||||
|
assert.match(workspaceHub, /<NetworkWorkspace \/>/);
|
||||||
|
assert.doesNotMatch(workspaceHub, /worker-telemetry|worker-profile|DESKTOP-OPJ8J04/);
|
||||||
|
assert.match(core, /\/api\/v1\/system\/worker-telemetry/);
|
||||||
|
assert.match(core, /\/api\/v1\/system\/worker-profile/);
|
||||||
|
assert.doesNotMatch(core, /@nodedc\/ui-react/);
|
||||||
|
assert.match(computeWorkspace, /useWorkerTelemetry/);
|
||||||
|
assert.match(networkWorkspace, /127\.0\.0\.1:8000/);
|
||||||
|
assert.doesNotMatch(networkWorkspace, /8765/);
|
||||||
|
assert.match(styles, /system-telemetry\.css/);
|
||||||
|
});
|
||||||
@@ -1138,7 +1138,11 @@ def _load_models(args: argparse.Namespace, common: dict[str, Any]) -> _LoadedMod
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
def run(
|
||||||
|
args: argparse.Namespace,
|
||||||
|
loaded: _LoadedModels | None = None,
|
||||||
|
runtime_state: dict[str, Any] | None = None,
|
||||||
|
) -> int:
|
||||||
import av
|
import av
|
||||||
import torch
|
import torch
|
||||||
import transformers
|
import transformers
|
||||||
@@ -1149,6 +1153,18 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
token = sys.stdin.readline().strip() if args.token_stdin else args.token
|
token = sys.stdin.readline().strip() if args.token_stdin else args.token
|
||||||
if not token or len(token) < 40:
|
if not token or len(token) < 40:
|
||||||
raise RuntimeError("LAB E15 shadow token is missing")
|
raise RuntimeError("LAB E15 shadow token is missing")
|
||||||
|
|
||||||
|
def report_stage(stage: str, frame_index: int | None = None) -> None:
|
||||||
|
if runtime_state is None:
|
||||||
|
return
|
||||||
|
runtime_state["current_stage"] = stage
|
||||||
|
runtime_state["stage_observed_at_utc"] = (
|
||||||
|
datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
)
|
||||||
|
if frame_index is not None:
|
||||||
|
runtime_state["active_frame_index"] = frame_index
|
||||||
|
|
||||||
|
report_stage("preprocessing")
|
||||||
common = _common(args) if loaded is None else loaded.common
|
common = _common(args) if loaded is None else loaded.common
|
||||||
live = common["live"]
|
live = common["live"]
|
||||||
e14 = common["e14"]
|
e14 = common["e14"]
|
||||||
@@ -1215,6 +1231,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
def on_decoded(frame: DecodedCameraFrame) -> None:
|
def on_decoded(frame: DecodedCameraFrame) -> None:
|
||||||
nonlocal decoded_frame_count
|
nonlocal decoded_frame_count
|
||||||
try:
|
try:
|
||||||
|
report_stage("camera-decode", frame.frame_index)
|
||||||
if not first_camera_epoch_ns:
|
if not first_camera_epoch_ns:
|
||||||
first_camera_epoch_ns.append(frame.metadata.captured_at_epoch_ns)
|
first_camera_epoch_ns.append(frame.metadata.captured_at_epoch_ns)
|
||||||
last_camera_epoch_ns[:] = [frame.metadata.captured_at_epoch_ns]
|
last_camera_epoch_ns[:] = [frame.metadata.captured_at_epoch_ns]
|
||||||
@@ -1383,6 +1400,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
)
|
)
|
||||||
semantic_thread.start()
|
semantic_thread.start()
|
||||||
decoder.start()
|
decoder.start()
|
||||||
|
report_stage("source-ingress")
|
||||||
receiver_thread = threading.Thread(
|
receiver_thread = threading.Thread(
|
||||||
target=_receiver,
|
target=_receiver,
|
||||||
kwargs={
|
kwargs={
|
||||||
@@ -1429,10 +1447,13 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
latency["decode_age_ms"].append(float(envelope.decode_ms))
|
latency["decode_age_ms"].append(float(envelope.decode_ms))
|
||||||
latency["queue_wait_ms"].append(max(0.0, (started - envelope.decoded_monotonic) * 1000))
|
latency["queue_wait_ms"].append(max(0.0, (started - envelope.decoded_monotonic) * 1000))
|
||||||
try:
|
try:
|
||||||
|
report_stage("preprocessing", envelope.frame_index)
|
||||||
detector_started = time.perf_counter()
|
detector_started = time.perf_counter()
|
||||||
tensor = _preprocess(envelope.image, valid_mask, detector)
|
tensor = _preprocess(envelope.image, valid_mask, detector)
|
||||||
|
report_stage("detector", envelope.frame_index)
|
||||||
output_tensor, _request_ms = _infer(args.triton_url, detector["model"], tensor)
|
output_tensor, _request_ms = _infer(args.triton_url, detector["model"], tensor)
|
||||||
detections, _rejected = _detections(output_tensor, detector, valid_mask)
|
detections, _rejected = _detections(output_tensor, detector, valid_mask)
|
||||||
|
report_stage("tracking", envelope.frame_index)
|
||||||
tracks = tracker.update(detections, envelope.frame_index)
|
tracks = tracker.update(detections, envelope.frame_index)
|
||||||
latency["detector_ms"].append((time.perf_counter() - detector_started) * 1000)
|
latency["detector_ms"].append((time.perf_counter() - detector_started) * 1000)
|
||||||
|
|
||||||
@@ -1463,6 +1484,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
).reshape((-1, 3))
|
).reshape((-1, 3))
|
||||||
position = binding.pose.position_xyz
|
position = binding.pose.position_xyz
|
||||||
quaternion = binding.pose.orientation_xyzw
|
quaternion = binding.pose.orientation_xyzw
|
||||||
|
report_stage("sensor-fusion", envelope.frame_index)
|
||||||
projection_started = time.perf_counter()
|
projection_started = time.perf_counter()
|
||||||
pixels, depths, source_indices, points_lidar = project_points(
|
pixels, depths, source_indices, points_lidar = project_points(
|
||||||
points_map,
|
points_map,
|
||||||
@@ -1533,6 +1555,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
if temporal_stabilizer is None:
|
if temporal_stabilizer is None:
|
||||||
fusion_objects = raw_fusion_objects
|
fusion_objects = raw_fusion_objects
|
||||||
else:
|
else:
|
||||||
|
report_stage("temporal-state", envelope.frame_index)
|
||||||
temporal_started = time.perf_counter()
|
temporal_started = time.perf_counter()
|
||||||
fusion_objects = temporal_stabilizer.update(
|
fusion_objects = temporal_stabilizer.update(
|
||||||
frame_index=envelope.frame_index,
|
frame_index=envelope.frame_index,
|
||||||
@@ -1619,6 +1642,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
objects=fusion_objects,
|
objects=fusion_objects,
|
||||||
delivery=world["delivery"],
|
delivery=world["delivery"],
|
||||||
)
|
)
|
||||||
|
report_stage("result-publication", envelope.frame_index)
|
||||||
try:
|
try:
|
||||||
result_queue.put_nowait(live_result)
|
result_queue.put_nowait(live_result)
|
||||||
except queue.Full:
|
except queue.Full:
|
||||||
@@ -1627,6 +1651,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
|
|||||||
result_queue.task_done()
|
result_queue.task_done()
|
||||||
transport.results_dropped += 1
|
transport.results_dropped += 1
|
||||||
result_queue.put_nowait(live_result)
|
result_queue.put_nowait(live_result)
|
||||||
|
report_stage("source-ingress", envelope.frame_index)
|
||||||
except Exception:
|
except Exception:
|
||||||
detector_failures += 1
|
detector_failures += 1
|
||||||
raise
|
raise
|
||||||
@@ -2063,7 +2088,15 @@ def serve(args: argparse.Namespace) -> int:
|
|||||||
loaded = _load_models(args, common)
|
loaded = _load_models(args, common)
|
||||||
model_load_seconds = time.perf_counter() - load_started
|
model_load_seconds = time.perf_counter() - load_started
|
||||||
run_lock = threading.Lock()
|
run_lock = threading.Lock()
|
||||||
state = {"busy": False, "completed_runs": 0, "failed_runs": 0}
|
state: dict[str, Any] = {
|
||||||
|
"busy": False,
|
||||||
|
"completed_runs": 0,
|
||||||
|
"failed_runs": 0,
|
||||||
|
"active_request_id": None,
|
||||||
|
"active_frame_index": None,
|
||||||
|
"current_stage": None,
|
||||||
|
"stage_observed_at_utc": None,
|
||||||
|
}
|
||||||
|
|
||||||
class Handler(BaseHTTPRequestHandler):
|
class Handler(BaseHTTPRequestHandler):
|
||||||
server_version = "MissionCorePersistentPerception/1"
|
server_version = "MissionCorePersistentPerception/1"
|
||||||
@@ -2100,6 +2133,10 @@ def serve(args: argparse.Namespace) -> int:
|
|||||||
"model_load_seconds": model_load_seconds,
|
"model_load_seconds": model_load_seconds,
|
||||||
"completed_runs": state["completed_runs"],
|
"completed_runs": state["completed_runs"],
|
||||||
"failed_runs": state["failed_runs"],
|
"failed_runs": state["failed_runs"],
|
||||||
|
"active_request_id": state["active_request_id"],
|
||||||
|
"active_frame_index": state["active_frame_index"],
|
||||||
|
"current_stage": state["current_stage"],
|
||||||
|
"stage_observed_at_utc": state["stage_observed_at_utc"],
|
||||||
"authority": common["live"]["authority"],
|
"authority": common["live"]["authority"],
|
||||||
"gpu": torch.cuda.get_device_name(),
|
"gpu": torch.cuda.get_device_name(),
|
||||||
},
|
},
|
||||||
@@ -2129,7 +2166,8 @@ def serve(args: argparse.Namespace) -> int:
|
|||||||
request_id = str(document.get("request_id", "invalid"))
|
request_id = str(document.get("request_id", "invalid"))
|
||||||
run_args = _persistent_run_arguments(args, document)
|
run_args = _persistent_run_arguments(args, document)
|
||||||
document["token"] = None
|
document["token"] = None
|
||||||
exit_code = run(run_args, loaded)
|
state["active_request_id"] = request_id
|
||||||
|
exit_code = run(run_args, loaded, state)
|
||||||
state["completed_runs"] += 1
|
state["completed_runs"] += 1
|
||||||
self._send(
|
self._send(
|
||||||
200,
|
200,
|
||||||
@@ -2164,6 +2202,10 @@ def serve(args: argparse.Namespace) -> int:
|
|||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
state["busy"] = False
|
state["busy"] = False
|
||||||
|
state["active_request_id"] = None
|
||||||
|
state["active_frame_index"] = None
|
||||||
|
state["current_stage"] = None
|
||||||
|
state["stage_observed_at_utc"] = None
|
||||||
run_lock.release()
|
run_lock.release()
|
||||||
|
|
||||||
server = ThreadingHTTPServer((args.listen_host, args.listen_port), Handler)
|
server = ThreadingHTTPServer((args.listen_host, args.listen_port), Handler)
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ from k1link.web.plugin_runtime import (
|
|||||||
)
|
)
|
||||||
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
||||||
from k1link.web.session_api import build_session_router
|
from k1link.web.session_api import build_session_router
|
||||||
|
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
||||||
|
|
||||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||||
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
||||||
@@ -618,6 +619,11 @@ app.include_router(
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
app.include_router(
|
||||||
|
build_system_telemetry_router(
|
||||||
|
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||||
|
|||||||
@@ -0,0 +1,834 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import ipaddress
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import deque
|
||||||
|
from collections.abc import Callable
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Final
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||||
|
|
||||||
|
PROFILE_SCHEMA: Final = "missioncore.worker-connection-profile/v1"
|
||||||
|
TELEMETRY_SCHEMA: Final = "missioncore.worker-telemetry/v1"
|
||||||
|
PROBE_SCHEMA: Final = "missioncore.worker-probe/v1"
|
||||||
|
DEFAULT_PROFILE_ID: Final = "worker-006"
|
||||||
|
DEFAULT_DISPLAY_NAME: Final = "Worker 006"
|
||||||
|
EXPECTED_NODE_ID: Final = "DESKTOP-OPJ8J04"
|
||||||
|
SSH_HOST_ALIAS: Final = "mission-gpu"
|
||||||
|
PROFILE_FILE_NAME: Final = "worker-006.json"
|
||||||
|
CONTAINER_NAMES: Final = (
|
||||||
|
"mission-core-triton",
|
||||||
|
"mission-core-perception-worker",
|
||||||
|
"sentinel-frigate",
|
||||||
|
"sentinel-ollama",
|
||||||
|
)
|
||||||
|
SAFE_HOSTNAME = re.compile(
|
||||||
|
r"^(?=.{1,253}\.?$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*"
|
||||||
|
r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.?$"
|
||||||
|
)
|
||||||
|
PROMETHEUS_SAMPLE = re.compile(
|
||||||
|
r"^(?P<name>nv_inference_(?:request_success|request_failure|count))"
|
||||||
|
r"(?:\{[^}]*\})?\s+(?P<value>[0-9.eE+-]+)$"
|
||||||
|
)
|
||||||
|
|
||||||
|
RootProvider = Callable[[], Path]
|
||||||
|
ProbeRunner = Callable[["WorkerConnectionProfile"], dict[str, Any]]
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerConnectionProfile(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
schema_version: str = PROFILE_SCHEMA
|
||||||
|
profile_id: str = DEFAULT_PROFILE_ID
|
||||||
|
display_name: str = DEFAULT_DISPLAY_NAME
|
||||||
|
expected_node_id: str = EXPECTED_NODE_ID
|
||||||
|
ssh_host_alias: str = SSH_HOST_ALIAS
|
||||||
|
address: str = ""
|
||||||
|
port: int = Field(default=22, ge=1, le=65535)
|
||||||
|
revision: int = Field(default=0, ge=0)
|
||||||
|
updated_at_utc: str | None = None
|
||||||
|
|
||||||
|
@field_validator("address")
|
||||||
|
@classmethod
|
||||||
|
def validate_address(cls, value: str) -> str:
|
||||||
|
normalized = value.strip()
|
||||||
|
if not normalized:
|
||||||
|
return ""
|
||||||
|
if any(character.isspace() or ord(character) < 32 for character in normalized):
|
||||||
|
raise ValueError("worker address contains whitespace or control characters")
|
||||||
|
try:
|
||||||
|
ipaddress.ip_address(normalized)
|
||||||
|
except ValueError:
|
||||||
|
if SAFE_HOSTNAME.fullmatch(normalized) is None:
|
||||||
|
raise ValueError("worker address is not a valid IP address or hostname") from None
|
||||||
|
return normalized.rstrip(".")
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerConnectionProfilePut(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
revision: int = Field(ge=0)
|
||||||
|
address: str = Field(default="", max_length=253)
|
||||||
|
port: int = Field(default=22, ge=1, le=65535)
|
||||||
|
|
||||||
|
@field_validator("address")
|
||||||
|
@classmethod
|
||||||
|
def validate_address(cls, value: str) -> str:
|
||||||
|
return WorkerConnectionProfile.validate_address(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now() -> str:
|
||||||
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerProfileStore:
|
||||||
|
def __init__(self, root: Path) -> None:
|
||||||
|
self.root = root
|
||||||
|
self.path = root / PROFILE_FILE_NAME
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
def read(self) -> WorkerConnectionProfile:
|
||||||
|
with self._lock:
|
||||||
|
if not self.path.is_file():
|
||||||
|
return WorkerConnectionProfile()
|
||||||
|
try:
|
||||||
|
document = json.loads(self.path.read_text(encoding="utf-8"))
|
||||||
|
return WorkerConnectionProfile.model_validate(document)
|
||||||
|
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
||||||
|
return WorkerConnectionProfile()
|
||||||
|
|
||||||
|
def save(self, request: WorkerConnectionProfilePut) -> WorkerConnectionProfile:
|
||||||
|
with self._lock:
|
||||||
|
current = self._read_unlocked()
|
||||||
|
if current.revision != request.revision:
|
||||||
|
raise RuntimeError("worker profile revision changed")
|
||||||
|
profile = current.model_copy(
|
||||||
|
update={
|
||||||
|
"address": request.address,
|
||||||
|
"port": request.port,
|
||||||
|
"revision": current.revision + 1,
|
||||||
|
"updated_at_utc": _utc_now(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
descriptor, temporary_name = tempfile.mkstemp(
|
||||||
|
prefix=".worker-006.",
|
||||||
|
suffix=".tmp",
|
||||||
|
dir=self.root,
|
||||||
|
)
|
||||||
|
temporary = Path(temporary_name)
|
||||||
|
try:
|
||||||
|
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
|
||||||
|
json.dump(
|
||||||
|
profile.model_dump(mode="json"),
|
||||||
|
stream,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
stream.write("\n")
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
os.chmod(temporary, 0o600)
|
||||||
|
os.replace(temporary, self.path)
|
||||||
|
finally:
|
||||||
|
temporary.unlink(missing_ok=True)
|
||||||
|
return profile
|
||||||
|
|
||||||
|
def _read_unlocked(self) -> WorkerConnectionProfile:
|
||||||
|
if not self.path.is_file():
|
||||||
|
return WorkerConnectionProfile()
|
||||||
|
try:
|
||||||
|
return WorkerConnectionProfile.model_validate_json(
|
||||||
|
self.path.read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return WorkerConnectionProfile()
|
||||||
|
|
||||||
|
|
||||||
|
WORKER_PROBE_POWERSHELL: Final = r"""
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$containers = @(
|
||||||
|
"mission-core-triton",
|
||||||
|
"mission-core-perception-worker",
|
||||||
|
"sentinel-frigate",
|
||||||
|
"sentinel-ollama"
|
||||||
|
)
|
||||||
|
$os = Get-CimInstance Win32_OperatingSystem
|
||||||
|
$cpu = Get-CimInstance Win32_Processor
|
||||||
|
$computer = Get-CimInstance Win32_ComputerSystem
|
||||||
|
$dockerStats = @{}
|
||||||
|
try {
|
||||||
|
$statsLines = @(docker stats --no-stream --format "{{json .}}" $containers)
|
||||||
|
foreach ($line in $statsLines) {
|
||||||
|
if ($line) {
|
||||||
|
$row = $line | ConvertFrom-Json
|
||||||
|
$dockerStats[$row.Name] = $row
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
$containerStates = @{}
|
||||||
|
foreach ($name in $containers) {
|
||||||
|
try {
|
||||||
|
$state = (docker inspect --format "{{json .State}}" $name) | ConvertFrom-Json
|
||||||
|
$image = docker inspect --format "{{.Config.Image}}" $name
|
||||||
|
$containerStates[$name] = [ordered]@{state=$state; image=$image}
|
||||||
|
} catch {
|
||||||
|
$containerStates[$name] = $null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$gpu = $null
|
||||||
|
try {
|
||||||
|
$gpuQuery = "name,utilization.gpu,memory.used,memory.total,power.draw,temperature.gpu"
|
||||||
|
$gpuLine = nvidia-smi "--query-gpu=$gpuQuery" --format=csv,noheader,nounits |
|
||||||
|
Select-Object -First 1
|
||||||
|
$gpuParts = @($gpuLine -split ",\s*")
|
||||||
|
if ($gpuParts.Count -ge 6) {
|
||||||
|
$gpu = [ordered]@{
|
||||||
|
name=$gpuParts[0]
|
||||||
|
utilization_percent=[double]$gpuParts[1]
|
||||||
|
memory_used_mib=[double]$gpuParts[2]
|
||||||
|
memory_total_mib=[double]$gpuParts[3]
|
||||||
|
power_watts=[double]$gpuParts[4]
|
||||||
|
temperature_celsius=[double]$gpuParts[5]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
$network = @()
|
||||||
|
foreach ($adapter in @(Get-NetAdapter | Where-Object Status -eq "Up")) {
|
||||||
|
try {
|
||||||
|
$statistics = Get-NetAdapterStatistics -Name $adapter.Name
|
||||||
|
$addresses = @(
|
||||||
|
Get-NetIPAddress -InterfaceIndex $adapter.ifIndex -AddressFamily IPv4 `
|
||||||
|
-ErrorAction SilentlyContinue |
|
||||||
|
Where-Object { $_.IPAddress -notlike "169.254.*" } |
|
||||||
|
Select-Object -ExpandProperty IPAddress
|
||||||
|
)
|
||||||
|
$network += [ordered]@{
|
||||||
|
name=$adapter.Name
|
||||||
|
description=$adapter.InterfaceDescription
|
||||||
|
status=$adapter.Status
|
||||||
|
link_speed_bps=[double]$adapter.Speed
|
||||||
|
mac_address=$adapter.MacAddress
|
||||||
|
addresses=$addresses
|
||||||
|
received_bytes=[double]$statistics.ReceivedBytes
|
||||||
|
sent_bytes=[double]$statistics.SentBytes
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
$tritonReady = $false
|
||||||
|
try {
|
||||||
|
$tritonResponse = Invoke-WebRequest -UseBasicParsing `
|
||||||
|
-Uri "http://127.0.0.1:8000/v2/health/ready" -TimeoutSec 2
|
||||||
|
$tritonReady = $tritonResponse.StatusCode -eq 200
|
||||||
|
} catch {}
|
||||||
|
$tritonMetrics = @()
|
||||||
|
try {
|
||||||
|
$metricsResponse = Invoke-WebRequest -UseBasicParsing `
|
||||||
|
-Uri "http://127.0.0.1:8002/metrics" -TimeoutSec 2
|
||||||
|
$tritonMetrics = @(
|
||||||
|
$metricsResponse.Content -split "`n" |
|
||||||
|
Where-Object { $_ -match "^nv_inference_(request_success|request_failure|count)" }
|
||||||
|
)
|
||||||
|
} catch {}
|
||||||
|
$perceptionHealth = $null
|
||||||
|
try {
|
||||||
|
$healthJson = docker exec mission-core-perception-worker python3 -c "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"
|
||||||
|
$perceptionHealth = $healthJson | ConvertFrom-Json
|
||||||
|
} catch {}
|
||||||
|
$disks = @(
|
||||||
|
Get-CimInstance Win32_LogicalDisk -Filter "DriveType=3" |
|
||||||
|
ForEach-Object {
|
||||||
|
[ordered]@{
|
||||||
|
name=$_.DeviceID
|
||||||
|
size_bytes=[double]$_.Size
|
||||||
|
free_bytes=[double]$_.FreeSpace
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
[ordered]@{
|
||||||
|
schema_version="missioncore.worker-probe-raw/v1"
|
||||||
|
observed_at_utc=[DateTime]::UtcNow.ToString("o")
|
||||||
|
node_id=$env:COMPUTERNAME
|
||||||
|
os=[ordered]@{
|
||||||
|
caption=$os.Caption
|
||||||
|
version=$os.Version
|
||||||
|
uptime_seconds=([DateTime]::UtcNow - $os.LastBootUpTime.ToUniversalTime()).TotalSeconds
|
||||||
|
}
|
||||||
|
cpu=[ordered]@{
|
||||||
|
name=(@($cpu | Select-Object -ExpandProperty Name) -join " + ")
|
||||||
|
logical_processors=[int]$computer.NumberOfLogicalProcessors
|
||||||
|
load_percent=[double](($cpu | Measure-Object LoadPercentage -Average).Average)
|
||||||
|
}
|
||||||
|
memory=[ordered]@{
|
||||||
|
total_bytes=[double]$os.TotalVisibleMemorySize * 1024
|
||||||
|
free_bytes=[double]$os.FreePhysicalMemory * 1024
|
||||||
|
}
|
||||||
|
disks=$disks
|
||||||
|
gpu=$gpu
|
||||||
|
network=$network
|
||||||
|
docker_stats=$dockerStats
|
||||||
|
container_states=$containerStates
|
||||||
|
triton=[ordered]@{ready=$tritonReady; metrics=$tritonMetrics}
|
||||||
|
perception=$perceptionHealth
|
||||||
|
} | ConvertTo-Json -Depth 12 -Compress
|
||||||
|
""".strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _ssh_arguments(profile: WorkerConnectionProfile) -> list[str]:
|
||||||
|
arguments = [
|
||||||
|
"ssh",
|
||||||
|
"-o",
|
||||||
|
"BatchMode=yes",
|
||||||
|
"-o",
|
||||||
|
"StrictHostKeyChecking=yes",
|
||||||
|
"-o",
|
||||||
|
f"HostKeyAlias={profile.ssh_host_alias}",
|
||||||
|
"-o",
|
||||||
|
"ConnectTimeout=5",
|
||||||
|
]
|
||||||
|
if profile.address:
|
||||||
|
arguments.extend(["-o", f"HostName={profile.address}"])
|
||||||
|
arguments.extend(
|
||||||
|
[
|
||||||
|
"-p",
|
||||||
|
str(profile.port),
|
||||||
|
profile.ssh_host_alias,
|
||||||
|
"powershell.exe",
|
||||||
|
"-NoLogo",
|
||||||
|
"-NoProfile",
|
||||||
|
"-NonInteractive",
|
||||||
|
"-Command",
|
||||||
|
(
|
||||||
|
"$encoded=[Console]::In.ReadToEnd();"
|
||||||
|
"Invoke-Expression "
|
||||||
|
"([Text.Encoding]::Unicode.GetString([Convert]::FromBase64String($encoded)))"
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return arguments
|
||||||
|
|
||||||
|
|
||||||
|
def run_worker_probe(profile: WorkerConnectionProfile) -> dict[str, Any]:
|
||||||
|
started = time.perf_counter()
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
_ssh_arguments(profile),
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
input=base64.b64encode(WORKER_PROBE_POWERSHELL.encode("utf-16le")),
|
||||||
|
timeout=12,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
return _failed_probe(profile, "worker-unreachable", started)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
return _failed_probe(profile, "worker-unreachable", started)
|
||||||
|
try:
|
||||||
|
raw = completed.stdout.decode("cp866")
|
||||||
|
document = json.loads(raw.strip())
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError, TypeError):
|
||||||
|
return _failed_probe(profile, "invalid-worker-response", started)
|
||||||
|
if not isinstance(document, dict):
|
||||||
|
return _failed_probe(profile, "invalid-worker-response", started)
|
||||||
|
node_id = document.get("node_id")
|
||||||
|
if node_id != profile.expected_node_id:
|
||||||
|
return {
|
||||||
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"reachable": True,
|
||||||
|
"identity_matches": False,
|
||||||
|
"node_id": node_id if isinstance(node_id, str) else None,
|
||||||
|
"latency_ms": (time.perf_counter() - started) * 1000,
|
||||||
|
"observed_at_utc": _utc_now(),
|
||||||
|
"error_code": "worker-identity-mismatch",
|
||||||
|
"raw": None,
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"reachable": True,
|
||||||
|
"identity_matches": True,
|
||||||
|
"node_id": node_id,
|
||||||
|
"latency_ms": (time.perf_counter() - started) * 1000,
|
||||||
|
"observed_at_utc": _utc_now(),
|
||||||
|
"error_code": None,
|
||||||
|
"raw": document,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _failed_probe(
|
||||||
|
profile: WorkerConnectionProfile,
|
||||||
|
error_code: str,
|
||||||
|
started: float,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"reachable": False,
|
||||||
|
"identity_matches": False,
|
||||||
|
"node_id": None,
|
||||||
|
"expected_node_id": profile.expected_node_id,
|
||||||
|
"latency_ms": (time.perf_counter() - started) * 1000,
|
||||||
|
"observed_at_utc": _utc_now(),
|
||||||
|
"error_code": error_code,
|
||||||
|
"raw": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _number(value: object) -> float | None:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||||
|
return None
|
||||||
|
result = float(value)
|
||||||
|
return result if math.isfinite(result) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _mapping(value: object) -> dict[str, Any]:
|
||||||
|
return value if isinstance(value, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _items(value: object) -> list[Any]:
|
||||||
|
return value if isinstance(value, list) else []
|
||||||
|
|
||||||
|
|
||||||
|
def _percent(used: float | None, total: float | None) -> float | None:
|
||||||
|
if used is None or total is None or total <= 0:
|
||||||
|
return None
|
||||||
|
return max(0.0, min(100.0, used / total * 100))
|
||||||
|
|
||||||
|
|
||||||
|
def _container_document(
|
||||||
|
name: str,
|
||||||
|
raw_stats: dict[str, Any],
|
||||||
|
raw_states: dict[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
stats = _mapping(raw_stats.get(name))
|
||||||
|
descriptor = _mapping(raw_states.get(name))
|
||||||
|
state = _mapping(descriptor.get("state"))
|
||||||
|
health = _mapping(state.get("Health"))
|
||||||
|
roles = {
|
||||||
|
"mission-core-triton": ("Inference Runtime", "mission-core", False),
|
||||||
|
"mission-core-perception-worker": ("Perception Pipeline", "mission-core", False),
|
||||||
|
"sentinel-frigate": ("Sentinel Frigate", "external", True),
|
||||||
|
"sentinel-ollama": ("Sentinel Ollama", "external", True),
|
||||||
|
}
|
||||||
|
role, owner, external = roles[name]
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"role": role,
|
||||||
|
"owner": owner,
|
||||||
|
"external": external,
|
||||||
|
"image": descriptor.get("image") if isinstance(descriptor.get("image"), str) else None,
|
||||||
|
"state": state.get("Status") if isinstance(state.get("Status"), str) else "unavailable",
|
||||||
|
"health": health.get("Status") if isinstance(health.get("Status"), str) else None,
|
||||||
|
"cpu_percent": _parse_percent(stats.get("CPUPerc")),
|
||||||
|
"memory_percent": _parse_percent(stats.get("MemPerc")),
|
||||||
|
"memory_usage": stats.get("MemUsage") if isinstance(stats.get("MemUsage"), str) else None,
|
||||||
|
"network_io": stats.get("NetIO") if isinstance(stats.get("NetIO"), str) else None,
|
||||||
|
"block_io": stats.get("BlockIO") if isinstance(stats.get("BlockIO"), str) else None,
|
||||||
|
"pids": _parse_integer(stats.get("PIDs")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_percent(value: object) -> float | None:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
number = float(value.rstrip("%"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return number if math.isfinite(number) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_integer(value: object) -> int | None:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _triton_metrics(lines: object) -> dict[str, float | None]:
|
||||||
|
totals: dict[str, float] = {
|
||||||
|
"nv_inference_request_success": 0,
|
||||||
|
"nv_inference_request_failure": 0,
|
||||||
|
"nv_inference_count": 0,
|
||||||
|
}
|
||||||
|
matched: set[str] = set()
|
||||||
|
for line in _items(lines):
|
||||||
|
if not isinstance(line, str):
|
||||||
|
continue
|
||||||
|
match = PROMETHEUS_SAMPLE.fullmatch(line.strip())
|
||||||
|
if match is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
value = float(match.group("value"))
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if math.isfinite(value):
|
||||||
|
name = match.group("name")
|
||||||
|
totals[name] += value
|
||||||
|
matched.add(name)
|
||||||
|
return {
|
||||||
|
"requests_succeeded": (
|
||||||
|
totals["nv_inference_request_success"]
|
||||||
|
if "nv_inference_request_success" in matched
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"requests_failed": (
|
||||||
|
totals["nv_inference_request_failure"]
|
||||||
|
if "nv_inference_request_failure" in matched
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"inferences": (
|
||||||
|
totals["nv_inference_count"] if "nv_inference_count" in matched else None
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
perception = _mapping(raw.get("perception"))
|
||||||
|
current_stage = perception.get("current_stage")
|
||||||
|
if not isinstance(current_stage, str):
|
||||||
|
current_stage = None
|
||||||
|
busy = perception.get("state") == "busy"
|
||||||
|
stages = (
|
||||||
|
("source-ingress", "Приём сенсорного потока"),
|
||||||
|
("camera-decode", "Декодирование камеры"),
|
||||||
|
("preprocessing", "Предобработка"),
|
||||||
|
("detector", "Детектор объектов"),
|
||||||
|
("semantic-model", "Семантическая модель"),
|
||||||
|
("sensor-fusion", "Camera ↔ LiDAR fusion"),
|
||||||
|
("tracking", "Трекинг"),
|
||||||
|
("temporal-state", "Временное состояние"),
|
||||||
|
("result-publication", "Публикация результата"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def stage_state(stage_id: str) -> str:
|
||||||
|
if not perception:
|
||||||
|
return "unavailable"
|
||||||
|
if not busy:
|
||||||
|
return "ready"
|
||||||
|
if stage_id == current_stage or stage_id in {
|
||||||
|
"source-ingress",
|
||||||
|
"camera-decode",
|
||||||
|
"semantic-model",
|
||||||
|
}:
|
||||||
|
return "active"
|
||||||
|
return "waiting"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"service_state": (
|
||||||
|
perception.get("state") if isinstance(perception.get("state"), str) else "unavailable"
|
||||||
|
),
|
||||||
|
"current_stage": current_stage,
|
||||||
|
"active_request_id": (
|
||||||
|
perception.get("active_request_id")
|
||||||
|
if isinstance(perception.get("active_request_id"), str)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"active_frame_index": (
|
||||||
|
int(perception["active_frame_index"])
|
||||||
|
if isinstance(perception.get("active_frame_index"), int)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"completed_runs": (
|
||||||
|
int(perception["completed_runs"])
|
||||||
|
if isinstance(perception.get("completed_runs"), int)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"failed_runs": (
|
||||||
|
int(perception["failed_runs"])
|
||||||
|
if isinstance(perception.get("failed_runs"), int)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"model_load_seconds": _number(perception.get("model_load_seconds")),
|
||||||
|
"stages": [
|
||||||
|
{
|
||||||
|
"id": stage_id,
|
||||||
|
"label": label,
|
||||||
|
"state": stage_state(stage_id),
|
||||||
|
}
|
||||||
|
for stage_id, label in stages
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerTelemetryService:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
store: WorkerProfileStore,
|
||||||
|
probe_runner: ProbeRunner = run_worker_probe,
|
||||||
|
*,
|
||||||
|
cache_seconds: float = 1.5,
|
||||||
|
) -> None:
|
||||||
|
self.store = store
|
||||||
|
self.probe_runner = probe_runner
|
||||||
|
self.cache_seconds = cache_seconds
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._cached_at = 0.0
|
||||||
|
self._cached: dict[str, Any] | None = None
|
||||||
|
self._previous_network: tuple[float, float, float] | None = None
|
||||||
|
self._history: deque[dict[str, Any]] = deque(maxlen=300)
|
||||||
|
|
||||||
|
def profile_document(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema_version": PROFILE_SCHEMA,
|
||||||
|
"profile": self.store.read().model_dump(mode="json"),
|
||||||
|
"security": {
|
||||||
|
"transport": "ssh",
|
||||||
|
"host_key_policy": "strict-pinned",
|
||||||
|
"credentials_managed_by_ui": False,
|
||||||
|
"ssh_host_alias": SSH_HOST_ALIAS,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_profile(self, request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||||
|
current = self.store.read()
|
||||||
|
candidate = current.model_copy(
|
||||||
|
update={"address": request.address, "port": request.port}
|
||||||
|
)
|
||||||
|
return self._public_probe(self.probe_runner(candidate))
|
||||||
|
|
||||||
|
def apply_profile(self, request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||||
|
current = self.store.read()
|
||||||
|
if request.revision != current.revision:
|
||||||
|
raise RuntimeError("worker profile revision changed")
|
||||||
|
candidate = current.model_copy(
|
||||||
|
update={"address": request.address, "port": request.port}
|
||||||
|
)
|
||||||
|
probe = self.probe_runner(candidate)
|
||||||
|
if not probe.get("reachable") or not probe.get("identity_matches"):
|
||||||
|
raise ConnectionError(str(probe.get("error_code") or "worker-unreachable"))
|
||||||
|
saved = self.store.save(request)
|
||||||
|
with self._lock:
|
||||||
|
self._cached = None
|
||||||
|
self._cached_at = 0
|
||||||
|
self._previous_network = None
|
||||||
|
self._history.clear()
|
||||||
|
return {
|
||||||
|
**self.profile_document(),
|
||||||
|
"verification": self._public_probe(probe),
|
||||||
|
"profile": saved.model_dump(mode="json"),
|
||||||
|
}
|
||||||
|
|
||||||
|
def snapshot(self, history_limit: int) -> dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
if self._cached is not None and now - self._cached_at < self.cache_seconds:
|
||||||
|
return {
|
||||||
|
**self._cached,
|
||||||
|
"history": list(self._history)[-history_limit:],
|
||||||
|
}
|
||||||
|
profile = self.store.read()
|
||||||
|
probe = self.probe_runner(profile)
|
||||||
|
document = self._telemetry_document(profile, probe, now)
|
||||||
|
self._cached = document
|
||||||
|
self._cached_at = now
|
||||||
|
return {
|
||||||
|
**document,
|
||||||
|
"history": list(self._history)[-history_limit:],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _telemetry_document(
|
||||||
|
self,
|
||||||
|
profile: WorkerConnectionProfile,
|
||||||
|
probe: dict[str, Any],
|
||||||
|
monotonic_now: float,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
public_probe = self._public_probe(probe)
|
||||||
|
if not probe.get("reachable") or not probe.get("identity_matches"):
|
||||||
|
return {
|
||||||
|
"schema_version": TELEMETRY_SCHEMA,
|
||||||
|
"profile": profile.model_dump(mode="json"),
|
||||||
|
"connection": public_probe,
|
||||||
|
"node": None,
|
||||||
|
"runtimes": [],
|
||||||
|
"pipeline": _pipeline_document({}),
|
||||||
|
"network": {"interfaces": [], "aggregate": None},
|
||||||
|
}
|
||||||
|
raw = _mapping(probe.get("raw"))
|
||||||
|
memory = _mapping(raw.get("memory"))
|
||||||
|
memory_total = _number(memory.get("total_bytes"))
|
||||||
|
memory_free = _number(memory.get("free_bytes"))
|
||||||
|
memory_used = (
|
||||||
|
memory_total - memory_free
|
||||||
|
if memory_total is not None and memory_free is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
gpu = _mapping(raw.get("gpu"))
|
||||||
|
gpu_used = _number(gpu.get("memory_used_mib"))
|
||||||
|
gpu_total = _number(gpu.get("memory_total_mib"))
|
||||||
|
interfaces = [
|
||||||
|
self._network_interface(item)
|
||||||
|
for item in _items(raw.get("network"))
|
||||||
|
if isinstance(item, dict)
|
||||||
|
]
|
||||||
|
received = sum(
|
||||||
|
value
|
||||||
|
for item in interfaces
|
||||||
|
if (value := _number(item.get("received_bytes"))) is not None
|
||||||
|
)
|
||||||
|
sent = sum(
|
||||||
|
value
|
||||||
|
for item in interfaces
|
||||||
|
if (value := _number(item.get("sent_bytes"))) is not None
|
||||||
|
)
|
||||||
|
receive_rate: float | None = None
|
||||||
|
send_rate: float | None = None
|
||||||
|
if self._previous_network is not None:
|
||||||
|
previous_at, previous_received, previous_sent = self._previous_network
|
||||||
|
elapsed = monotonic_now - previous_at
|
||||||
|
if elapsed > 0 and received >= previous_received and sent >= previous_sent:
|
||||||
|
receive_rate = (received - previous_received) / elapsed
|
||||||
|
send_rate = (sent - previous_sent) / elapsed
|
||||||
|
self._previous_network = (monotonic_now, received, sent)
|
||||||
|
raw_stats = _mapping(raw.get("docker_stats"))
|
||||||
|
raw_states = _mapping(raw.get("container_states"))
|
||||||
|
runtimes = [
|
||||||
|
_container_document(name, raw_stats, raw_states) for name in CONTAINER_NAMES
|
||||||
|
]
|
||||||
|
triton = _mapping(raw.get("triton"))
|
||||||
|
triton_document = {
|
||||||
|
"ready": triton.get("ready") is True,
|
||||||
|
**_triton_metrics(triton.get("metrics")),
|
||||||
|
}
|
||||||
|
memory_used_percent = _percent(memory_used, memory_total)
|
||||||
|
gpu_memory_used_percent = _percent(gpu_used, gpu_total)
|
||||||
|
node: dict[str, Any] = {
|
||||||
|
"node_id": raw.get("node_id"),
|
||||||
|
"observed_at_utc": raw.get("observed_at_utc"),
|
||||||
|
"os": _mapping(raw.get("os")),
|
||||||
|
"cpu": _mapping(raw.get("cpu")),
|
||||||
|
"memory": {
|
||||||
|
"total_bytes": memory_total,
|
||||||
|
"used_bytes": memory_used,
|
||||||
|
"free_bytes": memory_free,
|
||||||
|
"used_percent": memory_used_percent,
|
||||||
|
},
|
||||||
|
"disks": _items(raw.get("disks")),
|
||||||
|
"gpu": {
|
||||||
|
**gpu,
|
||||||
|
"memory_used_percent": gpu_memory_used_percent,
|
||||||
|
}
|
||||||
|
if gpu
|
||||||
|
else None,
|
||||||
|
"triton": triton_document,
|
||||||
|
}
|
||||||
|
network = {
|
||||||
|
"interfaces": interfaces,
|
||||||
|
"aggregate": {
|
||||||
|
"received_bytes": received,
|
||||||
|
"sent_bytes": sent,
|
||||||
|
"receive_bytes_per_second": receive_rate,
|
||||||
|
"send_bytes_per_second": send_rate,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
history_row = {
|
||||||
|
"observed_at_utc": raw.get("observed_at_utc") or _utc_now(),
|
||||||
|
"cpu_percent": _number(_mapping(raw.get("cpu")).get("load_percent")),
|
||||||
|
"memory_percent": memory_used_percent,
|
||||||
|
"gpu_percent": _number(gpu.get("utilization_percent")),
|
||||||
|
"gpu_memory_percent": gpu_memory_used_percent,
|
||||||
|
"network_receive_bytes_per_second": receive_rate,
|
||||||
|
"network_send_bytes_per_second": send_rate,
|
||||||
|
}
|
||||||
|
self._history.append(history_row)
|
||||||
|
return {
|
||||||
|
"schema_version": TELEMETRY_SCHEMA,
|
||||||
|
"profile": profile.model_dump(mode="json"),
|
||||||
|
"connection": public_probe,
|
||||||
|
"node": node,
|
||||||
|
"runtimes": runtimes,
|
||||||
|
"pipeline": _pipeline_document(raw),
|
||||||
|
"network": network,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _network_interface(item: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
addresses = [
|
||||||
|
address for address in _items(item.get("addresses")) if isinstance(address, str)
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"name": item.get("name") if isinstance(item.get("name"), str) else "unknown",
|
||||||
|
"description": (
|
||||||
|
item.get("description") if isinstance(item.get("description"), str) else None
|
||||||
|
),
|
||||||
|
"status": item.get("status") if isinstance(item.get("status"), str) else None,
|
||||||
|
"link_speed_bps": _number(item.get("link_speed_bps")),
|
||||||
|
"mac_address": (
|
||||||
|
item.get("mac_address") if isinstance(item.get("mac_address"), str) else None
|
||||||
|
),
|
||||||
|
"addresses": addresses,
|
||||||
|
"received_bytes": _number(item.get("received_bytes")),
|
||||||
|
"sent_bytes": _number(item.get("sent_bytes")),
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _public_probe(probe: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema_version": PROBE_SCHEMA,
|
||||||
|
"reachable": probe.get("reachable") is True,
|
||||||
|
"identity_matches": probe.get("identity_matches") is True,
|
||||||
|
"node_id": probe.get("node_id") if isinstance(probe.get("node_id"), str) else None,
|
||||||
|
"latency_ms": _number(probe.get("latency_ms")),
|
||||||
|
"observed_at_utc": (
|
||||||
|
probe.get("observed_at_utc")
|
||||||
|
if isinstance(probe.get("observed_at_utc"), str)
|
||||||
|
else _utc_now()
|
||||||
|
),
|
||||||
|
"error_code": (
|
||||||
|
probe.get("error_code")
|
||||||
|
if isinstance(probe.get("error_code"), str)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_system_telemetry_router(
|
||||||
|
*,
|
||||||
|
root_provider: RootProvider,
|
||||||
|
probe_runner: ProbeRunner = run_worker_probe,
|
||||||
|
) -> APIRouter:
|
||||||
|
store = WorkerProfileStore(root_provider())
|
||||||
|
service = WorkerTelemetryService(store, probe_runner)
|
||||||
|
router = APIRouter(prefix="/api/v1/system", tags=["system"])
|
||||||
|
|
||||||
|
@router.get("/worker-profile")
|
||||||
|
def get_worker_profile() -> dict[str, Any]:
|
||||||
|
return service.profile_document()
|
||||||
|
|
||||||
|
@router.post("/worker-profile/test")
|
||||||
|
def test_worker_profile(request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||||
|
return service.test_profile(request)
|
||||||
|
|
||||||
|
@router.put("/worker-profile")
|
||||||
|
def put_worker_profile(request: WorkerConnectionProfilePut) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return service.apply_profile(request)
|
||||||
|
except RuntimeError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Профиль Worker 006 был изменён в другой сессии.",
|
||||||
|
) from exc
|
||||||
|
except ConnectionError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="Новый адрес не прошёл проверку узла Worker 006.",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
@router.get("/worker-telemetry")
|
||||||
|
def get_worker_telemetry(
|
||||||
|
history: int = Query(default=90, ge=1, le=300),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return service.snapshot(history)
|
||||||
|
|
||||||
|
return router
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import stat
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from fastapi.routing import APIRoute
|
||||||
|
from pydantic import ValidationError
|
||||||
|
|
||||||
|
from k1link.web.system_telemetry_api import (
|
||||||
|
EXPECTED_NODE_ID,
|
||||||
|
WorkerConnectionProfile,
|
||||||
|
WorkerConnectionProfilePut,
|
||||||
|
WorkerProfileStore,
|
||||||
|
WorkerTelemetryService,
|
||||||
|
_ssh_arguments,
|
||||||
|
build_system_telemetry_router,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
||||||
|
for route in router.routes:
|
||||||
|
if (
|
||||||
|
isinstance(route, APIRoute)
|
||||||
|
and route.path == path
|
||||||
|
and route.methods is not None
|
||||||
|
and method in route.methods
|
||||||
|
):
|
||||||
|
return route.endpoint
|
||||||
|
raise AssertionError(f"{method} {path} route is missing")
|
||||||
|
|
||||||
|
|
||||||
|
def _probe(
|
||||||
|
*,
|
||||||
|
node_id: str = EXPECTED_NODE_ID,
|
||||||
|
received_bytes: float = 1_000,
|
||||||
|
sent_bytes: float = 500,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"reachable": True,
|
||||||
|
"identity_matches": node_id == EXPECTED_NODE_ID,
|
||||||
|
"node_id": node_id,
|
||||||
|
"latency_ms": 12.5,
|
||||||
|
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||||
|
"error_code": None,
|
||||||
|
"raw": {
|
||||||
|
"node_id": node_id,
|
||||||
|
"observed_at_utc": "2026-07-27T12:00:00Z",
|
||||||
|
"os": {
|
||||||
|
"caption": "Windows 11 Pro",
|
||||||
|
"version": "10.0",
|
||||||
|
"uptime_seconds": 100,
|
||||||
|
},
|
||||||
|
"cpu": {
|
||||||
|
"name": "CPU",
|
||||||
|
"logical_processors": 32,
|
||||||
|
"load_percent": 25,
|
||||||
|
},
|
||||||
|
"memory": {
|
||||||
|
"total_bytes": 1000,
|
||||||
|
"free_bytes": 400,
|
||||||
|
},
|
||||||
|
"disks": [],
|
||||||
|
"gpu": {
|
||||||
|
"name": "GPU",
|
||||||
|
"utilization_percent": 30,
|
||||||
|
"memory_used_mib": 100,
|
||||||
|
"memory_total_mib": 200,
|
||||||
|
},
|
||||||
|
"network": [
|
||||||
|
{
|
||||||
|
"name": "LAN",
|
||||||
|
"description": "Ethernet",
|
||||||
|
"status": "Up",
|
||||||
|
"link_speed_bps": 1_000_000_000,
|
||||||
|
"addresses": ["192.0.2.10"],
|
||||||
|
"received_bytes": received_bytes,
|
||||||
|
"sent_bytes": sent_bytes,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"docker_stats": {
|
||||||
|
"mission-core-triton": {
|
||||||
|
"Name": "mission-core-triton",
|
||||||
|
"CPUPerc": "2.5%",
|
||||||
|
"MemPerc": "3.5%",
|
||||||
|
"MemUsage": "1GiB / 32GiB",
|
||||||
|
"NetIO": "1MB / 2MB",
|
||||||
|
"BlockIO": "0B / 0B",
|
||||||
|
"PIDs": "12",
|
||||||
|
},
|
||||||
|
"sentinel-frigate": {
|
||||||
|
"Name": "sentinel-frigate",
|
||||||
|
"CPUPerc": "150%",
|
||||||
|
"MemPerc": "20%",
|
||||||
|
"MemUsage": "6GiB / 32GiB",
|
||||||
|
"NetIO": "3GB / 1GB",
|
||||||
|
"BlockIO": "1GB / 1GB",
|
||||||
|
"PIDs": "200",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"container_states": {
|
||||||
|
"mission-core-triton": {
|
||||||
|
"state": {"Status": "running", "Health": {"Status": "healthy"}},
|
||||||
|
"image": "triton@sha256:accepted",
|
||||||
|
},
|
||||||
|
"sentinel-frigate": {
|
||||||
|
"state": {"Status": "running", "Health": {"Status": "healthy"}},
|
||||||
|
"image": "frigate@sha256:external",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"triton": {
|
||||||
|
"ready": True,
|
||||||
|
"metrics": [
|
||||||
|
'nv_inference_request_success{model="detector"} 12',
|
||||||
|
'nv_inference_request_failure{model="detector"} 0',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"perception": {
|
||||||
|
"state": "busy",
|
||||||
|
"current_stage": "detector",
|
||||||
|
"active_request_id": "run-001",
|
||||||
|
"active_frame_index": 42,
|
||||||
|
"completed_runs": 3,
|
||||||
|
"failed_runs": 0,
|
||||||
|
"model_load_seconds": 10.5,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_profile_rejects_ssh_option_injection() -> None:
|
||||||
|
for unsafe in (
|
||||||
|
"-oProxyCommand=touch /tmp/unsafe",
|
||||||
|
"worker.local -p 2200",
|
||||||
|
"worker.local/../../unsafe",
|
||||||
|
"worker.local\nHost evil",
|
||||||
|
):
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
WorkerConnectionProfile(address=unsafe)
|
||||||
|
|
||||||
|
assert WorkerConnectionProfile(address="192.0.2.15").address == "192.0.2.15"
|
||||||
|
assert WorkerConnectionProfile(address="worker-006.local").address == "worker-006.local"
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_profile_is_atomic_versioned_and_private(tmp_path: Path) -> None:
|
||||||
|
store = WorkerProfileStore(tmp_path / "system")
|
||||||
|
initial = store.read()
|
||||||
|
assert initial.revision == 0
|
||||||
|
saved = store.save(
|
||||||
|
WorkerConnectionProfilePut(
|
||||||
|
revision=0,
|
||||||
|
address="192.0.2.15",
|
||||||
|
port=2200,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert saved.revision == 1
|
||||||
|
assert store.read() == saved
|
||||||
|
assert stat.S_IMODE(store.path.stat().st_mode) == 0o600
|
||||||
|
with pytest.raises(RuntimeError, match="revision changed"):
|
||||||
|
store.save(
|
||||||
|
WorkerConnectionProfilePut(
|
||||||
|
revision=0,
|
||||||
|
address="192.0.2.16",
|
||||||
|
port=22,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ssh_command_keeps_identity_pinned_and_values_as_arguments() -> None:
|
||||||
|
arguments = _ssh_arguments(
|
||||||
|
WorkerConnectionProfile(address="192.0.2.15", port=2200)
|
||||||
|
)
|
||||||
|
assert "BatchMode=yes" in arguments
|
||||||
|
assert "StrictHostKeyChecking=yes" in arguments
|
||||||
|
assert "HostKeyAlias=mission-gpu" in arguments
|
||||||
|
assert "HostName=192.0.2.15" in arguments
|
||||||
|
assert arguments[arguments.index("-p") + 1] == "2200"
|
||||||
|
assert "mission-gpu" in arguments
|
||||||
|
assert "192.0.2.15; touch unsafe" not in arguments
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_telemetry_separates_mission_core_and_external_load(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
service = WorkerTelemetryService(
|
||||||
|
WorkerProfileStore(tmp_path / "system"),
|
||||||
|
lambda _: _probe(),
|
||||||
|
cache_seconds=0,
|
||||||
|
)
|
||||||
|
document = service.snapshot(10)
|
||||||
|
runtimes = {runtime["name"]: runtime for runtime in document["runtimes"]}
|
||||||
|
|
||||||
|
assert document["connection"]["identity_matches"] is True
|
||||||
|
assert document["node"]["memory"]["used_percent"] == 60
|
||||||
|
assert document["node"]["gpu"]["memory_used_percent"] == 50
|
||||||
|
assert document["node"]["triton"]["requests_succeeded"] == 12
|
||||||
|
assert runtimes["mission-core-triton"]["external"] is False
|
||||||
|
assert runtimes["mission-core-triton"]["cpu_percent"] == 2.5
|
||||||
|
assert runtimes["sentinel-frigate"]["external"] is True
|
||||||
|
assert runtimes["sentinel-frigate"]["cpu_percent"] == 150
|
||||||
|
assert document["pipeline"]["active_request_id"] == "run-001"
|
||||||
|
assert next(
|
||||||
|
stage
|
||||||
|
for stage in document["pipeline"]["stages"]
|
||||||
|
if stage["id"] == "detector"
|
||||||
|
)["state"] == "active"
|
||||||
|
assert next(
|
||||||
|
stage
|
||||||
|
for stage in document["pipeline"]["stages"]
|
||||||
|
if stage["id"] == "semantic-model"
|
||||||
|
)["state"] == "active"
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
router = build_system_telemetry_router(
|
||||||
|
root_provider=lambda: tmp_path / "system",
|
||||||
|
probe_runner=lambda _: _probe(node_id="OTHER-NODE"),
|
||||||
|
)
|
||||||
|
apply_profile = _endpoint(router, "/api/v1/system/worker-profile", "PUT")
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException) as error:
|
||||||
|
apply_profile(
|
||||||
|
WorkerConnectionProfilePut(
|
||||||
|
revision=0,
|
||||||
|
address="192.0.2.20",
|
||||||
|
port=22,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert error.value.status_code == 409
|
||||||
|
assert WorkerProfileStore(tmp_path / "system").read().revision == 0
|
||||||
|
assert not WorkerProfileStore(tmp_path / "system").path.exists()
|
||||||
Reference in New Issue
Block a user