fix(telemetry): restore worker agent delivery and truthful system status
This commit is contained in:
@@ -38,6 +38,8 @@ export function useWorkerTelemetry(
|
|||||||
setError(null);
|
setError(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
setTelemetry(null);
|
||||||
|
setError(null);
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const stopPolling = startSequentialPolling(async () => {
|
const stopPolling = startSequentialPolling(async () => {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
@@ -48,6 +50,7 @@ export function useWorkerTelemetry(
|
|||||||
setError(null);
|
setError(null);
|
||||||
} catch (reason: unknown) {
|
} catch (reason: unknown) {
|
||||||
if (controller.signal.aborted) return;
|
if (controller.signal.aborted) return;
|
||||||
|
setTelemetry(null);
|
||||||
setError(
|
setError(
|
||||||
reason instanceof Error
|
reason instanceof Error
|
||||||
? reason.message
|
? reason.message
|
||||||
|
|||||||
@@ -114,6 +114,9 @@ export interface WorkerTelemetry {
|
|||||||
power_watts?: number;
|
power_watts?: number;
|
||||||
temperature_celsius?: number;
|
temperature_celsius?: number;
|
||||||
memory_used_percent?: number;
|
memory_used_percent?: number;
|
||||||
|
sm_clock_mhz?: number | null;
|
||||||
|
memory_clock_mhz?: number | null;
|
||||||
|
driver_version?: string | null;
|
||||||
} | null;
|
} | null;
|
||||||
triton: {
|
triton: {
|
||||||
ready: boolean;
|
ready: boolean;
|
||||||
@@ -131,6 +134,13 @@ export interface WorkerTelemetry {
|
|||||||
completed_runs: number | null;
|
completed_runs: number | null;
|
||||||
failed_runs: number | null;
|
failed_runs: number | null;
|
||||||
model_load_seconds: number | null;
|
model_load_seconds: number | null;
|
||||||
|
profile_name?: string | null;
|
||||||
|
input_state?: string | null;
|
||||||
|
wait_reason?: string | null;
|
||||||
|
live_children?: number | null;
|
||||||
|
input_pauses?: number | null;
|
||||||
|
pending_bundles?: number | null;
|
||||||
|
buffer_bytes?: number | null;
|
||||||
stages: WorkerPipelineStage[];
|
stages: WorkerPipelineStage[];
|
||||||
};
|
};
|
||||||
network: {
|
network: {
|
||||||
@@ -189,7 +199,9 @@ export async function fetchWorkerTelemetry(
|
|||||||
`/api/v1/system/contours/${encodeURIComponent(contourId)}/telemetry?history=90`,
|
`/api/v1/system/contours/${encodeURIComponent(contourId)}/telemetry?history=90`,
|
||||||
{
|
{
|
||||||
method: "GET",
|
method: "GET",
|
||||||
signal,
|
signal: signal
|
||||||
|
? AbortSignal.any([signal, AbortSignal.timeout(5_000)])
|
||||||
|
: AbortSignal.timeout(5_000),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
"missioncore.worker-telemetry/v1",
|
"missioncore.worker-telemetry/v1",
|
||||||
|
|||||||
@@ -17,9 +17,14 @@ import {
|
|||||||
} from "../../core/system/useWorkerTelemetry";
|
} from "../../core/system/useWorkerTelemetry";
|
||||||
|
|
||||||
function pipelineStateLabel(state: string): string {
|
function pipelineStateLabel(state: string): string {
|
||||||
if (state === "busy") return "Выполняет задачу";
|
if (state === "busy" || state === "running") return "Выполняет задачу";
|
||||||
|
if (state === "waiting") return "Ожидает восстановления данных";
|
||||||
|
if (state === "synchronizing") return "Синхронизирует поток";
|
||||||
|
if (state === "starting") return "Загружает профиль";
|
||||||
|
if (state === "stopped" || state === "cancelled") return "Профиль остановлен";
|
||||||
|
if (state === "failed") return "Ошибка профиля";
|
||||||
if (state === "ready") return "Готов к задаче";
|
if (state === "ready") return "Готов к задаче";
|
||||||
return "Нет live-состояния";
|
return "Нет свежих данных о задаче";
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatResourcePair(
|
function formatResourcePair(
|
||||||
@@ -116,7 +121,7 @@ export function ComputeModulesWorkspace() {
|
|||||||
{!legacyDiagnostic && !agentTelemetry ? (
|
{!legacyDiagnostic && !agentTelemetry ? (
|
||||||
<GlassSurface className="system-workspace__notice" padding="md" tone="soft">
|
<GlassSurface className="system-workspace__notice" padding="md" tone="soft">
|
||||||
<StatusBadge tone="warning">
|
<StatusBadge tone="warning">
|
||||||
Агентный data-plane ещё не опубликовал нормализованный срез этого контура.
|
Ожидаем свежую телеметрию выбранного узла.
|
||||||
</StatusBadge>
|
</StatusBadge>
|
||||||
</GlassSurface>
|
</GlassSurface>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -187,6 +192,8 @@ export function ComputeModulesWorkspace() {
|
|||||||
? `${node.gpu.temperature_celsius} °C`
|
? `${node.gpu.temperature_celsius} °C`
|
||||||
: "—"
|
: "—"
|
||||||
}</dd></div>
|
}</dd></div>
|
||||||
|
<div><dt>Частота GPU</dt><dd>{node?.gpu?.sm_clock_mhz == null ? "—" : `${node.gpu.sm_clock_mhz} МГц`}</dd></div>
|
||||||
|
<div><dt>Частота памяти GPU</dt><dd>{node?.gpu?.memory_clock_mhz == null ? "—" : `${node.gpu.memory_clock_mhz} МГц`}</dd></div>
|
||||||
<div><dt>Мощность</dt><dd>{
|
<div><dt>Мощность</dt><dd>{
|
||||||
typeof node?.gpu?.power_watts === "number"
|
typeof node?.gpu?.power_watts === "number"
|
||||||
? `${node.gpu.power_watts.toFixed(1)} Вт`
|
? `${node.gpu.power_watts.toFixed(1)} Вт`
|
||||||
@@ -214,7 +221,7 @@ export function ComputeModulesWorkspace() {
|
|||||||
<div>
|
<div>
|
||||||
<span className="section-eyebrow">PROCESSING RUNTIME</span>
|
<span className="section-eyebrow">PROCESSING RUNTIME</span>
|
||||||
<h3>Контейнеры Mission Core</h3>
|
<h3>Контейнеры Mission Core</h3>
|
||||||
<p>Два ограниченных runtime: inference server и прикладной perception pipeline.</p>
|
<p>Состояние контейнера и готовность вычислительного профиля проверяются отдельно.</p>
|
||||||
</div>
|
</div>
|
||||||
<StatusBadge tone={node?.triton.ready ? "success" : "danger"}>
|
<StatusBadge tone={node?.triton.ready ? "success" : "danger"}>
|
||||||
{node?.triton.ready ? "Triton ready" : "Triton недоступен"}
|
{node?.triton.ready ? "Triton ready" : "Triton недоступен"}
|
||||||
@@ -233,19 +240,31 @@ export function ComputeModulesWorkspace() {
|
|||||||
<span className="section-eyebrow">ТЕКУЩАЯ ЗАДАЧА</span>
|
<span className="section-eyebrow">ТЕКУЩАЯ ЗАДАЧА</span>
|
||||||
<h3>{pipelineStateLabel(telemetry?.pipeline.service_state ?? "unavailable")}</h3>
|
<h3>{pipelineStateLabel(telemetry?.pipeline.service_state ?? "unavailable")}</h3>
|
||||||
<p>
|
<p>
|
||||||
|
{telemetry?.pipeline.profile_name ?? "Perception"}
|
||||||
|
{" · "}
|
||||||
{telemetry?.pipeline.active_request_id
|
{telemetry?.pipeline.active_request_id
|
||||||
? `Run ${telemetry.pipeline.active_request_id}`
|
? `Run ${telemetry.pipeline.active_request_id}`
|
||||||
: "Очередь свободна; модели остаются загруженными в persistent worker."}
|
: telemetry?.pipeline.service_state === "ready"
|
||||||
|
? "Готов к приёму данных."
|
||||||
|
: "Готовность моделей не подтверждена."}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<StatusBadge tone={
|
<StatusBadge tone={
|
||||||
telemetry?.pipeline.service_state === "busy" ? "warning"
|
["busy", "running", "waiting", "synchronizing", "starting"].includes(telemetry?.pipeline.service_state ?? "") ? "warning"
|
||||||
: telemetry?.pipeline.service_state === "ready" ? "success"
|
: telemetry?.pipeline.service_state === "ready" ? "success"
|
||||||
: "danger"
|
: "danger"
|
||||||
}>
|
}>
|
||||||
{telemetry?.pipeline.service_state ?? "unavailable"}
|
{pipelineStateLabel(telemetry?.pipeline.service_state ?? "unavailable")}
|
||||||
</StatusBadge>
|
</StatusBadge>
|
||||||
</header>
|
</header>
|
||||||
|
{telemetry?.pipeline.profile_name ? (
|
||||||
|
<div className="worker-pipeline__summary">
|
||||||
|
<div><span>Процессы профиля</span><strong>{telemetry.pipeline.live_children ?? "—"}</strong></div>
|
||||||
|
<div><span>Ожиданий потока</span><strong>{telemetry.pipeline.input_pauses ?? "—"}</strong></div>
|
||||||
|
<div><span>Пакетов в очереди</span><strong>{telemetry.pipeline.pending_bundles ?? "—"}</strong></div>
|
||||||
|
<div><span>Входные буферы</span><strong>{formatBytes(telemetry.pipeline.buffer_bytes)}</strong></div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
<div className="worker-pipeline__summary">
|
<div className="worker-pipeline__summary">
|
||||||
<div><span>Завершено запусков</span><strong>{telemetry?.pipeline.completed_runs ?? "—"}</strong></div>
|
<div><span>Завершено запусков</span><strong>{telemetry?.pipeline.completed_runs ?? "—"}</strong></div>
|
||||||
<div><span>Ошибок запусков</span><strong>{telemetry?.pipeline.failed_runs ?? "—"}</strong></div>
|
<div><span>Ошибок запусков</span><strong>{telemetry?.pipeline.failed_runs ?? "—"}</strong></div>
|
||||||
|
|||||||
@@ -124,6 +124,11 @@ test("Compute-contour telemetry remains a bounded system feature slice", async (
|
|||||||
assert.match(pollIntervalContract, /MIN_TELEMETRY_POLL_INTERVAL_SECONDS\s*=\s*1/);
|
assert.match(pollIntervalContract, /MIN_TELEMETRY_POLL_INTERVAL_SECONDS\s*=\s*1/);
|
||||||
assert.match(telemetryPolling, /normalizeWorkerTelemetryPollMilliseconds/);
|
assert.match(telemetryPolling, /normalizeWorkerTelemetryPollMilliseconds/);
|
||||||
assert.match(telemetryPolling, /startSequentialPolling/);
|
assert.match(telemetryPolling, /startSequentialPolling/);
|
||||||
|
assert.match(telemetryPolling, /catch \(reason: unknown\)[\s\S]*?setTelemetry\(null\)/);
|
||||||
|
assert.match(computeWorkspace, /sm_clock_mhz/);
|
||||||
|
assert.match(computeWorkspace, /memory_clock_mhz/);
|
||||||
|
assert.match(computeWorkspace, /Ожидает восстановления данных/);
|
||||||
|
assert.doesNotMatch(computeWorkspace, /Очередь свободна; модели остаются загруженными/);
|
||||||
assert.doesNotMatch(telemetryPolling, /if\s*\(\s*!enabled\s*\|\|\s*loading\s*\)/);
|
assert.doesNotMatch(telemetryPolling, /if\s*\(\s*!enabled\s*\|\|\s*loading\s*\)/);
|
||||||
assert.match(pollingScheduler, /A transient failure must not stop polling/);
|
assert.match(pollingScheduler, /A transient failure must not stop polling/);
|
||||||
assert.match(styles, /system-telemetry\.css/);
|
assert.match(styles, /system-telemetry\.css/);
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# Local, read-only source override for repairing the existing normalizer without
|
||||||
|
# rebuilding an image on a memory-constrained operator machine. No extra service.
|
||||||
|
services:
|
||||||
|
normalizer:
|
||||||
|
volumes:
|
||||||
|
- type: bind
|
||||||
|
source: ${MISSIONCORE_NORMALIZER_SOURCE:?Set the absolute normalizer.py path}
|
||||||
|
target: /app/normalizer.py
|
||||||
|
read_only: true
|
||||||
@@ -49,6 +49,8 @@ services:
|
|||||||
|
|
||||||
timescale:
|
timescale:
|
||||||
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
|
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
|
||||||
|
# The HA image auto-tunes for the Docker VM, not this 1 GiB container.
|
||||||
|
command: ["postgres", "-c", "shared_buffers=128MB", "-c", "work_mem=4MB", "-c", "maintenance_work_mem=64MB", "-c", "effective_cache_size=512MB", "-c", "max_parallel_workers_per_gather=0", "-c", "max_connections=32"]
|
||||||
container_name: ndc-mission-core-telemetry-timescaledb
|
container_name: ndc-mission-core-telemetry-timescaledb
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
security_opt:
|
security_opt:
|
||||||
|
|||||||
@@ -31,7 +31,8 @@ SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
|
|||||||
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
|
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
|
||||||
MAX_TAGS: Final = 256
|
MAX_TAGS: Final = 256
|
||||||
MAX_SERIES_KEY_BYTES: Final = 4096
|
MAX_SERIES_KEY_BYTES: Final = 4096
|
||||||
RETENTION_INTERVAL_SECONDS: Final = 86_400
|
RETENTION_INTERVAL_SECONDS: Final = 60
|
||||||
|
RETENTION_BATCH_ROWS: Final = 1000
|
||||||
ALLOWED_TELEMETRY_TAGS: Final = frozenset(
|
ALLOWED_TELEMETRY_TAGS: Final = frozenset(
|
||||||
{
|
{
|
||||||
"agent_id",
|
"agent_id",
|
||||||
@@ -434,6 +435,39 @@ def _start_query_server(
|
|||||||
return server
|
return server
|
||||||
|
|
||||||
|
|
||||||
|
def _retention_batch(connection: Any) -> int:
|
||||||
|
"""Preserve the existing 30-day policy without an unbounded startup DELETE."""
|
||||||
|
with connection.transaction(), connection.cursor() as cursor:
|
||||||
|
cursor.execute("SET LOCAL statement_timeout = '2s'")
|
||||||
|
cursor.execute("SET LOCAL lock_timeout = '250ms'")
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
WITH expired AS (
|
||||||
|
SELECT tableoid, ctid FROM contour_telemetry_samples
|
||||||
|
WHERE observed_at < CURRENT_TIMESTAMP - INTERVAL '30 days'
|
||||||
|
ORDER BY observed_at LIMIT %s
|
||||||
|
)
|
||||||
|
DELETE FROM contour_telemetry_samples AS samples USING expired
|
||||||
|
WHERE samples.tableoid = expired.tableoid AND samples.ctid = expired.ctid
|
||||||
|
""",
|
||||||
|
(RETENTION_BATCH_ROWS,),
|
||||||
|
)
|
||||||
|
return cursor.rowcount
|
||||||
|
|
||||||
|
|
||||||
|
def _retention_loop(dsn: str) -> None:
|
||||||
|
import psycopg # type: ignore[import-not-found]
|
||||||
|
|
||||||
|
while True:
|
||||||
|
# Neither a retention timeout nor database recovery blocks MQTT I/O.
|
||||||
|
time.sleep(RETENTION_INTERVAL_SECONDS)
|
||||||
|
try:
|
||||||
|
with psycopg.connect(dsn, connect_timeout=3) as connection:
|
||||||
|
_retention_batch(connection)
|
||||||
|
except psycopg.Error:
|
||||||
|
print("telemetry retention deferred", flush=True)
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
import paho.mqtt.client as mqtt
|
import paho.mqtt.client as mqtt
|
||||||
import psycopg # type: ignore[import-not-found]
|
import psycopg # type: ignore[import-not-found]
|
||||||
@@ -447,7 +481,9 @@ def main() -> None:
|
|||||||
password = _required("MISSIONCORE_MQTT_PASSWORD")
|
password = _required("MISSIONCORE_MQTT_PASSWORD")
|
||||||
runtime_health = TelemetryRuntimeHealth()
|
runtime_health = TelemetryRuntimeHealth()
|
||||||
connection = psycopg.connect(dsn, autocommit=True)
|
connection = psycopg.connect(dsn, autocommit=True)
|
||||||
last_retention_monotonic = 0.0
|
threading.Thread(
|
||||||
|
target=_retention_loop, args=(dsn,), name="telemetry-retention", daemon=True
|
||||||
|
).start()
|
||||||
runtime_health.set_database(True)
|
runtime_health.set_database(True)
|
||||||
_start_query_server(dsn, runtime_health)
|
_start_query_server(dsn, runtime_health)
|
||||||
client = mqtt.Client(
|
client = mqtt.Client(
|
||||||
@@ -480,7 +516,7 @@ def main() -> None:
|
|||||||
runtime_health.set_mqtt(False, f"MQTT disconnected: {reason_code}")
|
runtime_health.set_mqtt(False, f"MQTT disconnected: {reason_code}")
|
||||||
|
|
||||||
def on_message(_client: Any, _userdata: object, message: Any) -> None:
|
def on_message(_client: Any, _userdata: object, message: Any) -> None:
|
||||||
nonlocal connection, last_retention_monotonic
|
nonlocal connection
|
||||||
try:
|
try:
|
||||||
row = _normalize(message.topic, message.payload)
|
row = _normalize(message.topic, message.payload)
|
||||||
except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
@@ -489,19 +525,6 @@ def main() -> None:
|
|||||||
for attempt in range(2):
|
for attempt in range(2):
|
||||||
try:
|
try:
|
||||||
with connection.cursor() as cursor:
|
with connection.cursor() as cursor:
|
||||||
now_monotonic = time.monotonic()
|
|
||||||
if (
|
|
||||||
now_monotonic - last_retention_monotonic
|
|
||||||
>= RETENTION_INTERVAL_SECONDS
|
|
||||||
):
|
|
||||||
cursor.execute(
|
|
||||||
"""
|
|
||||||
DELETE FROM contour_telemetry_samples
|
|
||||||
WHERE observed_at
|
|
||||||
< CURRENT_TIMESTAMP - INTERVAL '30 days'
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
last_retention_monotonic = now_monotonic
|
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO contour_telemetry_samples (
|
INSERT INTO contour_telemetry_samples (
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
[string]$ContainerName = "ndc-mission-core-perception-worker",
|
[string]$ContainerName = "ndc-mission-core-perception-worker",
|
||||||
[string]$HealthUrl = "http://127.0.0.1:18020/health"
|
[string]$HealthUrl = "http://127.0.0.1:18020/health",
|
||||||
|
[string]$RuntimeSnapshot = "C:\ProgramData\NDC\MissionCore\telemetry-agent\perception\current.json"
|
||||||
)
|
)
|
||||||
|
|
||||||
$ErrorActionPreference = "Stop"
|
$ErrorActionPreference = "Stop"
|
||||||
@@ -18,6 +19,46 @@ $stageIds = @(
|
|||||||
)
|
)
|
||||||
$health = $null
|
$health = $null
|
||||||
$collectorState = "unavailable"
|
$collectorState = "unavailable"
|
||||||
|
# A fresh full-profile export supersedes the historical service. Presence of an
|
||||||
|
# expired/broken export is NOT permission to relabel a legacy model as current.
|
||||||
|
if (Test-Path -LiteralPath $RuntimeSnapshot -PathType Leaf) {
|
||||||
|
$runtime = $null
|
||||||
|
try {
|
||||||
|
$item = Get-Item -LiteralPath $RuntimeSnapshot
|
||||||
|
$age = ([DateTime]::UtcNow - $item.LastWriteTimeUtc).TotalSeconds
|
||||||
|
if ($item.Length -le 8192 -and $age -ge -1 -and $age -le 5) {
|
||||||
|
$stream = [IO.File]::Open($RuntimeSnapshot, 'Open', 'Read', 'ReadWrite')
|
||||||
|
try {
|
||||||
|
$bytes = New-Object byte[] 8193
|
||||||
|
$count = $stream.Read($bytes, 0, $bytes.Length)
|
||||||
|
if ($count -le 8192) {
|
||||||
|
$runtime = [Text.Encoding]::UTF8.GetString($bytes, 0, $count) | ConvertFrom-Json
|
||||||
|
}
|
||||||
|
} finally { $stream.Dispose() }
|
||||||
|
if ($runtime.schema_version -ne 'missioncore.perception-runtime-observation/v1') {
|
||||||
|
$runtime = $null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch { $runtime = $null }
|
||||||
|
$samples = foreach ($stageId in $stageIds) {
|
||||||
|
$sample = [ordered]@{
|
||||||
|
stage_id = $stageId
|
||||||
|
stage_state = 'unavailable'
|
||||||
|
service_state = 'unavailable'
|
||||||
|
collector_state = 'unavailable'
|
||||||
|
}
|
||||||
|
if ($runtime) {
|
||||||
|
$sample.collector_state = 'live'
|
||||||
|
foreach ($name in @('service_state', 'profile_name', 'active_request_id',
|
||||||
|
'input_state', 'wait_reason', 'live_children', 'input_pauses', 'pending_bundles', 'buffer_bytes')) {
|
||||||
|
$sample[$name] = $runtime.$name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
[pscustomobject]$sample
|
||||||
|
}
|
||||||
|
@($samples) | ConvertTo-Json -Compress -Depth 4
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
$healthJson = docker exec $ContainerName python3 -c `
|
$healthJson = docker exec $ContainerName python3 -c `
|
||||||
"import urllib.request;print(urllib.request.urlopen('$HealthUrl',timeout=2).read().decode())" `
|
"import urllib.request;print(urllib.request.urlopen('$HealthUrl',timeout=2).read().decode())" `
|
||||||
@@ -68,13 +109,13 @@ $completedRuns = if ($health -and $null -ne $health.completed_runs) {
|
|||||||
[int64]$health.completed_runs
|
[int64]$health.completed_runs
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
[int64]0
|
[int64]-1
|
||||||
}
|
}
|
||||||
$failedRuns = if ($health -and $null -ne $health.failed_runs) {
|
$failedRuns = if ($health -and $null -ne $health.failed_runs) {
|
||||||
[int64]$health.failed_runs
|
[int64]$health.failed_runs
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
[int64]0
|
[int64]-1
|
||||||
}
|
}
|
||||||
$modelLoadSeconds = if ($health -and $null -ne $health.model_load_seconds) {
|
$modelLoadSeconds = if ($health -and $null -ne $health.model_load_seconds) {
|
||||||
[double]$health.model_load_seconds
|
[double]$health.model_load_seconds
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
[agent]
|
[agent]
|
||||||
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||||
|
flush_interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||||
|
metric_batch_size = 200
|
||||||
|
metric_buffer_limit = 2000
|
||||||
round_interval = true
|
round_interval = true
|
||||||
omit_hostname = false
|
omit_hostname = false
|
||||||
|
|
||||||
@@ -26,6 +29,7 @@
|
|||||||
total = true
|
total = true
|
||||||
|
|
||||||
[[outputs.mqtt]]
|
[[outputs.mqtt]]
|
||||||
|
startup_error_behavior = "retry"
|
||||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
||||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
[agent]
|
[agent]
|
||||||
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||||
|
flush_interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||||
|
metric_batch_size = 200
|
||||||
|
metric_buffer_limit = 2000
|
||||||
round_interval = true
|
round_interval = true
|
||||||
omit_hostname = false
|
omit_hostname = false
|
||||||
|
|
||||||
@@ -30,6 +33,7 @@
|
|||||||
[[inputs.docker]]
|
[[inputs.docker]]
|
||||||
endpoint = "npipe:////./pipe/docker_engine"
|
endpoint = "npipe:////./pipe/docker_engine"
|
||||||
container_name_include = []
|
container_name_include = []
|
||||||
|
container_state_include = ["running", "exited", "restarting", "paused"]
|
||||||
|
|
||||||
[[inputs.http_response]]
|
[[inputs.http_response]]
|
||||||
urls = ["http://127.0.0.1:8000/v2/health/ready"]
|
urls = ["http://127.0.0.1:8000/v2/health/ready"]
|
||||||
@@ -59,6 +63,9 @@
|
|||||||
"current_stage",
|
"current_stage",
|
||||||
"active_request_id",
|
"active_request_id",
|
||||||
"collector_state",
|
"collector_state",
|
||||||
|
"profile_name",
|
||||||
|
"input_state",
|
||||||
|
"wait_reason",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[inputs.tail]]
|
[[inputs.tail]]
|
||||||
@@ -73,6 +80,7 @@
|
|||||||
data_type = "string"
|
data_type = "string"
|
||||||
|
|
||||||
[[outputs.mqtt]]
|
[[outputs.mqtt]]
|
||||||
|
startup_error_behavior = "retry"
|
||||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
||||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||||
@@ -83,6 +91,7 @@
|
|||||||
namedrop = ["missioncore_pipeline", "missioncore_pipeline_event"]
|
namedrop = ["missioncore_pipeline", "missioncore_pipeline_event"]
|
||||||
|
|
||||||
[[outputs.mqtt]]
|
[[outputs.mqtt]]
|
||||||
|
startup_error_behavior = "retry"
|
||||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/pipeline"
|
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/pipeline"
|
||||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||||
|
|||||||
@@ -486,7 +486,7 @@ def _run_compose_broker(telemetry_plane_root: Path) -> None:
|
|||||||
]
|
]
|
||||||
for suffix, timeout in (
|
for suffix, timeout in (
|
||||||
(["config", "--quiet"], 20),
|
(["config", "--quiet"], 20),
|
||||||
(["up", "-d", "--no-build", "broker"], 45),
|
(["up", "-d", "--no-deps", "--no-build", "--force-recreate", "broker"], 45),
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
completed = subprocess.run(
|
completed = subprocess.run(
|
||||||
@@ -525,11 +525,17 @@ def apply_broker_network(
|
|||||||
target.mqtt_bind_address,
|
target.mqtt_bind_address,
|
||||||
)
|
)
|
||||||
changed = after != before
|
changed = after != before
|
||||||
|
repaired = False
|
||||||
mode = environment_path.stat().st_mode
|
mode = environment_path.stat().st_mode
|
||||||
try:
|
try:
|
||||||
if changed:
|
if changed:
|
||||||
_atomic_private_write(environment_path, after, mode=mode)
|
_atomic_private_write(environment_path, after, mode=mode)
|
||||||
_run_compose_broker(telemetry_plane_root)
|
_run_compose_broker(telemetry_plane_root)
|
||||||
|
elif not _tcp_reachable(target.mqtt_bind_address, target.mqtt_port, timeout=3):
|
||||||
|
# An unchanged file does not prove Docker still publishes the port
|
||||||
|
# after a host/network restart. This is an explicit Apply, not a GET.
|
||||||
|
_run_compose_broker(telemetry_plane_root)
|
||||||
|
repaired = True
|
||||||
if not _tcp_reachable(
|
if not _tcp_reachable(
|
||||||
target.mqtt_bind_address,
|
target.mqtt_bind_address,
|
||||||
target.mqtt_port,
|
target.mqtt_port,
|
||||||
@@ -553,6 +559,7 @@ def apply_broker_network(
|
|||||||
"contour_id": target.contour_id,
|
"contour_id": target.contour_id,
|
||||||
"target": "broker",
|
"target": "broker",
|
||||||
"changed": changed,
|
"changed": changed,
|
||||||
|
"repaired": repaired,
|
||||||
"applied_at_utc": _utc_now(),
|
"applied_at_utc": _utc_now(),
|
||||||
"ready": True,
|
"ready": True,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -478,7 +478,7 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
|||||||
perception: dict[str, Any] = raw["perception"]
|
perception: dict[str, Any] = raw["perception"]
|
||||||
pipeline_stage_metrics: dict[str, dict[str, Any]] = {}
|
pipeline_stage_metrics: dict[str, dict[str, Any]] = {}
|
||||||
pipeline_active_stages: set[str] = set()
|
pipeline_active_stages: set[str] = set()
|
||||||
for sample in samples:
|
for sample in sorted(samples, key=lambda item: str(item.get("observed_at_utc", ""))):
|
||||||
measurement = sample.get("measurement")
|
measurement = sample.get("measurement")
|
||||||
payload, fields, tags = _metric_payload(sample)
|
payload, fields, tags = _metric_payload(sample)
|
||||||
if sample.get("kind") == "pipeline":
|
if sample.get("kind") == "pipeline":
|
||||||
@@ -497,6 +497,9 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
|||||||
"service_state",
|
"service_state",
|
||||||
"current_stage",
|
"current_stage",
|
||||||
"active_request_id",
|
"active_request_id",
|
||||||
|
"profile_name",
|
||||||
|
"input_state",
|
||||||
|
"wait_reason",
|
||||||
):
|
):
|
||||||
value = fields.get(name)
|
value = fields.get(name)
|
||||||
if isinstance(value, str):
|
if isinstance(value, str):
|
||||||
@@ -511,6 +514,8 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
|||||||
if model_load_seconds is not None:
|
if model_load_seconds is not None:
|
||||||
perception["model_load_seconds"] = model_load_seconds
|
perception["model_load_seconds"] = model_load_seconds
|
||||||
perception["collector_state"] = fields.get("collector_state")
|
perception["collector_state"] = fields.get("collector_state")
|
||||||
|
for name in ("live_children", "input_pauses", "pending_bundles", "buffer_bytes"):
|
||||||
|
perception[name] = _number(fields.get(name))
|
||||||
elif sample.get("source_schema") == PIPELINE_TELEMETRY_SCHEMA:
|
elif sample.get("source_schema") == PIPELINE_TELEMETRY_SCHEMA:
|
||||||
# Native lifecycle rows are immutable run evidence. The current
|
# Native lifecycle rows are immutable run evidence. The current
|
||||||
# service snapshot remains owned by the periodic health sample,
|
# service snapshot remains owned by the periodic health sample,
|
||||||
@@ -593,6 +598,9 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
|||||||
"memory_total_mib": memory_total,
|
"memory_total_mib": memory_total,
|
||||||
"power_watts": _number(fields.get("power_draw")),
|
"power_watts": _number(fields.get("power_draw")),
|
||||||
"temperature_celsius": _number(fields.get("temperature_gpu")),
|
"temperature_celsius": _number(fields.get("temperature_gpu")),
|
||||||
|
"sm_clock_mhz": _number(fields.get("clocks_current_sm")),
|
||||||
|
"memory_clock_mhz": _number(fields.get("clocks_current_memory")),
|
||||||
|
"driver_version": fields.get("driver_version"),
|
||||||
}
|
}
|
||||||
elif measurement == "missioncore_triton_health":
|
elif measurement == "missioncore_triton_health":
|
||||||
raw["triton"] = {
|
raw["triton"] = {
|
||||||
@@ -637,7 +645,15 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
|||||||
"image": tags.get("container_image"),
|
"image": tags.get("container_image"),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
if measurement == "docker_container_cpu":
|
if measurement == "docker_container_status":
|
||||||
|
container_states[container_name]["state"]["Status"] = (
|
||||||
|
tags.get("container_status") or "unavailable"
|
||||||
|
)
|
||||||
|
elif measurement == "docker_container_health":
|
||||||
|
container_states[container_name]["state"]["Health"] = {
|
||||||
|
"Status": fields.get("health_status") or "unknown",
|
||||||
|
}
|
||||||
|
elif measurement == "docker_container_cpu":
|
||||||
usage = _number(fields.get("usage_percent"))
|
usage = _number(fields.get("usage_percent"))
|
||||||
if usage is not None:
|
if usage is not None:
|
||||||
stats["CPUPerc"] = f"{usage:.3f}%"
|
stats["CPUPerc"] = f"{usage:.3f}%"
|
||||||
@@ -906,7 +922,12 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def stage_state(stage_id: str) -> str:
|
def stage_state(stage_id: str) -> str:
|
||||||
if not perception:
|
if not perception or perception.get("state") == "unavailable":
|
||||||
|
return "unavailable"
|
||||||
|
if perception.get("state") in ("waiting", "synchronizing", "starting"):
|
||||||
|
return "waiting"
|
||||||
|
# A full-profile heartbeat is not a measurement of each individual stage.
|
||||||
|
if perception.get("profile_name"):
|
||||||
return "unavailable"
|
return "unavailable"
|
||||||
if not busy:
|
if not busy:
|
||||||
return "ready"
|
return "ready"
|
||||||
@@ -970,6 +991,13 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
|
|||||||
else None
|
else None
|
||||||
),
|
),
|
||||||
"model_load_seconds": _number(perception.get("model_load_seconds")),
|
"model_load_seconds": _number(perception.get("model_load_seconds")),
|
||||||
|
"profile_name": perception.get("profile_name"),
|
||||||
|
"input_state": perception.get("input_state"),
|
||||||
|
"wait_reason": perception.get("wait_reason"),
|
||||||
|
"live_children": _number(perception.get("live_children")),
|
||||||
|
"input_pauses": _number(perception.get("input_pauses")),
|
||||||
|
"pending_bundles": _number(perception.get("pending_bundles")),
|
||||||
|
"buffer_bytes": _number(perception.get("buffer_bytes")),
|
||||||
"stages": [stage_document(stage_id, label) for stage_id, label in stages],
|
"stages": [stage_document(stage_id, label) for stage_id, label in stages],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -63,6 +63,21 @@ def test_broker_apply_preserves_private_environment_and_changes_only_bind(
|
|||||||
assert result["ready"] is True
|
assert result["ready"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_broker_apply_repairs_missing_listener_without_changing_secrets(tmp_path, monkeypatch):
|
||||||
|
before = "MISSIONCORE_MQTT_BIND_ADDRESS=192.168.68.56\nSECRET=keep\n"
|
||||||
|
(tmp_path / ".env").write_text(before)
|
||||||
|
(tmp_path / "compose.yaml").write_text("services: {}\n")
|
||||||
|
reachable = iter((False, True))
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(network, "_address_belongs_to_host", lambda _: True)
|
||||||
|
monkeypatch.setattr(network, "_tcp_reachable", lambda *a, **k: next(reachable))
|
||||||
|
monkeypatch.setattr(network, "_run_compose_broker", lambda path: calls.append(path))
|
||||||
|
result = network.apply_broker_network(_target(), tmp_path)
|
||||||
|
assert not result["changed"] and result["repaired"] and result["ready"]
|
||||||
|
assert calls == [tmp_path]
|
||||||
|
assert (tmp_path / ".env").read_text() == before
|
||||||
|
|
||||||
|
|
||||||
def test_broker_apply_rolls_back_environment_when_listener_does_not_open(
|
def test_broker_apply_rolls_back_environment_when_listener_does_not_open(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|||||||
@@ -447,6 +447,36 @@ def test_compute_contour_maps_to_worker_identity_without_singleton_defaults() ->
|
|||||||
assert profile.address == "192.0.2.25"
|
assert profile.address == "192.0.2.25"
|
||||||
|
|
||||||
|
|
||||||
|
def test_gpu_clocks_and_explicit_container_status_are_not_invented() -> None:
|
||||||
|
raw = _agent_raw_document({"samples": [
|
||||||
|
{"measurement": "nvidia_smi", "payload": {"fields": {
|
||||||
|
"clocks_current_sm": 210, "clocks_current_memory": 405,
|
||||||
|
}}},
|
||||||
|
{"measurement": "docker_container_status", "payload": {
|
||||||
|
"tags": {"container_name": "sentinel-frigate", "container_status": "exited"},
|
||||||
|
"fields": {},
|
||||||
|
}},
|
||||||
|
]})
|
||||||
|
assert raw["gpu"]["sm_clock_mhz"] == 210
|
||||||
|
assert raw["gpu"]["memory_clock_mhz"] == 405
|
||||||
|
assert raw["container_states"]["sentinel-frigate"]["state"]["Status"] == "exited"
|
||||||
|
|
||||||
|
|
||||||
|
def test_unavailable_pipeline_is_not_ready_and_profile_wait_survives(tmp_path) -> None:
|
||||||
|
probe = _probe()
|
||||||
|
probe["raw"]["perception"] = {"state": "unavailable", "collector_state": "unavailable"}
|
||||||
|
service = WorkerTelemetryService(WorkerProfileStore(tmp_path), lambda _: probe, cache_seconds=0)
|
||||||
|
assert all(s["state"] == "unavailable" for s in service.snapshot(1)["pipeline"]["stages"])
|
||||||
|
probe["raw"]["perception"] = {
|
||||||
|
"state": "waiting", "profile_name": "K1 DDRNet", "input_pauses": 1,
|
||||||
|
"live_children": 4, "buffer_bytes": 1024,
|
||||||
|
}
|
||||||
|
profile = service.snapshot(1)["pipeline"]
|
||||||
|
assert profile["service_state"] == "waiting"
|
||||||
|
assert profile["live_children"] == 4 and profile["buffer_bytes"] == 1024
|
||||||
|
assert all(s["state"] == "waiting" for s in profile["stages"])
|
||||||
|
|
||||||
|
|
||||||
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -126,6 +126,22 @@ def test_normalizer_enforces_oss_compatible_bounded_retention() -> None:
|
|||||||
assert "DELETE FROM contour_telemetry_samples" in NORMALIZER_SOURCE
|
assert "DELETE FROM contour_telemetry_samples" in NORMALIZER_SOURCE
|
||||||
assert "INTERVAL '30 days'" in NORMALIZER_SOURCE
|
assert "INTERVAL '30 days'" in NORMALIZER_SOURCE
|
||||||
assert "RETENTION_INTERVAL_SECONDS" in NORMALIZER_SOURCE
|
assert "RETENTION_INTERVAL_SECONDS" in NORMALIZER_SOURCE
|
||||||
|
assert "ORDER BY observed_at LIMIT %s" in NORMALIZER_SOURCE
|
||||||
|
assert "statement_timeout = '2s'" in NORMALIZER_SOURCE
|
||||||
|
assert "samples.tableoid = expired.tableoid" in NORMALIZER_SOURCE
|
||||||
|
assert "target=_retention_loop" in NORMALIZER_SOURCE
|
||||||
|
callback = NORMALIZER_SOURCE.split(" def on_message(", 1)[1]
|
||||||
|
assert "DELETE FROM" not in callback
|
||||||
|
|
||||||
|
|
||||||
|
def test_telegraf_network_recovery_and_database_memory_are_bounded() -> None:
|
||||||
|
compose = yaml.safe_load(COMPOSE_PATH.read_text())
|
||||||
|
assert "shared_buffers=128MB" in compose["services"]["timescale"]["command"]
|
||||||
|
for platform, outputs in (("windows", 2), ("linux", 1)):
|
||||||
|
template = (REPOSITORY_ROOT / "deploy/telemetry-plane/telegraf" /
|
||||||
|
f"mission-core-{platform}.conf.tmpl").read_text()
|
||||||
|
assert template.count('startup_error_behavior = "retry"') == outputs
|
||||||
|
assert "metric_buffer_limit = 2000" in template
|
||||||
|
|
||||||
|
|
||||||
def test_normalizer_rejects_unschematized_pipeline_payload() -> None:
|
def test_normalizer_rejects_unschematized_pipeline_payload() -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user