SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(telemetry): restore worker agent delivery and truthful system status

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 17:35:20 +03:00
parent 03e8e71bea
commit c570f7d938
15 changed files with 254 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/telemetry-plane/compose.source.yaml
+9
View File
@@ -0,0 +1,9 @@
# Local, read-only source override for repairing the existing normalizer without
# rebuilding an image on a memory-constrained operator machine. No extra service.
services:
normalizer:
volumes:
- type: bind
source: ${MISSIONCORE_NORMALIZER_SOURCE:?Set the absolute normalizer.py path}
target: /app/normalizer.py
read_only: true
deploy/telemetry-plane/compose.yaml
+2
View File
@@ -49,6 +49,8 @@ services:
timescale:
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
# The HA image auto-tunes for the Docker VM, not this 1 GiB container.
command: ["postgres", "-c", "shared_buffers=128MB", "-c", "work_mem=4MB", "-c", "maintenance_work_mem=64MB", "-c", "effective_cache_size=512MB", "-c", "max_parallel_workers_per_gather=0", "-c", "max_connections=32"]
container_name: ndc-mission-core-telemetry-timescaledb
restart: unless-stopped
security_opt:
deploy/telemetry-plane/normalizer/normalizer.py
+39 -16
View File
@@ -31,7 +31,8 @@ SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
MAX_TAGS: Final = 256
MAX_SERIES_KEY_BYTES: Final = 4096
RETENTION_INTERVAL_SECONDS: Final = 86_400
RETENTION_INTERVAL_SECONDS: Final = 60
RETENTION_BATCH_ROWS: Final = 1000
ALLOWED_TELEMETRY_TAGS: Final = frozenset(
{
"agent_id",
@@ -434,6 +435,39 @@ def _start_query_server(
return server
def _retention_batch(connection: Any) -> int:
"""Preserve the existing 30-day policy without an unbounded startup DELETE."""
with connection.transaction(), connection.cursor() as cursor:
cursor.execute("SET LOCAL statement_timeout = '2s'")
cursor.execute("SET LOCAL lock_timeout = '250ms'")
cursor.execute(
"""
WITH expired AS (
SELECT tableoid, ctid FROM contour_telemetry_samples
WHERE observed_at < CURRENT_TIMESTAMP - INTERVAL '30 days'
ORDER BY observed_at LIMIT %s
)
DELETE FROM contour_telemetry_samples AS samples USING expired
WHERE samples.tableoid = expired.tableoid AND samples.ctid = expired.ctid
""",
(RETENTION_BATCH_ROWS,),
)
return cursor.rowcount
def _retention_loop(dsn: str) -> None:
import psycopg # type: ignore[import-not-found]
while True:
# Neither a retention timeout nor database recovery blocks MQTT I/O.
time.sleep(RETENTION_INTERVAL_SECONDS)
try:
with psycopg.connect(dsn, connect_timeout=3) as connection:
_retention_batch(connection)
except psycopg.Error:
print("telemetry retention deferred", flush=True)
def main() -> None:
import paho.mqtt.client as mqtt
import psycopg # type: ignore[import-not-found]
@@ -447,7 +481,9 @@ def main() -> None:
password = _required("MISSIONCORE_MQTT_PASSWORD")
runtime_health = TelemetryRuntimeHealth()
connection = psycopg.connect(dsn, autocommit=True)
last_retention_monotonic = 0.0
threading.Thread(
target=_retention_loop, args=(dsn,), name="telemetry-retention", daemon=True
).start()
runtime_health.set_database(True)
_start_query_server(dsn, runtime_health)
client = mqtt.Client(
@@ -480,7 +516,7 @@ def main() -> None:
runtime_health.set_mqtt(False, f"MQTT disconnected: {reason_code}")
def on_message(_client: Any, _userdata: object, message: Any) -> None:
nonlocal connection, last_retention_monotonic
nonlocal connection
try:
row = _normalize(message.topic, message.payload)
except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
@@ -489,19 +525,6 @@ def main() -> None:
for attempt in range(2):
try:
with connection.cursor() as cursor:
now_monotonic = time.monotonic()
if (
now_monotonic - last_retention_monotonic
>= RETENTION_INTERVAL_SECONDS
):
cursor.execute(
"""
DELETE FROM contour_telemetry_samples
WHERE observed_at
< CURRENT_TIMESTAMP - INTERVAL '30 days'
"""
)
last_retention_monotonic = now_monotonic
cursor.execute(
"""
INSERT INTO contour_telemetry_samples (
deploy/telemetry-plane/telegraf/Get-NdcMissionCorePipelineTelemetry.ps1
+44 -3
View File
@@ -1,7 +1,8 @@
[CmdletBinding()]
param(
[string]$ContainerName = "ndc-mission-core-perception-worker",
[string]$HealthUrl = "http://127.0.0.1:18020/health"
[string]$HealthUrl = "http://127.0.0.1:18020/health",
[string]$RuntimeSnapshot = "C:\ProgramData\NDC\MissionCore\telemetry-agent\perception\current.json"
)
$ErrorActionPreference = "Stop"
@@ -18,6 +19,46 @@ $stageIds = @(
)
$health = $null
$collectorState = "unavailable"
# A fresh full-profile export supersedes the historical service. Presence of an
# expired/broken export is NOT permission to relabel a legacy model as current.
if (Test-Path -LiteralPath $RuntimeSnapshot -PathType Leaf) {
$runtime = $null
try {
$item = Get-Item -LiteralPath $RuntimeSnapshot
$age = ([DateTime]::UtcNow - $item.LastWriteTimeUtc).TotalSeconds
if ($item.Length -le 8192 -and $age -ge -1 -and $age -le 5) {
$stream = [IO.File]::Open($RuntimeSnapshot, 'Open', 'Read', 'ReadWrite')
try {
$bytes = New-Object byte[] 8193
$count = $stream.Read($bytes, 0, $bytes.Length)
if ($count -le 8192) {
$runtime = [Text.Encoding]::UTF8.GetString($bytes, 0, $count) | ConvertFrom-Json
}
} finally { $stream.Dispose() }
if ($runtime.schema_version -ne 'missioncore.perception-runtime-observation/v1') {
$runtime = $null
}
}
} catch { $runtime = $null }
$samples = foreach ($stageId in $stageIds) {
$sample = [ordered]@{
stage_id = $stageId
stage_state = 'unavailable'
service_state = 'unavailable'
collector_state = 'unavailable'
}
if ($runtime) {
$sample.collector_state = 'live'
foreach ($name in @('service_state', 'profile_name', 'active_request_id',
'input_state', 'wait_reason', 'live_children', 'input_pauses', 'pending_bundles', 'buffer_bytes')) {
$sample[$name] = $runtime.$name
}
}
[pscustomobject]$sample
}
@($samples) | ConvertTo-Json -Compress -Depth 4
exit 0
}
try {
$healthJson = docker exec $ContainerName python3 -c `
"import urllib.request;print(urllib.request.urlopen('$HealthUrl',timeout=2).read().decode())" `
@@ -68,13 +109,13 @@ $completedRuns = if ($health -and $null -ne $health.completed_runs) {
[int64]$health.completed_runs
}
else {
[int64]0
[int64]-1
}
$failedRuns = if ($health -and $null -ne $health.failed_runs) {
[int64]$health.failed_runs
}
else {
[int64]0
[int64]-1
}
$modelLoadSeconds = if ($health -and $null -ne $health.model_load_seconds) {
[double]$health.model_load_seconds
deploy/telemetry-plane/telegraf/mission-core-linux.conf.tmpl
+4
View File
@@ -1,5 +1,8 @@
[agent]
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
flush_interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
metric_batch_size = 200
metric_buffer_limit = 2000
round_interval = true
omit_hostname = false
@@ -26,6 +29,7 @@
total = true
[[outputs.mqtt]]
startup_error_behavior = "retry"
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
username = "${MISSIONCORE_MQTT_USERNAME}"
deploy/telemetry-plane/telegraf/mission-core-windows.conf.tmpl
+9
View File
@@ -1,5 +1,8 @@
[agent]
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
flush_interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
metric_batch_size = 200
metric_buffer_limit = 2000
round_interval = true
omit_hostname = false
@@ -30,6 +33,7 @@
[[inputs.docker]]
endpoint = "npipe:////./pipe/docker_engine"
container_name_include = []
container_state_include = ["running", "exited", "restarting", "paused"]
[[inputs.http_response]]
urls = ["http://127.0.0.1:8000/v2/health/ready"]
@@ -59,6 +63,9 @@
"current_stage",
"active_request_id",
"collector_state",
"profile_name",
"input_state",
"wait_reason",
]
[[inputs.tail]]
@@ -73,6 +80,7 @@
data_type = "string"
[[outputs.mqtt]]
startup_error_behavior = "retry"
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
username = "${MISSIONCORE_MQTT_USERNAME}"
@@ -83,6 +91,7 @@
namedrop = ["missioncore_pipeline", "missioncore_pipeline_event"]
[[outputs.mqtt]]
startup_error_behavior = "retry"
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/pipeline"
username = "${MISSIONCORE_MQTT_USERNAME}"