fix(telemetry): restore worker agent delivery and truthful system status
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,9 @@
|
||||
# Local, read-only source override for repairing the existing normalizer without
|
||||
# rebuilding an image on a memory-constrained operator machine. No extra service.
|
||||
services:
|
||||
normalizer:
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${MISSIONCORE_NORMALIZER_SOURCE:?Set the absolute normalizer.py path}
|
||||
target: /app/normalizer.py
|
||||
read_only: true
|
||||
@@ -49,6 +49,8 @@ services:
|
||||
|
||||
timescale:
|
||||
image: timescale/timescaledb-ha:pg16.14-ts2.28.2-all-oss
|
||||
# The HA image auto-tunes for the Docker VM, not this 1 GiB container.
|
||||
command: ["postgres", "-c", "shared_buffers=128MB", "-c", "work_mem=4MB", "-c", "maintenance_work_mem=64MB", "-c", "effective_cache_size=512MB", "-c", "max_parallel_workers_per_gather=0", "-c", "max_connections=32"]
|
||||
container_name: ndc-mission-core-telemetry-timescaledb
|
||||
restart: unless-stopped
|
||||
security_opt:
|
||||
|
||||
@@ -31,7 +31,8 @@ SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
|
||||
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
|
||||
MAX_TAGS: Final = 256
|
||||
MAX_SERIES_KEY_BYTES: Final = 4096
|
||||
RETENTION_INTERVAL_SECONDS: Final = 86_400
|
||||
RETENTION_INTERVAL_SECONDS: Final = 60
|
||||
RETENTION_BATCH_ROWS: Final = 1000
|
||||
ALLOWED_TELEMETRY_TAGS: Final = frozenset(
|
||||
{
|
||||
"agent_id",
|
||||
@@ -434,6 +435,39 @@ def _start_query_server(
|
||||
return server
|
||||
|
||||
|
||||
def _retention_batch(connection: Any) -> int:
|
||||
"""Preserve the existing 30-day policy without an unbounded startup DELETE."""
|
||||
with connection.transaction(), connection.cursor() as cursor:
|
||||
cursor.execute("SET LOCAL statement_timeout = '2s'")
|
||||
cursor.execute("SET LOCAL lock_timeout = '250ms'")
|
||||
cursor.execute(
|
||||
"""
|
||||
WITH expired AS (
|
||||
SELECT tableoid, ctid FROM contour_telemetry_samples
|
||||
WHERE observed_at < CURRENT_TIMESTAMP - INTERVAL '30 days'
|
||||
ORDER BY observed_at LIMIT %s
|
||||
)
|
||||
DELETE FROM contour_telemetry_samples AS samples USING expired
|
||||
WHERE samples.tableoid = expired.tableoid AND samples.ctid = expired.ctid
|
||||
""",
|
||||
(RETENTION_BATCH_ROWS,),
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def _retention_loop(dsn: str) -> None:
|
||||
import psycopg # type: ignore[import-not-found]
|
||||
|
||||
while True:
|
||||
# Neither a retention timeout nor database recovery blocks MQTT I/O.
|
||||
time.sleep(RETENTION_INTERVAL_SECONDS)
|
||||
try:
|
||||
with psycopg.connect(dsn, connect_timeout=3) as connection:
|
||||
_retention_batch(connection)
|
||||
except psycopg.Error:
|
||||
print("telemetry retention deferred", flush=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import paho.mqtt.client as mqtt
|
||||
import psycopg # type: ignore[import-not-found]
|
||||
@@ -447,7 +481,9 @@ def main() -> None:
|
||||
password = _required("MISSIONCORE_MQTT_PASSWORD")
|
||||
runtime_health = TelemetryRuntimeHealth()
|
||||
connection = psycopg.connect(dsn, autocommit=True)
|
||||
last_retention_monotonic = 0.0
|
||||
threading.Thread(
|
||||
target=_retention_loop, args=(dsn,), name="telemetry-retention", daemon=True
|
||||
).start()
|
||||
runtime_health.set_database(True)
|
||||
_start_query_server(dsn, runtime_health)
|
||||
client = mqtt.Client(
|
||||
@@ -480,7 +516,7 @@ def main() -> None:
|
||||
runtime_health.set_mqtt(False, f"MQTT disconnected: {reason_code}")
|
||||
|
||||
def on_message(_client: Any, _userdata: object, message: Any) -> None:
|
||||
nonlocal connection, last_retention_monotonic
|
||||
nonlocal connection
|
||||
try:
|
||||
row = _normalize(message.topic, message.payload)
|
||||
except (ValueError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
@@ -489,19 +525,6 @@ def main() -> None:
|
||||
for attempt in range(2):
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
now_monotonic = time.monotonic()
|
||||
if (
|
||||
now_monotonic - last_retention_monotonic
|
||||
>= RETENTION_INTERVAL_SECONDS
|
||||
):
|
||||
cursor.execute(
|
||||
"""
|
||||
DELETE FROM contour_telemetry_samples
|
||||
WHERE observed_at
|
||||
< CURRENT_TIMESTAMP - INTERVAL '30 days'
|
||||
"""
|
||||
)
|
||||
last_retention_monotonic = now_monotonic
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO contour_telemetry_samples (
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ContainerName = "ndc-mission-core-perception-worker",
|
||||
[string]$HealthUrl = "http://127.0.0.1:18020/health"
|
||||
[string]$HealthUrl = "http://127.0.0.1:18020/health",
|
||||
[string]$RuntimeSnapshot = "C:\ProgramData\NDC\MissionCore\telemetry-agent\perception\current.json"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -18,6 +19,46 @@ $stageIds = @(
|
||||
)
|
||||
$health = $null
|
||||
$collectorState = "unavailable"
|
||||
# A fresh full-profile export supersedes the historical service. Presence of an
|
||||
# expired/broken export is NOT permission to relabel a legacy model as current.
|
||||
if (Test-Path -LiteralPath $RuntimeSnapshot -PathType Leaf) {
|
||||
$runtime = $null
|
||||
try {
|
||||
$item = Get-Item -LiteralPath $RuntimeSnapshot
|
||||
$age = ([DateTime]::UtcNow - $item.LastWriteTimeUtc).TotalSeconds
|
||||
if ($item.Length -le 8192 -and $age -ge -1 -and $age -le 5) {
|
||||
$stream = [IO.File]::Open($RuntimeSnapshot, 'Open', 'Read', 'ReadWrite')
|
||||
try {
|
||||
$bytes = New-Object byte[] 8193
|
||||
$count = $stream.Read($bytes, 0, $bytes.Length)
|
||||
if ($count -le 8192) {
|
||||
$runtime = [Text.Encoding]::UTF8.GetString($bytes, 0, $count) | ConvertFrom-Json
|
||||
}
|
||||
} finally { $stream.Dispose() }
|
||||
if ($runtime.schema_version -ne 'missioncore.perception-runtime-observation/v1') {
|
||||
$runtime = $null
|
||||
}
|
||||
}
|
||||
} catch { $runtime = $null }
|
||||
$samples = foreach ($stageId in $stageIds) {
|
||||
$sample = [ordered]@{
|
||||
stage_id = $stageId
|
||||
stage_state = 'unavailable'
|
||||
service_state = 'unavailable'
|
||||
collector_state = 'unavailable'
|
||||
}
|
||||
if ($runtime) {
|
||||
$sample.collector_state = 'live'
|
||||
foreach ($name in @('service_state', 'profile_name', 'active_request_id',
|
||||
'input_state', 'wait_reason', 'live_children', 'input_pauses', 'pending_bundles', 'buffer_bytes')) {
|
||||
$sample[$name] = $runtime.$name
|
||||
}
|
||||
}
|
||||
[pscustomobject]$sample
|
||||
}
|
||||
@($samples) | ConvertTo-Json -Compress -Depth 4
|
||||
exit 0
|
||||
}
|
||||
try {
|
||||
$healthJson = docker exec $ContainerName python3 -c `
|
||||
"import urllib.request;print(urllib.request.urlopen('$HealthUrl',timeout=2).read().decode())" `
|
||||
@@ -68,13 +109,13 @@ $completedRuns = if ($health -and $null -ne $health.completed_runs) {
|
||||
[int64]$health.completed_runs
|
||||
}
|
||||
else {
|
||||
[int64]0
|
||||
[int64]-1
|
||||
}
|
||||
$failedRuns = if ($health -and $null -ne $health.failed_runs) {
|
||||
[int64]$health.failed_runs
|
||||
}
|
||||
else {
|
||||
[int64]0
|
||||
[int64]-1
|
||||
}
|
||||
$modelLoadSeconds = if ($health -and $null -ne $health.model_load_seconds) {
|
||||
[double]$health.model_load_seconds
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
[agent]
|
||||
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||
flush_interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||
metric_batch_size = 200
|
||||
metric_buffer_limit = 2000
|
||||
round_interval = true
|
||||
omit_hostname = false
|
||||
|
||||
@@ -26,6 +29,7 @@
|
||||
total = true
|
||||
|
||||
[[outputs.mqtt]]
|
||||
startup_error_behavior = "retry"
|
||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
[agent]
|
||||
interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||
flush_interval = "${MISSIONCORE_TELEMETRY_INTERVAL}"
|
||||
metric_batch_size = 200
|
||||
metric_buffer_limit = 2000
|
||||
round_interval = true
|
||||
omit_hostname = false
|
||||
|
||||
@@ -30,6 +33,7 @@
|
||||
[[inputs.docker]]
|
||||
endpoint = "npipe:////./pipe/docker_engine"
|
||||
container_name_include = []
|
||||
container_state_include = ["running", "exited", "restarting", "paused"]
|
||||
|
||||
[[inputs.http_response]]
|
||||
urls = ["http://127.0.0.1:8000/v2/health/ready"]
|
||||
@@ -59,6 +63,9 @@
|
||||
"current_stage",
|
||||
"active_request_id",
|
||||
"collector_state",
|
||||
"profile_name",
|
||||
"input_state",
|
||||
"wait_reason",
|
||||
]
|
||||
|
||||
[[inputs.tail]]
|
||||
@@ -73,6 +80,7 @@
|
||||
data_type = "string"
|
||||
|
||||
[[outputs.mqtt]]
|
||||
startup_error_behavior = "retry"
|
||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||
@@ -83,6 +91,7 @@
|
||||
namedrop = ["missioncore_pipeline", "missioncore_pipeline_event"]
|
||||
|
||||
[[outputs.mqtt]]
|
||||
startup_error_behavior = "retry"
|
||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/pipeline"
|
||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||
|
||||
Reference in New Issue
Block a user