fix(telemetry): restore worker agent delivery and truthful system status

This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 17:35:20 +03:00
parent 03e8e71bea
commit c570f7d938
15 changed files with 254 additions and 31 deletions
+8 -1
View File
@@ -486,7 +486,7 @@ def _run_compose_broker(telemetry_plane_root: Path) -> None:
]
for suffix, timeout in (
(["config", "--quiet"], 20),
(["up", "-d", "--no-build", "broker"], 45),
(["up", "-d", "--no-deps", "--no-build", "--force-recreate", "broker"], 45),
):
try:
completed = subprocess.run(
@@ -525,11 +525,17 @@ def apply_broker_network(
target.mqtt_bind_address,
)
changed = after != before
repaired = False
mode = environment_path.stat().st_mode
try:
if changed:
_atomic_private_write(environment_path, after, mode=mode)
_run_compose_broker(telemetry_plane_root)
elif not _tcp_reachable(target.mqtt_bind_address, target.mqtt_port, timeout=3):
# An unchanged file does not prove Docker still publishes the port
# after a host/network restart. This is an explicit Apply, not a GET.
_run_compose_broker(telemetry_plane_root)
repaired = True
if not _tcp_reachable(
target.mqtt_bind_address,
target.mqtt_port,
@@ -553,6 +559,7 @@ def apply_broker_network(
"contour_id": target.contour_id,
"target": "broker",
"changed": changed,
"repaired": repaired,
"applied_at_utc": _utc_now(),
"ready": True,
}
+31 -3
View File
@@ -478,7 +478,7 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
perception: dict[str, Any] = raw["perception"]
pipeline_stage_metrics: dict[str, dict[str, Any]] = {}
pipeline_active_stages: set[str] = set()
for sample in samples:
for sample in sorted(samples, key=lambda item: str(item.get("observed_at_utc", ""))):
measurement = sample.get("measurement")
payload, fields, tags = _metric_payload(sample)
if sample.get("kind") == "pipeline":
@@ -497,6 +497,9 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
"service_state",
"current_stage",
"active_request_id",
"profile_name",
"input_state",
"wait_reason",
):
value = fields.get(name)
if isinstance(value, str):
@@ -511,6 +514,8 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
if model_load_seconds is not None:
perception["model_load_seconds"] = model_load_seconds
perception["collector_state"] = fields.get("collector_state")
for name in ("live_children", "input_pauses", "pending_bundles", "buffer_bytes"):
perception[name] = _number(fields.get(name))
elif sample.get("source_schema") == PIPELINE_TELEMETRY_SCHEMA:
# Native lifecycle rows are immutable run evidence. The current
# service snapshot remains owned by the periodic health sample,
@@ -593,6 +598,9 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
"memory_total_mib": memory_total,
"power_watts": _number(fields.get("power_draw")),
"temperature_celsius": _number(fields.get("temperature_gpu")),
"sm_clock_mhz": _number(fields.get("clocks_current_sm")),
"memory_clock_mhz": _number(fields.get("clocks_current_memory")),
"driver_version": fields.get("driver_version"),
}
elif measurement == "missioncore_triton_health":
raw["triton"] = {
@@ -637,7 +645,15 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
"image": tags.get("container_image"),
},
)
if measurement == "docker_container_cpu":
if measurement == "docker_container_status":
container_states[container_name]["state"]["Status"] = (
tags.get("container_status") or "unavailable"
)
elif measurement == "docker_container_health":
container_states[container_name]["state"]["Health"] = {
"Status": fields.get("health_status") or "unknown",
}
elif measurement == "docker_container_cpu":
usage = _number(fields.get("usage_percent"))
if usage is not None:
stats["CPUPerc"] = f"{usage:.3f}%"
@@ -906,7 +922,12 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
)
def stage_state(stage_id: str) -> str:
if not perception:
if not perception or perception.get("state") == "unavailable":
return "unavailable"
if perception.get("state") in ("waiting", "synchronizing", "starting"):
return "waiting"
# A full-profile heartbeat is not a measurement of each individual stage.
if perception.get("profile_name"):
return "unavailable"
if not busy:
return "ready"
@@ -970,6 +991,13 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
else None
),
"model_load_seconds": _number(perception.get("model_load_seconds")),
"profile_name": perception.get("profile_name"),
"input_state": perception.get("input_state"),
"wait_reason": perception.get("wait_reason"),
"live_children": _number(perception.get("live_children")),
"input_pauses": _number(perception.get("input_pauses")),
"pending_bundles": _number(perception.get("pending_bundles")),
"buffer_bytes": _number(perception.get("buffer_bytes")),
"stages": [stage_document(stage_id, label) for stage_id, label in stages],
}