fix(telemetry): restore worker agent delivery and truthful system status
This commit is contained in:
@@ -486,7 +486,7 @@ def _run_compose_broker(telemetry_plane_root: Path) -> None:
|
||||
]
|
||||
for suffix, timeout in (
|
||||
(["config", "--quiet"], 20),
|
||||
(["up", "-d", "--no-build", "broker"], 45),
|
||||
(["up", "-d", "--no-deps", "--no-build", "--force-recreate", "broker"], 45),
|
||||
):
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
@@ -525,11 +525,17 @@ def apply_broker_network(
|
||||
target.mqtt_bind_address,
|
||||
)
|
||||
changed = after != before
|
||||
repaired = False
|
||||
mode = environment_path.stat().st_mode
|
||||
try:
|
||||
if changed:
|
||||
_atomic_private_write(environment_path, after, mode=mode)
|
||||
_run_compose_broker(telemetry_plane_root)
|
||||
elif not _tcp_reachable(target.mqtt_bind_address, target.mqtt_port, timeout=3):
|
||||
# An unchanged file does not prove Docker still publishes the port
|
||||
# after a host/network restart. This is an explicit Apply, not a GET.
|
||||
_run_compose_broker(telemetry_plane_root)
|
||||
repaired = True
|
||||
if not _tcp_reachable(
|
||||
target.mqtt_bind_address,
|
||||
target.mqtt_port,
|
||||
@@ -553,6 +559,7 @@ def apply_broker_network(
|
||||
"contour_id": target.contour_id,
|
||||
"target": "broker",
|
||||
"changed": changed,
|
||||
"repaired": repaired,
|
||||
"applied_at_utc": _utc_now(),
|
||||
"ready": True,
|
||||
}
|
||||
|
||||
@@ -478,7 +478,7 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
||||
perception: dict[str, Any] = raw["perception"]
|
||||
pipeline_stage_metrics: dict[str, dict[str, Any]] = {}
|
||||
pipeline_active_stages: set[str] = set()
|
||||
for sample in samples:
|
||||
for sample in sorted(samples, key=lambda item: str(item.get("observed_at_utc", ""))):
|
||||
measurement = sample.get("measurement")
|
||||
payload, fields, tags = _metric_payload(sample)
|
||||
if sample.get("kind") == "pipeline":
|
||||
@@ -497,6 +497,9 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
||||
"service_state",
|
||||
"current_stage",
|
||||
"active_request_id",
|
||||
"profile_name",
|
||||
"input_state",
|
||||
"wait_reason",
|
||||
):
|
||||
value = fields.get(name)
|
||||
if isinstance(value, str):
|
||||
@@ -511,6 +514,8 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
||||
if model_load_seconds is not None:
|
||||
perception["model_load_seconds"] = model_load_seconds
|
||||
perception["collector_state"] = fields.get("collector_state")
|
||||
for name in ("live_children", "input_pauses", "pending_bundles", "buffer_bytes"):
|
||||
perception[name] = _number(fields.get(name))
|
||||
elif sample.get("source_schema") == PIPELINE_TELEMETRY_SCHEMA:
|
||||
# Native lifecycle rows are immutable run evidence. The current
|
||||
# service snapshot remains owned by the periodic health sample,
|
||||
@@ -593,6 +598,9 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
||||
"memory_total_mib": memory_total,
|
||||
"power_watts": _number(fields.get("power_draw")),
|
||||
"temperature_celsius": _number(fields.get("temperature_gpu")),
|
||||
"sm_clock_mhz": _number(fields.get("clocks_current_sm")),
|
||||
"memory_clock_mhz": _number(fields.get("clocks_current_memory")),
|
||||
"driver_version": fields.get("driver_version"),
|
||||
}
|
||||
elif measurement == "missioncore_triton_health":
|
||||
raw["triton"] = {
|
||||
@@ -637,7 +645,15 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
|
||||
"image": tags.get("container_image"),
|
||||
},
|
||||
)
|
||||
if measurement == "docker_container_cpu":
|
||||
if measurement == "docker_container_status":
|
||||
container_states[container_name]["state"]["Status"] = (
|
||||
tags.get("container_status") or "unavailable"
|
||||
)
|
||||
elif measurement == "docker_container_health":
|
||||
container_states[container_name]["state"]["Health"] = {
|
||||
"Status": fields.get("health_status") or "unknown",
|
||||
}
|
||||
elif measurement == "docker_container_cpu":
|
||||
usage = _number(fields.get("usage_percent"))
|
||||
if usage is not None:
|
||||
stats["CPUPerc"] = f"{usage:.3f}%"
|
||||
@@ -906,7 +922,12 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
)
|
||||
|
||||
def stage_state(stage_id: str) -> str:
|
||||
if not perception:
|
||||
if not perception or perception.get("state") == "unavailable":
|
||||
return "unavailable"
|
||||
if perception.get("state") in ("waiting", "synchronizing", "starting"):
|
||||
return "waiting"
|
||||
# A full-profile heartbeat is not a measurement of each individual stage.
|
||||
if perception.get("profile_name"):
|
||||
return "unavailable"
|
||||
if not busy:
|
||||
return "ready"
|
||||
@@ -970,6 +991,13 @@ def _pipeline_document(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
else None
|
||||
),
|
||||
"model_load_seconds": _number(perception.get("model_load_seconds")),
|
||||
"profile_name": perception.get("profile_name"),
|
||||
"input_state": perception.get("input_state"),
|
||||
"wait_reason": perception.get("wait_reason"),
|
||||
"live_children": _number(perception.get("live_children")),
|
||||
"input_pauses": _number(perception.get("input_pauses")),
|
||||
"pending_bundles": _number(perception.get("pending_bundles")),
|
||||
"buffer_bytes": _number(perception.get("buffer_bytes")),
|
||||
"stages": [stage_document(stage_id, label) for stage_id, label in stages],
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user