fix(telemetry): restore worker agent delivery and truthful system status

This commit is contained in:
DCCONSTRUCTIONS
2026-09-02 17:35:20 +03:00
parent 03e8e71bea
commit c570f7d938
15 changed files with 254 additions and 31 deletions
+15
View File
@@ -63,6 +63,21 @@ def test_broker_apply_preserves_private_environment_and_changes_only_bind(
assert result["ready"] is True
def test_broker_apply_repairs_missing_listener_without_changing_secrets(tmp_path, monkeypatch):
before = "MISSIONCORE_MQTT_BIND_ADDRESS=192.168.68.56\nSECRET=keep\n"
(tmp_path / ".env").write_text(before)
(tmp_path / "compose.yaml").write_text("services: {}\n")
reachable = iter((False, True))
calls = []
monkeypatch.setattr(network, "_address_belongs_to_host", lambda _: True)
monkeypatch.setattr(network, "_tcp_reachable", lambda *a, **k: next(reachable))
monkeypatch.setattr(network, "_run_compose_broker", lambda path: calls.append(path))
result = network.apply_broker_network(_target(), tmp_path)
assert not result["changed"] and result["repaired"] and result["ready"]
assert calls == [tmp_path]
assert (tmp_path / ".env").read_text() == before
def test_broker_apply_rolls_back_environment_when_listener_does_not_open(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
+30
View File
@@ -447,6 +447,36 @@ def test_compute_contour_maps_to_worker_identity_without_singleton_defaults() ->
assert profile.address == "192.0.2.25"
def test_gpu_clocks_and_explicit_container_status_are_not_invented() -> None:
raw = _agent_raw_document({"samples": [
{"measurement": "nvidia_smi", "payload": {"fields": {
"clocks_current_sm": 210, "clocks_current_memory": 405,
}}},
{"measurement": "docker_container_status", "payload": {
"tags": {"container_name": "sentinel-frigate", "container_status": "exited"},
"fields": {},
}},
]})
assert raw["gpu"]["sm_clock_mhz"] == 210
assert raw["gpu"]["memory_clock_mhz"] == 405
assert raw["container_states"]["sentinel-frigate"]["state"]["Status"] == "exited"
def test_unavailable_pipeline_is_not_ready_and_profile_wait_survives(tmp_path) -> None:
probe = _probe()
probe["raw"]["perception"] = {"state": "unavailable", "collector_state": "unavailable"}
service = WorkerTelemetryService(WorkerProfileStore(tmp_path), lambda _: probe, cache_seconds=0)
assert all(s["state"] == "unavailable" for s in service.snapshot(1)["pipeline"]["stages"])
probe["raw"]["perception"] = {
"state": "waiting", "profile_name": "K1 DDRNet", "input_pauses": 1,
"live_children": 4, "buffer_bytes": 1024,
}
profile = service.snapshot(1)["pipeline"]
assert profile["service_state"] == "waiting"
assert profile["live_children"] == 4 and profile["buffer_bytes"] == 1024
assert all(s["state"] == "waiting" for s in profile["stages"])
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
tmp_path: Path,
) -> None:
@@ -126,6 +126,22 @@ def test_normalizer_enforces_oss_compatible_bounded_retention() -> None:
assert "DELETE FROM contour_telemetry_samples" in NORMALIZER_SOURCE
assert "INTERVAL '30 days'" in NORMALIZER_SOURCE
assert "RETENTION_INTERVAL_SECONDS" in NORMALIZER_SOURCE
assert "ORDER BY observed_at LIMIT %s" in NORMALIZER_SOURCE
assert "statement_timeout = '2s'" in NORMALIZER_SOURCE
assert "samples.tableoid = expired.tableoid" in NORMALIZER_SOURCE
assert "target=_retention_loop" in NORMALIZER_SOURCE
callback = NORMALIZER_SOURCE.split(" def on_message(", 1)[1]
assert "DELETE FROM" not in callback
def test_telegraf_network_recovery_and_database_memory_are_bounded() -> None:
compose = yaml.safe_load(COMPOSE_PATH.read_text())
assert "shared_buffers=128MB" in compose["services"]["timescale"]["command"]
for platform, outputs in (("windows", 2), ("linux", 1)):
template = (REPOSITORY_ROOT / "deploy/telemetry-plane/telegraf" /
f"mission-core-{platform}.conf.tmpl").read_text()
assert template.count('startup_error_behavior = "retry"') == outputs
assert "metric_buffer_limit = 2000" in template
def test_normalizer_rejects_unschematized_pipeline_payload() -> None: