feat(system): instrument worker pipeline stages

This commit is contained in:
DCCONSTRUCTIONS
2026-07-27 21:41:40 +03:00
parent 67e12a47a4
commit 667db00b52
10 changed files with 446 additions and 152 deletions
+28
View File
@@ -394,3 +394,31 @@ def test_runtime_telemetry_captures_bounded_queue_snapshots() -> None:
row = json.loads(stream.getvalue())
assert row["schema_version"] == "missioncore.worker-runtime-telemetry/v1"
assert row["queues"]["detector"]["maximum_depth"] == 1
def test_stage_execution_telemetry_measures_named_spans_without_process_claims() -> None:
module = _module()
telemetry = module._StageExecutionTelemetry(("detector", "tracking"))
with telemetry.measure("detector", frame_index=42):
pass
with telemetry.measure("tracking", frame_index=42):
pass
snapshot = telemetry.snapshot()
assert snapshot["current_stage"] is None
assert snapshot["active_stages"] == []
assert snapshot["active_frame_index"] == 42
assert snapshot["stages"]["detector"]["activations"] == 1
assert snapshot["stages"]["tracking"]["activations"] == 1
assert snapshot["stages"]["detector"]["elapsed_seconds"] >= 0
assert snapshot["stages"]["tracking"]["elapsed_seconds"] >= 0
assert sum(
stage["share_percent"] for stage in snapshot["stages"].values()
) == pytest.approx(100)
with (
pytest.raises(RuntimeError, match="unknown pipeline stage"),
telemetry.measure("unregistered"),
):
pass
+24 -2
View File
@@ -121,11 +121,24 @@ def _probe(
"perception": {
"state": "busy",
"current_stage": "detector",
"active_stages": ["detector", "semantic-model"],
"active_request_id": "run-001",
"active_frame_index": 42,
"completed_runs": 3,
"failed_runs": 0,
"model_load_seconds": 10.5,
"stage_metrics": {
"detector": {
"elapsed_seconds": 2.5,
"activations": 42,
"share_percent": 62.5,
},
"semantic-model": {
"elapsed_seconds": 1.5,
"activations": 11,
"share_percent": 37.5,
},
},
},
},
}
@@ -203,16 +216,25 @@ def test_worker_telemetry_separates_mission_core_and_external_load(
assert runtimes["sentinel-frigate"]["external"] is True
assert runtimes["sentinel-frigate"]["cpu_percent"] == 150
assert document["pipeline"]["active_request_id"] == "run-001"
assert next(
detector_stage = next(
stage
for stage in document["pipeline"]["stages"]
if stage["id"] == "detector"
)["state"] == "active"
)
assert detector_stage["state"] == "active"
assert detector_stage["elapsed_seconds"] == 2.5
assert detector_stage["activations"] == 42
assert detector_stage["share_percent"] == 62.5
assert next(
stage
for stage in document["pipeline"]["stages"]
if stage["id"] == "semantic-model"
)["state"] == "active"
assert next(
stage
for stage in document["pipeline"]["stages"]
if stage["id"] == "preprocessing"
)["share_percent"] is None
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(