SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(telemetry): add bounded native pipeline lifecycle

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 12:32:52 +03:00
parent d8ccc9345a
commit 636db089ae
13 changed files with 830 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/compute/pipeline_telemetry.py
+143 -3
View File
@@ -31,6 +31,7 @@ SAFE_TOPIC_IDENTIFIER: Final = re.compile(
MAX_TEXT_LENGTH: Final = 256
MAX_PAYLOAD_BYTES: Final = 1024 * 1024
STAGE_STATES: Final = frozenset({"started", "completed", "failed"})
RUN_STATES: Final = STAGE_STATES
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
@@ -162,6 +163,52 @@ class PipelineTelemetryEmitter:
duration_ms=duration_ms,
)
def run(
self,
state: str,
*,
duration_ms: float | None = None,
exit_code: int | None = None,
error_type: str | None = None,
) -> None:
"""Publish one run-level lifecycle event with an explicit outcome."""
document = build_pipeline_run_telemetry_document(
identity=self.identity,
state=state,
duration_ms=duration_ms,
exit_code=exit_code,
error_type=error_type,
)
self._publish(document)
def stage_event(
self,
stage_id: str,
state: str,
*,
duration_ms: float | None = None,
activation_count: int = 1,
input_count: int | None = None,
output_count: int | None = None,
queue_wait_ms: float | None = None,
error_type: str | None = None,
) -> None:
"""Publish one explicit stage event, including a per-run aggregate."""
document = build_pipeline_telemetry_document(
identity=self.identity,
stage_id=stage_id,
state=state,
duration_ms=duration_ms,
activation_count=activation_count,
input_count=input_count,
output_count=output_count,
queue_wait_ms=queue_wait_ms,
error_type=error_type,
)
self._publish(document)
def _emit(
self,
*,
@@ -177,16 +224,18 @@ class PipelineTelemetryEmitter:
outcome.queue_wait_ms,
"queue_wait_ms",
)
document = build_pipeline_telemetry_document(
identity=self.identity,
self.stage_event(
stage_id=stage_id,
state=state,
duration_ms=duration_ms,
activation_count=1,
input_count=outcome.input_count,
output_count=outcome.output_count,
queue_wait_ms=outcome.queue_wait_ms,
error_type=error_type,
)
def _publish(self, document: dict[str, Any]) -> None:
payload = _canonical_json(document)
if len(payload) > MAX_PAYLOAD_BYTES:
raise PipelineTelemetryError("pipeline telemetry exceeds the 1 MiB contract")
@@ -245,6 +294,7 @@ def build_pipeline_telemetry_document(
stage_id: str,
state: str,
duration_ms: float | None = None,
activation_count: int = 1,
input_count: int | None = None,
output_count: int | None = None,
queue_wait_ms: float | None = None,
@@ -257,6 +307,9 @@ def build_pipeline_telemetry_document(
if state not in STAGE_STATES:
raise PipelineTelemetryError("stage telemetry state is invalid")
duration_ms = _optional_duration(duration_ms, "duration_ms")
activation_count = _optional_count(activation_count, "activation_count")
if activation_count in {None, 0}:
raise PipelineTelemetryError("activation_count must be positive")
input_count = _optional_count(input_count, "input_count")
output_count = _optional_count(output_count, "output_count")
queue_wait_ms = _optional_duration(queue_wait_ms, "queue_wait_ms")
@@ -289,7 +342,7 @@ def build_pipeline_telemetry_document(
"elapsed_seconds": (
round(duration_ms / 1000.0, 9) if duration_ms is not None else None
),
"activations": 1,
"activations": activation_count,
"input_count": input_count,
"output_count": output_count,
"queue_wait_ms": queue_wait_ms,
@@ -298,6 +351,7 @@ def build_pipeline_telemetry_document(
"stage_id": stage_id,
"state": state,
"duration_ms": duration_ms,
"activation_count": activation_count,
"input_count": input_count,
"output_count": output_count,
"queue_wait_ms": queue_wait_ms,
@@ -336,6 +390,92 @@ def build_pipeline_telemetry_document(
return document
def build_pipeline_run_telemetry_document(
*,
identity: PipelineTelemetryIdentity,
state: str,
duration_ms: float | None = None,
exit_code: int | None = None,
error_type: str | None = None,
observed_at_utc: str | None = None,
) -> dict[str, Any]:
"""Build a run-level start or terminal outcome document."""
if state not in RUN_STATES:
raise PipelineTelemetryError("run telemetry state is invalid")
duration_ms = _optional_duration(duration_ms, "duration_ms")
if state == "started":
if duration_ms is not None or exit_code is not None or error_type is not None:
raise PipelineTelemetryError("a started run cannot have an outcome")
elif duration_ms is None:
raise PipelineTelemetryError("a terminal run requires duration_ms")
if exit_code is not None and (
isinstance(exit_code, bool) or not isinstance(exit_code, int) or exit_code < 0
):
raise PipelineTelemetryError("exit_code must be a non-negative integer")
if state == "completed" and exit_code is None:
raise PipelineTelemetryError("a completed run requires exit_code")
if error_type is not None:
_validate_text(error_type, "error_type")
if state == "failed" and error_type is None:
raise PipelineTelemetryError("a failed run requires error_type")
if state != "failed" and error_type is not None:
raise PipelineTelemetryError("only a failed run can have error_type")
tags = {
"agent_id": identity.agent_id,
"contour_id": identity.contour_id,
"event_type": "run",
"lab_id": identity.lab_id,
"method_id": identity.method_id,
"node_id": identity.node_id,
"run_id": identity.run_id,
"run_state": state,
"source_id": identity.source_id,
"source_package_id": identity.source_package_id,
}
if identity.request_id is not None:
tags["request_id"] = identity.request_id
document: dict[str, Any] = {
"schema_version": PIPELINE_TELEMETRY_SCHEMA,
"observed_at_utc": observed_at_utc or _utc_now(),
"node_id": identity.node_id,
"lab_id": identity.lab_id,
"run_id": identity.run_id,
"source_id": identity.source_id,
"source_package_id": identity.source_package_id,
"method_id": identity.method_id,
"event_type": "run",
"run_state": state,
"tags": tags,
"payload": {
"state": (
"busy"
if state == "started"
else ("failed" if state == "failed" else "ready")
),
"current_stage": None,
"active_request_id": (
identity.request_id or identity.run_id
if state == "started"
else None
),
"active_stages": [],
"event": {
"event_type": "run",
"state": state,
"duration_ms": duration_ms,
"exit_code": exit_code,
"error_type": error_type,
},
},
"authority": _AUTHORITY,
}
if identity.request_id is not None:
document["request_id"] = identity.request_id
return document
def _validate_text(value: object, name: str) -> str:
if (
not isinstance(value, str)
src/k1link/web/system_telemetry_api.py
+7
View File
@@ -75,6 +75,7 @@ PROMETHEUS_SAMPLE = re.compile(
r"(?:\{[^}]*\})?\s+(?P<value>[0-9.eE+-]+)$"
)
TELEMETRY_QUERY_SCHEMA: Final = "missioncore.telemetry-query/v1"
PIPELINE_TELEMETRY_SCHEMA: Final = "missioncore.agent-pipeline-telemetry/v1"
DEFAULT_TELEMETRY_QUERY_URL: Final = "http://127.0.0.1:18030"
RootProvider = Callable[[], Path]
@@ -510,6 +511,12 @@ def _agent_raw_document(document: dict[str, Any]) -> dict[str, Any]:
if model_load_seconds is not None:
perception["model_load_seconds"] = model_load_seconds
perception["collector_state"] = fields.get("collector_state")
elif sample.get("source_schema") == PIPELINE_TELEMETRY_SCHEMA:
# Native lifecycle rows are immutable run evidence. The current
# service snapshot remains owned by the periodic health sample,
# so a terminal event cannot make a healthy worker look idle,
# failed, or busy after the fact.
continue
else:
pipeline_payload = _mapping(payload.get("payload")) or payload
for name, value in pipeline_payload.items():