feat(telemetry): add bounded native pipeline lifecycle

2026-07-29 12:32:52 +03:00
parent d8ccc9345a
commit 636db089ae
13 changed files with 830 additions and 13 deletions
@@ -24,6 +24,8 @@ SOURCE_SCHEMAS: Final = {
    "heartbeat": "missioncore.agent-heartbeat/v1",
 }
 TELEGRAF_SCHEMA: Final = "telegraf.metric-json/v1"
+PIPELINE_RECORD_SCHEMA: Final = "missioncore.pipeline-telemetry-record/v1"
+PIPELINE_EVENT_MEASUREMENT: Final = "missioncore_pipeline_event"
 QUERY_SCHEMA: Final = "missioncore.telemetry-query/v1"
 SAFE_IDENTIFIER: Final = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$")
 MAX_PAYLOAD_BYTES: Final = 1024 * 1024
@@ -43,6 +45,7 @@ ALLOWED_TELEMETRY_TAGS: Final = frozenset(
        "cpu",
        "device",
        "engine_host",
+        "event_type",
        "gpu_name",
        "host",
        "interface",
@@ -53,6 +56,7 @@ ALLOWED_TELEMETRY_TAGS: Final = frozenset(
        "path",
        "request_id",
        "run_id",
+        "run_state",
        "server_version",
        "source_id",
        "source_package_id",
@@ -135,6 +139,30 @@ def _sanitize_tags(document: dict[str, Any]) -> dict[str, Any]:
    return {**document, "tags": sanitized}


+def _unwrap_pipeline_event(
+    topic: str,
+    kind: str,
+    document: dict[str, Any],
+) -> dict[str, Any]:
+    if kind != "pipeline" or document.get("name") != PIPELINE_EVENT_MEASUREMENT:
+        return document
+    fields = document.get("fields")
+    if not isinstance(fields, dict) or not isinstance(fields.get("value"), str):
+        raise ValueError("Telegraf pipeline event value is missing")
+    record = json.loads(fields["value"])
+    if (
+        not isinstance(record, dict)
+        or record.get("schema_version") != PIPELINE_RECORD_SCHEMA
+        or record.get("topic") != topic
+        or not isinstance(record.get("payload"), dict)
+    ):
+        raise ValueError("pipeline event record is invalid")
+    payload = record["payload"]
+    if payload.get("schema_version") != SOURCE_SCHEMAS["pipeline"]:
+        raise ValueError("pipeline event payload schema is invalid")
+    return payload
+
+
 def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
    if len(payload) > MAX_PAYLOAD_BYTES:
        raise ValueError("telemetry payload exceeds the 1 MiB contract")
@@ -144,8 +172,9 @@ def _normalize(topic: str, payload: bytes) -> tuple[object, ...]:
    document = json.loads(payload.decode("utf-8"))
    if not isinstance(document, dict):
        raise ValueError("payload must be an object")
-    document = _sanitize_tags(document)
    kind = match.group("kind")
+    document = _unwrap_pipeline_event(topic, kind, document)
+    document = _sanitize_tags(document)
    source_schema = document.get("schema_version")
    if source_schema == SOURCE_SCHEMAS[kind]:
        observed_value = document.get("observed_at_utc")
@@ -3,7 +3,8 @@ param(
    [string]$Version = "1.38.4",
    [string]$ExpectedSha256 = "6c7878ec319471ac85b82443baec2f3fa5dbcf1b6e2da5d5cd2cbb60fff2bb45",
    [string]$ConfigurationTemplate = "$PSScriptRoot\mission-core-windows.conf.tmpl",
-    [string]$PipelineCollector = "$PSScriptRoot\Get-NdcMissionCorePipelineTelemetry.ps1"
+    [string]$PipelineCollector = "$PSScriptRoot\Get-NdcMissionCorePipelineTelemetry.ps1",
+    [string]$PipelineJournal = "D:\NDC_MISSIONCORE\runtime\derived\.perception-persistent-publish\pipeline-telemetry.jsonl"
 )

 $ErrorActionPreference = "Stop"
@@ -41,6 +42,26 @@ if (-not (Test-Path -LiteralPath $ConfigurationTemplate -PathType Leaf)) {
 if (-not (Test-Path -LiteralPath $PipelineCollector -PathType Leaf)) {
    throw "Pipeline collector not found: $PipelineCollector"
 }
+$journalParent = Get-Item -LiteralPath (
+    Resolve-Path -LiteralPath (Split-Path $PipelineJournal -Parent)
+).Path -Force
+if (
+    -not $journalParent.PSIsContainer -or
+    ($journalParent.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
+    [IO.Path]::GetPathRoot($journalParent.FullName).TrimEnd("\") -ine "D:"
+) {
+    throw "Pipeline journal parent must be a real D: directory"
+}
+if (-not (Test-Path -LiteralPath $PipelineJournal)) {
+    New-Item -ItemType File -Path $PipelineJournal | Out-Null
+}
+$journal = Get-Item -LiteralPath $PipelineJournal -Force
+if (
+    $journal.PSIsContainer -or
+    ($journal.Attributes -band [IO.FileAttributes]::ReparsePoint)
+) {
+    throw "Pipeline journal must be a regular file"
+}

 $temporaryRoot = Join-Path $env:TEMP "ndc-mission-core-telegraf-$([Guid]::NewGuid().ToString('N'))"
 $archivePath = Join-Path $temporaryRoot "telegraf.zip"
@@ -121,6 +142,7 @@ try {
        StartType = $service.StartType.ToString()
        Configuration = $configurationPath
        PipelineCollector = $collectorPath
+        PipelineJournal = $PipelineJournal
        Sha256 = $actualSha256
    } | ConvertTo-Json -Compress
 }
@@ -6,14 +6,22 @@ param(
    [string]$ExpectedPredecessorSha256,
    [Parameter(Mandatory = $true)]
    [string]$ExpectedCandidateSha256,
+    [Parameter(Mandatory = $true)]
+    [string]$PipelineTelemetryCandidate,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedPipelineTelemetrySha256,
+    [string]$ExpectedPipelineTelemetryPredecessorSha256 = "absent",
    [string]$ContainerName = "ndc-mission-core-perception-worker",
    [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
 )

 $ErrorActionPreference = "Stop"
 $target = Join-Path $RunnerRoot "run_e15_shadow_inference.py"
+$telemetryTarget = Join-Path $RunnerRoot "pipeline_telemetry.py"
 $expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
 $expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
+$expectedTelemetry = $ExpectedPipelineTelemetrySha256.ToLowerInvariant()
+$expectedTelemetryPredecessor = $ExpectedPipelineTelemetryPredecessorSha256.ToLowerInvariant()
 $predecessor = (
    Get-FileHash -Algorithm SHA256 -LiteralPath $target
 ).Hash.ToLowerInvariant()
@@ -26,12 +34,34 @@ if (
 ) {
    throw "Runner candidate digest changed"
 }
+$telemetryCandidateDigest = (
+    Get-FileHash -Algorithm SHA256 -LiteralPath $PipelineTelemetryCandidate
+).Hash.ToLowerInvariant()
+if ($telemetryCandidateDigest -ne $expectedTelemetry) {
+    throw "Pipeline telemetry candidate digest changed"
+}
+$hadTelemetryModule = Test-Path -LiteralPath $telemetryTarget -PathType Leaf
+$telemetryPredecessor = if ($hadTelemetryModule) {
+    (Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget).Hash.ToLowerInvariant()
+}
+else {
+    "absent"
+}
+if ($telemetryPredecessor -ne $expectedTelemetryPredecessor) {
+    throw "Pipeline telemetry predecessor digest changed"
+}
 $backup = Join-Path $RunnerRoot (
    "run_e15_shadow_inference.py.rollback-" +
    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
    "-" +
    $predecessor.Substring(0, 12)
 )
+$telemetryBackup = Join-Path $RunnerRoot (
+    "pipeline_telemetry.py.rollback-" +
+    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
+    "-" +
+    $telemetryPredecessor.Substring(0, [Math]::Min(12, $telemetryPredecessor.Length))
+)
 $healthCode = "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:18020/health',timeout=2).read().decode())"

 function Wait-PerceptionHealth {
@@ -52,7 +82,11 @@ function Wait-PerceptionHealth {
            if (
                $health.ok -and
                $health.state -eq "ready" -and
-                (-not $RequireStageMetrics -or $health.stage_metrics)
+                (-not $RequireStageMetrics -or (
+                    $health.stage_metrics -and
+                    $health.native_pipeline_telemetry -and
+                    $health.native_pipeline_telemetry.ready
+                ))
            ) {
                return $health
            }
@@ -63,7 +97,12 @@ function Wait-PerceptionHealth {
 }

 Copy-Item -LiteralPath $target -Destination $backup
+if ($hadTelemetryModule) {
+    Copy-Item -LiteralPath $telemetryTarget -Destination $telemetryBackup
+}
 try {
+    Copy-Item -LiteralPath $PipelineTelemetryCandidate `
+        -Destination $telemetryTarget -Force
    Copy-Item -LiteralPath $Candidate -Destination $target -Force
    if (
        (Get-FileHash -Algorithm SHA256 -LiteralPath $target).Hash.ToLowerInvariant() `
@@ -71,6 +110,12 @@ try {
    ) {
        throw "Runner replacement digest changed"
    }
+    if (
+        (Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget).Hash.ToLowerInvariant() `
+            -ne $expectedTelemetry
+    ) {
+        throw "Pipeline telemetry replacement digest changed"
+    }
    docker restart --time 20 $ContainerName | Out-Null
    if ($LASTEXITCODE -ne 0) {
        throw "Perception container restart failed"
@@ -82,6 +127,12 @@ try {
 }
 catch {
    Copy-Item -LiteralPath $backup -Destination $target -Force
+    if ($hadTelemetryModule) {
+        Copy-Item -LiteralPath $telemetryBackup -Destination $telemetryTarget -Force
+    }
+    else {
+        Remove-Item -LiteralPath $telemetryTarget -Force -ErrorAction SilentlyContinue
+    }
    docker restart --time 20 $ContainerName | Out-Null
    $rollbackHealth = Wait-PerceptionHealth -RequireStageMetrics $false
    if (-not $rollbackHealth) {
@@ -98,6 +149,11 @@ catch {
    CandidateSha256 = (
        Get-FileHash -Algorithm SHA256 -LiteralPath $target
    ).Hash.ToLowerInvariant()
+    PipelineTelemetryPredecessorSha256 = $telemetryPredecessor
+    PipelineTelemetryCandidateSha256 = (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $telemetryTarget
+    ).Hash.ToLowerInvariant()
+    PipelineTelemetryBackup = if ($hadTelemetryModule) { $telemetryBackup } else { $null }
    Backup = $backup
    Health = $health
 } | ConvertTo-Json -Depth 12 -Compress
@@ -0,0 +1,67 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$Candidate,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedPredecessorSha256,
+    [Parameter(Mandatory = $true)]
+    [string]$ExpectedCandidateSha256,
+    [string]$RunnerRoot = "D:\NDC_MISSIONCORE\runtime\derived\e23-runner-20260724-002"
+)
+
+$ErrorActionPreference = "Stop"
+$target = Join-Path $RunnerRoot "Invoke-E15PersistentShadowRun.ps1"
+$expectedPredecessor = $ExpectedPredecessorSha256.ToLowerInvariant()
+$expectedCandidate = $ExpectedCandidateSha256.ToLowerInvariant()
+$predecessor = (
+    Get-FileHash -Algorithm SHA256 -LiteralPath $target
+).Hash.ToLowerInvariant()
+if ($predecessor -ne $expectedPredecessor) {
+    throw "Persistent launcher predecessor digest changed"
+}
+if (
+    (Get-FileHash -Algorithm SHA256 -LiteralPath $Candidate).Hash.ToLowerInvariant() `
+        -ne $expectedCandidate
+) {
+    throw "Persistent launcher candidate digest changed"
+}
+$backup = Join-Path $RunnerRoot (
+    "Invoke-E15PersistentShadowRun.ps1.rollback-" +
+    [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssZ") +
+    "-" +
+    $predecessor.Substring(0, 12)
+)
+
+Copy-Item -LiteralPath $target -Destination $backup
+try {
+    Copy-Item -LiteralPath $Candidate -Destination $target -Force
+    $installed = (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $target
+    ).Hash.ToLowerInvariant()
+    if ($installed -ne $expectedCandidate) {
+        throw "Persistent launcher replacement digest changed"
+    }
+    $tokens = $null
+    $parseErrors = $null
+    [Management.Automation.Language.Parser]::ParseFile(
+        $target,
+        [ref]$tokens,
+        [ref]$parseErrors
+    ) | Out-Null
+    if (@($parseErrors).Count -ne 0) {
+        throw "Persistent launcher candidate has PowerShell parse errors"
+    }
+}
+catch {
+    Copy-Item -LiteralPath $backup -Destination $target -Force
+    throw
+}
+
+[ordered]@{
+    SchemaVersion = "missioncore.worker-persistent-launcher-deploy-result/v1"
+    PredecessorSha256 = $predecessor
+    CandidateSha256 = (
+        Get-FileHash -Algorithm SHA256 -LiteralPath $target
+    ).Hash.ToLowerInvariant()
+    Backup = $backup
+} | ConvertTo-Json -Compress
@@ -2,7 +2,8 @@
 param(
    [Parameter(Mandatory = $true)]
    [string]$ConfigurationTemplate,
-    [string]$PipelineCollector = "$PSScriptRoot\Get-NdcMissionCorePipelineTelemetry.ps1"
+    [string]$PipelineCollector = "$PSScriptRoot\Get-NdcMissionCorePipelineTelemetry.ps1",
+    [string]$PipelineJournal = "D:\NDC_MISSIONCORE\runtime\derived\.perception-persistent-publish\pipeline-telemetry.jsonl"
 )

 $ErrorActionPreference = "Stop"
@@ -24,6 +25,28 @@ if (-not (Test-Path -LiteralPath $ConfigurationTemplate -PathType Leaf)) {
 if (-not (Test-Path -LiteralPath $PipelineCollector -PathType Leaf)) {
    throw "Pipeline collector not found: $PipelineCollector"
 }
+$journalParent = Get-Item -LiteralPath (
+    Resolve-Path -LiteralPath (Split-Path $PipelineJournal -Parent)
+).Path -Force
+if (
+    -not $journalParent.PSIsContainer -or
+    ($journalParent.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
+    [IO.Path]::GetPathRoot($journalParent.FullName).TrimEnd("\") -ine "D:"
+) {
+    throw "Pipeline journal parent must be a real D: directory"
+}
+if (Test-Path -LiteralPath $PipelineJournal) {
+    $journal = Get-Item -LiteralPath $PipelineJournal -Force
+    if (
+        $journal.PSIsContainer -or
+        ($journal.Attributes -band [IO.FileAttributes]::ReparsePoint)
+    ) {
+        throw "Pipeline journal must be a regular file"
+    }
+}
+else {
+    New-Item -ItemType File -Path $PipelineJournal | Out-Null
+}

 $serviceEnvironmentBefore = @((Get-ItemProperty -Path $serviceRegistryPath).Environment)
 foreach ($entry in $serviceEnvironmentBefore) {
@@ -116,6 +139,7 @@ try {
        Backup = $backupPath
        PipelineCollector = $collectorPath
        PipelineCollectorBackup = if ($hadCollector) { $backupCollectorPath } else { $null }
+        PipelineJournal = $PipelineJournal
    } | ConvertTo-Json -Compress
 }
 catch {
@@ -61,6 +61,17 @@
    "collector_state",
  ]

+[[inputs.tail]]
+  files = ["D:\\NDC_MISSIONCORE\\runtime\\derived\\.perception-persistent-publish\\pipeline-telemetry.jsonl"]
+  initial_read_offset = "saved-or-beginning"
+  watch_method = "poll"
+  max_undelivered_lines = 1000
+  character_encoding = "utf-8"
+  path_tag = ""
+  name_override = "missioncore_pipeline_event"
+  data_format = "value"
+  data_type = "string"
+
 [[outputs.mqtt]]
  servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
  topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
@@ -69,7 +80,7 @@
  qos = 1
  data_format = "json"
  json_timestamp_units = "1s"
-  namedrop = ["missioncore_pipeline"]
+  namedrop = ["missioncore_pipeline", "missioncore_pipeline_event"]

 [[outputs.mqtt]]
  servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
@@ -79,4 +90,4 @@
  qos = 1
  data_format = "json"
  json_timestamp_units = "1s"
-  namepass = ["missioncore_pipeline"]
+  namepass = ["missioncore_pipeline", "missioncore_pipeline_event"]