feat(perception): connect scoped host telemetry to recoverable profile lifecycle

2026-09-02 16:50:57 +03:00
parent e91721fe6a
commit 61cbdb30a0
9 changed files with 895 additions and 7 deletions
@@ -0,0 +1,137 @@
+# Trusted Windows Worker adapter. Read-only Docker/NVIDIA commands only.
+# Run on the host, NEVER in the model container. Requests and responses are
+# separate directories; mount only responses read-only into the runtime.
+# Docker GPU access inventory is NOT a native host/WSL process audit.
+param(
+  [Parameter(Mandatory=$true)][string]$Requests,
+  [Parameter(Mandatory=$true)][string]$Responses,
+  [Parameter(Mandatory=$true)][string]$Container,
+  [string]$WorkerId = 'worker-006',
+  [ValidateRange(1,300)][int]$Seconds = 240,
+  # Explicit bounded lab fault, off by default. Never supplied by input data.
+  [string]$DiagnosticDelayAfterProgressFile = '',
+  [ValidateRange(0,5000)][int]$DiagnosticDelayMilliseconds = 0
+)
+$ErrorActionPreference = 'Stop'
+if ($Container -notmatch '^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$') { throw 'Invalid launcher target' }
+if ([IO.Path]::GetFullPath($Requests) -eq [IO.Path]::GetFullPath($Responses)) { throw 'Separate control mounts required' }
+$utf8 = New-Object System.Text.UTF8Encoding($false)
+$deadline = [Diagnostics.Stopwatch]::StartNew()
+$targetId = $null
+$lastNonce = $null
+$samples = $errors = 0
+$lastErrorPhase = $null
+$phase = 'request'
+$delayEvidence = $null
+
+function Read-Command([string]$File, [string]$Arguments, [int]$Limit = 65536) {
+  $info = New-Object Diagnostics.ProcessStartInfo
+  $info.FileName = $File
+  $info.Arguments = $Arguments
+  $info.UseShellExecute = $false
+  $info.CreateNoWindow = $true
+  $info.RedirectStandardOutput = $true
+  $info.RedirectStandardError = $true
+  $process = New-Object Diagnostics.Process
+  $process.StartInfo = $info
+  try {
+    if (-not $process.Start()) { throw 'Command did not start' }
+    # Every command below has a fixed projection and <=64 inspected objects.
+    # stderr is bounded by the known CLI, never copied into the channel/log.
+    $output = $process.StandardOutput.ReadToEndAsync()
+    $errorOutput = $process.StandardError.ReadToEndAsync()
+    if (-not $process.WaitForExit(750)) {
+      $process.Kill()
+      $process.WaitForExit()
+      throw 'Read-only command timeout'
+    }
+    $value = $output.GetAwaiter().GetResult()
+    if ($process.ExitCode -ne 0 -or $value.Length -gt $Limit) { throw 'Read-only command failed or exceeded bound' }
+    return $value.Trim()
+  } finally { $process.Dispose() }
+}
+
+# Never serialize Config.Env, mounts, credentials or full Docker inspect.
+$template = '{"id":{{json .Id}},"image":{{json .Image}},"running":{{json .State.Running}},"runtime":{{json .HostConfig.Runtime}},"privileged":{{json .HostConfig.Privileged}},"devices":{{if .HostConfig.Devices}}true{{else}}false{{end}},"gpu":{{if .HostConfig.DeviceRequests}}true{{else}}false{{end}},"nano":{{json .HostConfig.NanoCpus}},"quota":{{json .HostConfig.CpuQuota}},"period":{{json .HostConfig.CpuPeriod}},"cpuset":{{json .HostConfig.CpusetCpus}},"memory":{{json .HostConfig.Memory}}}'
+$escapedTemplate = $template.Replace('"', '\"')
+try {
+  while ($deadline.Elapsed.TotalSeconds -lt $Seconds -and -not (Test-Path (Join-Path $Requests 'stop'))) {
+    Start-Sleep -Milliseconds 25
+    try {
+      $phase = 'request'
+      $path = Join-Path $Requests 'request.json'
+      if (-not (Test-Path $path) -or (Get-Item $path).Length -gt 1024) { continue }
+      $request = [IO.File]::ReadAllText($path) | ConvertFrom-Json
+      if (@($request.PSObject.Properties.Name).Count -ne 4 -or
+          $request.schema_version -ne 'missioncore.worker-host-observation/v1' -or
+          $request.nonce -cnotmatch '^[a-f0-9]{64}$' -or
+          $request.activation_sha256 -cnotmatch '^[a-f0-9]{64}$' -or
+          $request.sequence -lt 1 -or $request.nonce -eq $lastNonce) { continue }
+      $facts = [ordered]@{
+        worker_id=$WorkerId; container_id=$null; image_sha256=$null
+        cpu_limit_millicores=$null; memory_limit_mib=$null
+        gpu_name=$null; driver_version=$null; sm_clock_mhz=$null; memory_clock_mhz=$null
+        gpu_telemetry_available=$false; inventory_complete=$false
+        competing_gpu_clients=@(); inventory_scope='docker-gpu-access'; host_process_inventory='unproved'
+      }
+      try {
+        $phase = 'gpu'
+        $gpu = (Read-Command 'nvidia-smi.exe' '--query-gpu=name,driver_version,clocks.sm,clocks.mem --format=csv,noheader,nounits -i 0' 1024).Split(',')
+        if ($gpu.Count -ne 4) { throw 'GPU schema' }
+        $facts.gpu_name=$gpu[0].Trim(); $facts.driver_version=$gpu[1].Trim()
+        $facts.sm_clock_mhz=[int]$gpu[2].Trim(); $facts.memory_clock_mhz=[int]$gpu[3].Trim()
+        $facts.gpu_telemetry_available=$true
+      } catch { $errors++; $lastErrorPhase=$phase }
+      try {
+        $phase = 'docker'
+        if (-not $targetId) {
+          $targetId = Read-Command 'docker.exe' "inspect --format {{.Id}} $Container" 128
+          if ($targetId -cnotmatch '^[a-f0-9]{64}$') { $targetId=$null; throw 'Target not ready' }
+        }
+        $ids = @((Read-Command 'docker.exe' 'ps --no-trunc -q' 8192) -split '\r?\n' | Where-Object { $_ })
+        if ($ids.Count -gt 64 -or @($ids | Where-Object { $_ -cnotmatch '^[a-f0-9]{64}$' }).Count) { throw 'Inventory bound' }
+        $inspectIds = @($ids + $targetId | Sort-Object -Unique)
+        $rows = (Read-Command 'docker.exe' ('inspect --format "' + $escapedTemplate + '" ' + ($inspectIds -join ' '))) -split '\r?\n' | ForEach-Object { $_ | ConvertFrom-Json }
+        $target = @($rows | Where-Object { $_.id -eq $targetId })
+        if ($target.Count -eq 1 -and $target[0].running) {
+          $t = $target[0]
+          $facts.container_id=$t.id; $facts.image_sha256=$t.image.Substring(7)
+          if ($t.nano -gt 0) { $facts.cpu_limit_millicores=[long]($t.nano / 1000000) }
+          elseif ($t.quota -gt 0 -and $t.period -gt 0) { $facts.cpu_limit_millicores=[long](1000 * $t.quota / $t.period) }
+          elseif (-not $t.cpuset) { $facts.cpu_limit_millicores=0 }
+          $facts.memory_limit_mib=[long][Math]::Ceiling($t.memory / 1048576)
+        }
+        $facts.competing_gpu_clients=@($rows | Where-Object {
+          $_.id -ne $targetId -and $_.id -in $ids -and
+          ($_.gpu -or $_.devices -or $_.privileged -or $_.runtime -eq 'nvidia')
+        } | ForEach-Object { 'docker:' + $_.id })
+        $after = @((Read-Command 'docker.exe' 'ps --no-trunc -q' 8192) -split '\r?\n' | Where-Object { $_ })
+        $facts.inventory_complete = (($ids | Sort-Object) -join ',') -eq (($after | Sort-Object) -join ',')
+      } catch { $errors++; $lastErrorPhase=$phase }
+      $phase = 'publish'
+      if ($DiagnosticDelayMilliseconds -gt 0 -and -not $delayEvidence -and
+          (Test-Path $DiagnosticDelayAfterProgressFile) -and
+          (Get-Item $DiagnosticDelayAfterProgressFile).Length -le 65536 -and
+          (Get-Content -Raw $DiagnosticDelayAfterProgressFile) -match '"completed": 16') {
+        $delayStarted = [Diagnostics.Stopwatch]::GetTimestamp()
+        Start-Sleep -Milliseconds $DiagnosticDelayMilliseconds
+        $delayEvidence = @{request_sequence=$request.sequence;requested_ms=$DiagnosticDelayMilliseconds;started_ticks=$delayStarted.ToString();ended_ticks=[Diagnostics.Stopwatch]::GetTimestamp().ToString();clock_domain='windows-stopwatch'}
+      }
+      $reply = [ordered]@{
+        schema_version=$request.schema_version; activation_sha256=$request.activation_sha256
+        nonce=$request.nonce; sequence=$request.sequence; facts=$facts
+      } | ConvertTo-Json -Depth 5 -Compress
+      if ($utf8.GetByteCount($reply) -gt 16384) { throw 'Reply bound' }
+      $temporary = Join-Path $Responses 'response.tmp'
+      $destination = Join-Path $Responses 'response.json'
+      [IO.File]::WriteAllText($temporary, $reply, $utf8)
+      # Windows PowerShell 5 coerces $null to an empty string for this .NET
+      # overload (invalid backup path). NullString preserves a true null.
+      if (Test-Path $destination) { [IO.File]::Replace($temporary, $destination, [NullString]::Value) }
+      else { [IO.File]::Move($temporary, $destination) }
+      $lastNonce=$request.nonce; $samples++
+    } catch { $errors++; $lastErrorPhase=$phase }
+  }
+} finally {
+  [ordered]@{schema_version='missioncore.worker-host-collector-exit/v1'; utc=[DateTime]::UtcNow.ToString('o'); monotonic_ticks=[Diagnostics.Stopwatch]::GetTimestamp().ToString(); clock_domain='windows-stopwatch'; elapsed_ms=$deadline.Elapsed.TotalMilliseconds; samples=$samples; errors=$errors; last_error_phase=$lastErrorPhase; container_id=$targetId; host_mutations=0; host_process_inventory='unproved';diagnostic_delay=$delayEvidence} | ConvertTo-Json -Depth 4 -Compress
+}
@@ -16,8 +16,12 @@ from pathlib import Path
 from pilot_freshness import CLOCK_DOMAIN

 from k1link.perception.realtime_contract import StreamStart
+from k1link.perception.streaming_continuity import StreamSuspended
 from k1link.perception.streaming_lifecycle import StreamingLifecycle
+from k1link.perception.worker_control import WorkerControlChannel
+from k1link.perception.worker_control_pump import WorkerControlPump
 from k1link.perception.worker_lease import WorkerLeaseError
+from k1link.perception.worker_operating_envelope import WorkerOperatingEnvelope


 def bounded_digest(path, limit=65536):
@@ -59,6 +63,21 @@ class FencedIngress:

 class PilotController:
    def __init__(self, args, report, mailbox, stop):
+        control_requests = getattr(args, "worker_control_requests", None)
+        control_config = None
+        if control_requests:
+            if not os.statvfs(args.worker_control_responses).f_flag & os.ST_RDONLY:
+                raise ValueError("host responses must be a read-only controller mount")
+            profile = json.loads(Path("/out/candidate-profile.json").read_text())
+            control_config = {
+                "envelope": {
+                    **profile["operating_envelope"]["reference_conditions"],
+                    "inventory_scope": "docker-gpu-access",
+                },
+                "mode": args.worker_readiness_mode,
+                "channel": "missioncore.worker-host-observation/v1",
+                "host_process_inventory": "unproved",
+            }
        config = {
            "pilot_options": {
                key: value
@@ -79,6 +98,8 @@ class PilotController:
                for key in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS")
            },
        }
+        if control_config:
+            config["worker_control"] = control_config
        self.start = StreamStart(
            run_id=args.run_id,
            source_id="RAVNOVES00-20260720T065719Z-viewer-live",
@@ -110,6 +131,25 @@ class PilotController:
            target=self._renew, name="pilot-controller-heartbeat", daemon=True
        )
        self.thread.start()
+        self.control = None
+        self.report = report
+        try:
+            if control_config:
+                self.control = WorkerControlPump(
+                    self.runtime,
+                    WorkerControlChannel(
+                        self.start,
+                        Path(control_requests),
+                        Path(args.worker_control_responses),
+                        container_id=os.environ["HOSTNAME"],
+                        clock_domain_id="worker-linux-monotonic",
+                    ),
+                    WorkerOperatingEnvelope(**control_config["envelope"]),
+                    mode=control_config["mode"],
+                )
+        except BaseException:
+            self.close("worker-control-bootstrap-failed")
+            raise
        report["runtime_binding"] = self.start.to_dict()
        report["runtime_effective_config"] = config
        report["lease_ttl_ms"] = 2000
@@ -133,6 +173,26 @@ class PilotController:
        self.heartbeat_stop.set()
        self.thread.join(timeout=1)

+    def ready(self):
+        if self.control:
+            self.control.ready()
+        else:
+            self.runtime.ready()
+
+    def spawn(self, factory):
+        # A bounded diagnostic still has its outer wall-clock watchdog. While
+        # warmup facts lag, preserve already loaded children and the local lease.
+        while True:
+            try:
+                return self.runtime.spawn(factory)
+            except StreamSuspended:
+                if not self.control:
+                    raise
+                self.heartbeat_stop.wait(0.025)
+
    def close(self, reason):
+        if self.control:
+            self.report["worker_control_thread_released"] = self.control.close()
+            self.report["worker_control"] = self.control.snapshot()
        self.stop_renewals()
        return self.runtime.close(reason)
@@ -303,7 +303,7 @@ def run(args):
                start_new_session=True,
            )

-        process = controller.runtime.spawn(spawn) if controller else spawn()
+        process = controller.spawn(spawn) if controller else spawn()
        children.append(process)
        return process

@@ -523,7 +523,7 @@ def run(args):
                return compute_gpu_impl(bundle)

        if controller:
-            controller.runtime.ready()
+            controller.ready()

        if args.telemetry_mode == "nvml":
            monitor = threading.Thread(
@@ -919,6 +919,13 @@ if __name__ == "__main__":
    parser.add_argument("--worker-lease-root")
    parser.add_argument("--lease-generation", type=int, default=1)
    parser.add_argument("--controller-image-sha256")
+    parser.add_argument("--worker-control-requests")
+    parser.add_argument("--worker-control-responses")
+    parser.add_argument(
+        "--worker-readiness-mode",
+        choices=("strict-envelope", "labelled-experiment"),
+        default="labelled-experiment",
+    )
    parser.add_argument("--stop-renew-after-sequence", type=int, default=-1)
    parser.add_argument("--recover-input", action="store_true")
    parser.add_argument("--input-gap", action="append", default=[], metavar="SEQUENCE:MILLISECONDS")
@@ -943,6 +950,10 @@ if __name__ == "__main__":
        parser.error("invalid bounded input gap plan")
    if args.recover_input and args.input_transport != "binary-ipc":
        parser.error("resumable full profile requires binary input")
+    if bool(args.worker_control_requests) != bool(args.worker_control_responses):
+        parser.error("host control needs separate request/response mounts")
+    if args.worker_control_requests and not args.recover_input:
+        parser.error("host control requires resumable input and common lifecycle")
    if gaps and (not args.recover_input or any(seq >= args.frames for seq, _ in gaps)):
        parser.error("input gaps require recovery and must be inside the bounded source")
    raise SystemExit(run(args))