fix(telemetry): recover worker agent and Tailscale transport automatically

2026-09-25 23:24:27 +03:00
parent bd5873ef34
commit f3640026cf
11 changed files with 755 additions and 4 deletions
@@ -0,0 +1,87 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)][string]$ExpectedNodeId,
+    [ValidateSet('Plan','Apply','Rollback')][string]$Action = 'Plan',
+    [string]$BackupDirectory
+)
+# Telegraf 1.38.4 can report MQTT startup failure as a clean Windows-service exit.
+# SCM failure actions alone cannot recover that stopped service.
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue'
+if ($env:COMPUTERNAME -cne $ExpectedNodeId) { throw 'Worker identity mismatch' }
+$taskName = 'ndc-mission-core-telemetry-recovery'
+$root = Join-Path $env:ProgramFiles 'NDC\Mission Core\TelemetryRecovery'
+$guardPath = Join-Path $root 'Resume-Telemetry.ps1'
+$service = Get-CimInstance Win32_Service -Filter "Name='telegraf'"
+if (-not $service -or $service.PathName -notlike '*NDC\Mission Core\Telegraf\telegraf.exe*') { throw 'Managed Telegraf service required' }
+$guard = @'
+$ErrorActionPreference = 'Stop'
+$service = Get-Service -Name 'telegraf'
+if ($service.Status -eq 'Running') { exit 0 }
+if ($service.Status -ne 'Stopped') { exit 0 }
+$values = @{}
+foreach ($entry in (Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Services\telegraf').Environment) {
+    $name, $value = $entry -split '=', 2
+    if ($name -in @('MISSIONCORE_MQTT_HOST','MISSIONCORE_MQTT_PORT')) { $values[$name] = $value }
+}
+if ($values['MISSIONCORE_MQTT_HOST'] -ne '127.0.0.1' -or $values['MISSIONCORE_MQTT_PORT'] -ne '1883') { exit 1 }
+$client = [Net.Sockets.TcpClient]::new()
+try {
+    $connect = $client.ConnectAsync('127.0.0.1', 1883)
+    if (-not $connect.Wait(3000) -or -not $client.Connected) { exit 0 }
+} catch { exit 0 } finally { $client.Dispose() }
+Start-Service -Name 'telegraf'
+(Get-Service -Name 'telegraf').WaitForStatus([ServiceProcess.ServiceControllerStatus]::Running, [TimeSpan]::FromSeconds(15))
+'@
+if ($Action -eq 'Plan') {
+    [ordered]@{node=$env:COMPUTERNAME;task=$taskName;path=$guardPath;startMode=$service.StartMode;triggers=@('boot','every-minute');scope='Start stopped telemetry service only';requiresInteractiveLogin=$false}|ConvertTo-Json -Compress
+    exit 0
+}
+if ($Action -eq 'Rollback') {
+    if (-not $BackupDirectory) { throw 'BackupDirectory required' }
+    $receipt = Get-Content (Join-Path $BackupDirectory 'receipt.json') -Raw | ConvertFrom-Json
+    if ($receipt.node -cne $ExpectedNodeId -or $receipt.task -ne $taskName) { throw 'Backup identity mismatch' }
+    if ((Get-FileHash $guardPath -Algorithm SHA256).Hash -ne $receipt.installedHash) { throw 'Installed recovery script changed' }
+    Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue
+    if (Test-Path (Join-Path $BackupDirectory 'task.xml')) {
+        Register-ScheduledTask -TaskName $taskName -Xml (Get-Content (Join-Path $BackupDirectory 'task.xml') -Raw) -Force | Out-Null
+    }
+    if (Test-Path (Join-Path $BackupDirectory 'guard.ps1')) {
+        Copy-Item (Join-Path $BackupDirectory 'guard.ps1') $guardPath -Force
+    } else { Remove-Item $guardPath }
+    [ordered]@{restored=$true;task=$taskName}|ConvertTo-Json -Compress
+    exit 0
+}
+$principal = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()
+if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { throw 'Administrative installer required' }
+if (Test-Path $root) {
+    if ((Get-Item $root -Force).Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'Recovery directory must not be a reparse point' }
+}
+New-Item -ItemType Directory -Path $root -Force | Out-Null
+# SYSTEM executes this file: only SYSTEM and Administrators may modify it.
+& icacls.exe $root /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
+if ($LASTEXITCODE -ne 0) { throw 'Recovery directory ACL failed' }
+$backup = Join-Path $root ('backups\' + [Guid]::NewGuid().ToString('N'))
+New-Item -ItemType Directory -Path $backup -Force | Out-Null
+$previousTask = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+if ($previousTask) { Export-ScheduledTask -TaskName $taskName | Set-Content (Join-Path $backup 'task.xml') -Encoding UTF8 }
+if (Test-Path $guardPath) { Copy-Item $guardPath (Join-Path $backup 'guard.ps1') }
+try {
+    [IO.File]::WriteAllText($guardPath, $guard, [Text.UTF8Encoding]::new($false))
+    $exe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe'
+    $run = New-ScheduledTaskAction -Execute $exe -Argument ('-NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "' + $guardPath + '"')
+    $triggers = @((New-ScheduledTaskTrigger -AtStartup), (New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -RepetitionInterval (New-TimeSpan -Minutes 1)))
+    $settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Seconds 45) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
+    $identity = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest
+    Register-ScheduledTask -TaskName $taskName -Action $run -Trigger $triggers -Settings $settings -Principal $identity -Force | Out-Null
+    $receipt = [ordered]@{node=$env:COMPUTERNAME;task=$taskName;installedHash=(Get-FileHash $guardPath -Algorithm SHA256).Hash;backup=$backup}
+    $receipt|ConvertTo-Json|Set-Content (Join-Path $backup 'receipt.json') -Encoding UTF8
+    Start-ScheduledTask -TaskName $taskName
+    $receipt|ConvertTo-Json -Compress
+} catch {
+    if ($previousTask) { Register-ScheduledTask -TaskName $taskName -Xml (Get-Content (Join-Path $backup 'task.xml') -Raw) -Force | Out-Null }
+    else { Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue }
+    if (Test-Path (Join-Path $backup 'guard.ps1')) { Copy-Item (Join-Path $backup 'guard.ps1') $guardPath -Force }
+    else { Remove-Item $guardPath -ErrorAction SilentlyContinue }
+    throw
+}