fix(telemetry): recover worker agent and Tailscale transport automatically
This commit is contained in:
DCCONSTRUCTIONS
@@ -33,6 +33,10 @@ foreach ($name in @(
|
||||
Set-Item -Path "Env:$name" -Value ([string]$value)
|
||||
}
|
||||
|
||||
$recoveryInstaller = Join-Path $PSScriptRoot 'Install-NdcMissionCoreTelemetryRecovery.ps1'
|
||||
if ($payload.MISSIONCORE_MQTT_HOST -eq '127.0.0.1' -and -not (Test-Path $recoveryInstaller -PathType Leaf)) {
|
||||
throw 'Telemetry recovery installer missing from bundle'
|
||||
}
|
||||
if (Get-Service -Name $serviceName -ErrorAction SilentlyContinue) {
|
||||
throw "Service '$serviceName' already exists; refusing an implicit replacement"
|
||||
}
|
||||
@@ -137,6 +141,10 @@ try {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to enable Telegraf recovery for non-crash failures"
|
||||
}
|
||||
$recoveryTask = $null
|
||||
if ($payload.MISSIONCORE_MQTT_HOST -eq '127.0.0.1') {
|
||||
$recoveryTask = & $recoveryInstaller -ExpectedNodeId $env:COMPUTERNAME -Action Apply | ConvertFrom-Json
|
||||
}
|
||||
Start-Service -Name $serviceName
|
||||
$service = Get-Service -Name $serviceName
|
||||
$service.WaitForStatus([ServiceProcess.ServiceControllerStatus]::Running, [TimeSpan]::FromSeconds(20))
|
||||
@@ -150,6 +158,7 @@ try {
|
||||
Status = $service.Status.ToString()
|
||||
StartType = $service.StartType.ToString()
|
||||
RecoveryConfigured = $true
|
||||
RecoveryTask = $recoveryTask
|
||||
Configuration = $configurationPath
|
||||
PipelineCollector = $collectorPath
|
||||
PipelineJournal = $PipelineJournal
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$ExpectedNodeId,
|
||||
[ValidateSet('Plan','Apply','Rollback')][string]$Action = 'Plan',
|
||||
[string]$BackupDirectory
|
||||
)
|
||||
# Telegraf 1.38.4 can report MQTT startup failure as a clean Windows-service exit.
|
||||
# SCM failure actions alone cannot recover that stopped service.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
if ($env:COMPUTERNAME -cne $ExpectedNodeId) { throw 'Worker identity mismatch' }
|
||||
$taskName = 'ndc-mission-core-telemetry-recovery'
|
||||
$root = Join-Path $env:ProgramFiles 'NDC\Mission Core\TelemetryRecovery'
|
||||
$guardPath = Join-Path $root 'Resume-Telemetry.ps1'
|
||||
$service = Get-CimInstance Win32_Service -Filter "Name='telegraf'"
|
||||
if (-not $service -or $service.PathName -notlike '*NDC\Mission Core\Telegraf\telegraf.exe*') { throw 'Managed Telegraf service required' }
|
||||
$guard = @'
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$service = Get-Service -Name 'telegraf'
|
||||
if ($service.Status -eq 'Running') { exit 0 }
|
||||
if ($service.Status -ne 'Stopped') { exit 0 }
|
||||
$values = @{}
|
||||
foreach ($entry in (Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Services\telegraf').Environment) {
|
||||
$name, $value = $entry -split '=', 2
|
||||
if ($name -in @('MISSIONCORE_MQTT_HOST','MISSIONCORE_MQTT_PORT')) { $values[$name] = $value }
|
||||
}
|
||||
if ($values['MISSIONCORE_MQTT_HOST'] -ne '127.0.0.1' -or $values['MISSIONCORE_MQTT_PORT'] -ne '1883') { exit 1 }
|
||||
$client = [Net.Sockets.TcpClient]::new()
|
||||
try {
|
||||
$connect = $client.ConnectAsync('127.0.0.1', 1883)
|
||||
if (-not $connect.Wait(3000) -or -not $client.Connected) { exit 0 }
|
||||
} catch { exit 0 } finally { $client.Dispose() }
|
||||
Start-Service -Name 'telegraf'
|
||||
(Get-Service -Name 'telegraf').WaitForStatus([ServiceProcess.ServiceControllerStatus]::Running, [TimeSpan]::FromSeconds(15))
|
||||
'@
|
||||
if ($Action -eq 'Plan') {
|
||||
[ordered]@{node=$env:COMPUTERNAME;task=$taskName;path=$guardPath;startMode=$service.StartMode;triggers=@('boot','every-minute');scope='Start stopped telemetry service only';requiresInteractiveLogin=$false}|ConvertTo-Json -Compress
|
||||
exit 0
|
||||
}
|
||||
if ($Action -eq 'Rollback') {
|
||||
if (-not $BackupDirectory) { throw 'BackupDirectory required' }
|
||||
$receipt = Get-Content (Join-Path $BackupDirectory 'receipt.json') -Raw | ConvertFrom-Json
|
||||
if ($receipt.node -cne $ExpectedNodeId -or $receipt.task -ne $taskName) { throw 'Backup identity mismatch' }
|
||||
if ((Get-FileHash $guardPath -Algorithm SHA256).Hash -ne $receipt.installedHash) { throw 'Installed recovery script changed' }
|
||||
Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue
|
||||
if (Test-Path (Join-Path $BackupDirectory 'task.xml')) {
|
||||
Register-ScheduledTask -TaskName $taskName -Xml (Get-Content (Join-Path $BackupDirectory 'task.xml') -Raw) -Force | Out-Null
|
||||
}
|
||||
if (Test-Path (Join-Path $BackupDirectory 'guard.ps1')) {
|
||||
Copy-Item (Join-Path $BackupDirectory 'guard.ps1') $guardPath -Force
|
||||
} else { Remove-Item $guardPath }
|
||||
[ordered]@{restored=$true;task=$taskName}|ConvertTo-Json -Compress
|
||||
exit 0
|
||||
}
|
||||
$principal = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()
|
||||
if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { throw 'Administrative installer required' }
|
||||
if (Test-Path $root) {
|
||||
if ((Get-Item $root -Force).Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'Recovery directory must not be a reparse point' }
|
||||
}
|
||||
New-Item -ItemType Directory -Path $root -Force | Out-Null
|
||||
# SYSTEM executes this file: only SYSTEM and Administrators may modify it.
|
||||
& icacls.exe $root /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Recovery directory ACL failed' }
|
||||
$backup = Join-Path $root ('backups\' + [Guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Path $backup -Force | Out-Null
|
||||
$previousTask = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
||||
if ($previousTask) { Export-ScheduledTask -TaskName $taskName | Set-Content (Join-Path $backup 'task.xml') -Encoding UTF8 }
|
||||
if (Test-Path $guardPath) { Copy-Item $guardPath (Join-Path $backup 'guard.ps1') }
|
||||
try {
|
||||
[IO.File]::WriteAllText($guardPath, $guard, [Text.UTF8Encoding]::new($false))
|
||||
$exe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe'
|
||||
$run = New-ScheduledTaskAction -Execute $exe -Argument ('-NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "' + $guardPath + '"')
|
||||
$triggers = @((New-ScheduledTaskTrigger -AtStartup), (New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -RepetitionInterval (New-TimeSpan -Minutes 1)))
|
||||
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Seconds 45) -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
|
||||
$identity = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest
|
||||
Register-ScheduledTask -TaskName $taskName -Action $run -Trigger $triggers -Settings $settings -Principal $identity -Force | Out-Null
|
||||
$receipt = [ordered]@{node=$env:COMPUTERNAME;task=$taskName;installedHash=(Get-FileHash $guardPath -Algorithm SHA256).Hash;backup=$backup}
|
||||
$receipt|ConvertTo-Json|Set-Content (Join-Path $backup 'receipt.json') -Encoding UTF8
|
||||
Start-ScheduledTask -TaskName $taskName
|
||||
$receipt|ConvertTo-Json -Compress
|
||||
} catch {
|
||||
if ($previousTask) { Register-ScheduledTask -TaskName $taskName -Xml (Get-Content (Join-Path $backup 'task.xml') -Raw) -Force | Out-Null }
|
||||
else { Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue }
|
||||
if (Test-Path (Join-Path $backup 'guard.ps1')) { Copy-Item (Join-Path $backup 'guard.ps1') $guardPath -Force }
|
||||
else { Remove-Item $guardPath -ErrorAction SilentlyContinue }
|
||||
throw
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
[CmdletBinding()]
|
||||
param([Parameter(Mandatory=$true)][string]$ExpectedNodeId)
|
||||
$ErrorActionPreference='Stop'
|
||||
$ProgressPreference='SilentlyContinue'
|
||||
if ($env:COMPUTERNAME -cne $ExpectedNodeId) { throw 'Worker identity mismatch' }
|
||||
$task=Get-ScheduledTask -TaskName 'ndc-mission-core-telemetry-recovery'
|
||||
$account=[Security.Principal.NTAccount]::new($task.Principal.UserId)
|
||||
$sid=$account.Translate([Security.Principal.SecurityIdentifier]).Value
|
||||
if (-not $task.Settings.Enabled -or $sid -ne 'S-1-5-18') { throw 'Recovery task not enabled as SYSTEM' }
|
||||
$before=Get-CimInstance Win32_Service -Filter "Name='telegraf'"
|
||||
if ($before.State -ne 'Running') { throw 'A running baseline is required' }
|
||||
$started=[DateTime]::UtcNow
|
||||
Stop-Service telegraf
|
||||
(Get-Service telegraf).WaitForStatus([ServiceProcess.ServiceControllerStatus]::Stopped,[TimeSpan]::FromSeconds(20))
|
||||
$automatic=$false
|
||||
try {
|
||||
while (([DateTime]::UtcNow-$started).TotalSeconds -lt 80) {
|
||||
Start-Sleep -Seconds 2
|
||||
$after=Get-CimInstance Win32_Service -Filter "Name='telegraf'"
|
||||
if ($after.State -eq 'Running' -and $after.ProcessId -ne $before.ProcessId) { $automatic=$true;break }
|
||||
}
|
||||
[ordered]@{node=$ExpectedNodeId;test='clean-service-stop';automaticRecovery=$automatic;elapsedSeconds=[Math]::Round(([DateTime]::UtcNow-$started).TotalSeconds,2);previousPid=$before.ProcessId;newPid=$after.ProcessId;state=$after.State}|ConvertTo-Json -Compress
|
||||
if (-not $automatic) { throw 'Automatic recovery failed; restoring baseline' }
|
||||
} finally {
|
||||
if ((Get-Service telegraf).Status -ne 'Running') { Start-Service telegraf }
|
||||
}
|
||||
@@ -67,6 +67,10 @@ if (-not ($serviceEnvironmentCandidate | Where-Object {
|
||||
)
|
||||
}
|
||||
|
||||
$recoveryInstaller = Join-Path $PSScriptRoot 'Install-NdcMissionCoreTelemetryRecovery.ps1'
|
||||
if ($env:MISSIONCORE_MQTT_HOST -eq '127.0.0.1' -and -not (Test-Path $recoveryInstaller -PathType Leaf)) {
|
||||
throw 'Telemetry recovery installer missing from bundle'
|
||||
}
|
||||
$temporaryRoot = Join-Path $env:TEMP "ndc-mission-core-telegraf-update-$([Guid]::NewGuid().ToString('N'))"
|
||||
$validationOutput = Join-Path $temporaryRoot "validation.out.log"
|
||||
$validationError = Join-Path $temporaryRoot "validation.error.log"
|
||||
@@ -127,6 +131,10 @@ try {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to enable Telegraf recovery for non-crash failures"
|
||||
}
|
||||
$recoveryTask = $null
|
||||
if ($env:MISSIONCORE_MQTT_HOST -eq '127.0.0.1') {
|
||||
$recoveryTask = & $recoveryInstaller -ExpectedNodeId $env:COMPUTERNAME -Action Apply | ConvertFrom-Json
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Stop-Service -Name $serviceName -Force -ErrorAction SilentlyContinue
|
||||
@@ -155,6 +163,7 @@ try {
|
||||
ServiceName = $serviceName
|
||||
Status = (Get-Service -Name $serviceName).Status.ToString()
|
||||
RecoveryConfigured = $true
|
||||
RecoveryTask = $recoveryTask
|
||||
Configuration = $configurationPath
|
||||
Backup = $backupPath
|
||||
PipelineCollector = $collectorPath
|
||||
|
||||
Reference in New Issue
Block a user