feat(telemetry): add portable local telemetry plane
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,120 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Version = "1.38.4",
|
||||
[string]$ExpectedSha256 = "6c7878ec319471ac85b82443baec2f3fa5dbcf1b6e2da5d5cd2cbb60fff2bb45",
|
||||
[string]$ConfigurationTemplate = "$PSScriptRoot\mission-core-windows.conf.tmpl"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$serviceName = "telegraf"
|
||||
$installRoot = "C:\Program Files\NDC\Mission Core\Telegraf"
|
||||
$configurationRoot = "C:\ProgramData\NDC\MissionCore\telemetry-agent"
|
||||
$configurationPath = Join-Path $configurationRoot "telegraf.conf"
|
||||
$archiveUrl = "https://dl.influxdata.com/telegraf/releases/telegraf-$($Version)_windows_amd64.zip"
|
||||
$payload = [Console]::In.ReadToEnd() | ConvertFrom-Json
|
||||
|
||||
foreach ($name in @(
|
||||
"MISSIONCORE_CONTOUR_ID",
|
||||
"MISSIONCORE_AGENT_ID",
|
||||
"MISSIONCORE_NODE_ID",
|
||||
"MISSIONCORE_MQTT_HOST",
|
||||
"MISSIONCORE_MQTT_PORT",
|
||||
"MISSIONCORE_MQTT_USERNAME",
|
||||
"MISSIONCORE_MQTT_PASSWORD"
|
||||
)) {
|
||||
$value = $payload.$name
|
||||
if (-not $value) {
|
||||
throw "Provisioning payload is missing $name"
|
||||
}
|
||||
Set-Item -Path "Env:$name" -Value ([string]$value)
|
||||
}
|
||||
|
||||
if (Get-Service -Name $serviceName -ErrorAction SilentlyContinue) {
|
||||
throw "Service '$serviceName' already exists; refusing an implicit replacement"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $ConfigurationTemplate -PathType Leaf)) {
|
||||
throw "Configuration template not found: $ConfigurationTemplate"
|
||||
}
|
||||
|
||||
$temporaryRoot = Join-Path $env:TEMP "ndc-mission-core-telegraf-$([Guid]::NewGuid().ToString('N'))"
|
||||
$archivePath = Join-Path $temporaryRoot "telegraf.zip"
|
||||
$expandedRoot = Join-Path $temporaryRoot "expanded"
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $temporaryRoot, $expandedRoot -Force | Out-Null
|
||||
Invoke-WebRequest -UseBasicParsing -Uri $archiveUrl -OutFile $archivePath
|
||||
$actualSha256 = (Get-FileHash -LiteralPath $archivePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($actualSha256 -ne $ExpectedSha256.ToLowerInvariant()) {
|
||||
throw "Telegraf archive SHA256 mismatch"
|
||||
}
|
||||
Expand-Archive -LiteralPath $archivePath -DestinationPath $expandedRoot
|
||||
$sourceExecutable = Get-ChildItem -Path $expandedRoot -Filter "telegraf.exe" -Recurse |
|
||||
Select-Object -First 1
|
||||
if (-not $sourceExecutable) {
|
||||
throw "telegraf.exe is missing from the verified archive"
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Path $installRoot, $configurationRoot -Force | Out-Null
|
||||
Copy-Item -LiteralPath $sourceExecutable.FullName -Destination (Join-Path $installRoot "telegraf.exe")
|
||||
Copy-Item -LiteralPath $ConfigurationTemplate -Destination $configurationPath
|
||||
& icacls.exe $configurationRoot /inheritance:r /grant:r `
|
||||
"*S-1-5-18:(OI)(CI)F" "*S-1-5-32-544:(OI)(CI)F" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to restrict the telemetry agent configuration directory"
|
||||
}
|
||||
|
||||
$executable = Join-Path $installRoot "telegraf.exe"
|
||||
$validationOutput = Join-Path $temporaryRoot "validation.out.log"
|
||||
$validationError = Join-Path $temporaryRoot "validation.error.log"
|
||||
$validation = Start-Process -FilePath $executable `
|
||||
-ArgumentList @("--config", $configurationPath, "--test") `
|
||||
-NoNewWindow -Wait -PassThru `
|
||||
-RedirectStandardOutput $validationOutput `
|
||||
-RedirectStandardError $validationError
|
||||
if ($validation.ExitCode -ne 0) {
|
||||
$validationDetail = Get-Content -LiteralPath $validationError -Tail 8 |
|
||||
Out-String
|
||||
throw "Telegraf configuration validation failed: $validationDetail"
|
||||
}
|
||||
|
||||
& $executable --service install --config $configurationPath
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Telegraf service installation failed"
|
||||
}
|
||||
$serviceEnvironment = @(
|
||||
"MISSIONCORE_CONTOUR_ID=$($payload.MISSIONCORE_CONTOUR_ID)",
|
||||
"MISSIONCORE_AGENT_ID=$($payload.MISSIONCORE_AGENT_ID)",
|
||||
"MISSIONCORE_NODE_ID=$($payload.MISSIONCORE_NODE_ID)",
|
||||
"MISSIONCORE_MQTT_HOST=$($payload.MISSIONCORE_MQTT_HOST)",
|
||||
"MISSIONCORE_MQTT_PORT=$($payload.MISSIONCORE_MQTT_PORT)",
|
||||
"MISSIONCORE_MQTT_USERNAME=$($payload.MISSIONCORE_MQTT_USERNAME)",
|
||||
"MISSIONCORE_MQTT_PASSWORD=$($payload.MISSIONCORE_MQTT_PASSWORD)"
|
||||
)
|
||||
$serviceRegistryPath = "HKLM:\SYSTEM\CurrentControlSet\Services\$serviceName"
|
||||
Set-ItemProperty -Path $serviceRegistryPath -Name Environment `
|
||||
-Type MultiString -Value $serviceEnvironment
|
||||
Set-ItemProperty -Path $serviceRegistryPath -Name DisplayName `
|
||||
-Value "NDC Mission Core Telemetry Agent"
|
||||
& sc.exe config $serviceName DisplayName= "NDC Mission Core Telemetry Agent" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to apply the NDC service display name"
|
||||
}
|
||||
Set-Service -Name $serviceName -StartupType Automatic
|
||||
Start-Service -Name $serviceName
|
||||
$service = Get-Service -Name $serviceName
|
||||
$service.WaitForStatus([ServiceProcess.ServiceControllerStatus]::Running, [TimeSpan]::FromSeconds(20))
|
||||
|
||||
[ordered]@{
|
||||
SchemaVersion = "missioncore.telemetry-agent-install-result/v1"
|
||||
Agent = "Telegraf"
|
||||
Version = $Version
|
||||
ServiceName = $service.Name
|
||||
DisplayName = $service.DisplayName
|
||||
Status = $service.Status.ToString()
|
||||
StartType = $service.StartType.ToString()
|
||||
Configuration = $configurationPath
|
||||
Sha256 = $actualSha256
|
||||
} | ConvertTo-Json -Compress
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $temporaryRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ConfigurationTemplate
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$serviceName = "telegraf"
|
||||
$installRoot = "C:\Program Files\NDC\Mission Core\Telegraf"
|
||||
$configurationRoot = "C:\ProgramData\NDC\MissionCore\telemetry-agent"
|
||||
$configurationPath = Join-Path $configurationRoot "telegraf.conf"
|
||||
$executable = Join-Path $installRoot "telegraf.exe"
|
||||
$serviceRegistryPath = "HKLM:\SYSTEM\CurrentControlSet\Services\$serviceName"
|
||||
|
||||
$service = Get-Service -Name $serviceName -ErrorAction Stop
|
||||
if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) {
|
||||
throw "NDC Mission Core Telegraf executable is missing"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $ConfigurationTemplate -PathType Leaf)) {
|
||||
throw "Configuration template not found: $ConfigurationTemplate"
|
||||
}
|
||||
|
||||
foreach ($entry in @((Get-ItemProperty -Path $serviceRegistryPath).Environment)) {
|
||||
$name, $value = $entry -split "=", 2
|
||||
if ($name -and $value) {
|
||||
Set-Item -Path "Env:$name" -Value $value
|
||||
}
|
||||
}
|
||||
|
||||
$temporaryRoot = Join-Path $env:TEMP "ndc-mission-core-telegraf-update-$([Guid]::NewGuid().ToString('N'))"
|
||||
$validationOutput = Join-Path $temporaryRoot "validation.out.log"
|
||||
$validationError = Join-Path $temporaryRoot "validation.error.log"
|
||||
$backupRoot = Join-Path $configurationRoot "backups"
|
||||
$backupPath = Join-Path $backupRoot "telegraf-$([DateTime]::UtcNow.ToString('yyyyMMddTHHmmssZ')).conf"
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $temporaryRoot, $backupRoot -Force | Out-Null
|
||||
$validation = Start-Process -FilePath $executable `
|
||||
-ArgumentList @("--config", $ConfigurationTemplate, "--test") `
|
||||
-NoNewWindow -Wait -PassThru `
|
||||
-RedirectStandardOutput $validationOutput `
|
||||
-RedirectStandardError $validationError
|
||||
if ($validation.ExitCode -ne 0) {
|
||||
$validationDetail = Get-Content -LiteralPath $validationError -Tail 8 |
|
||||
Out-String
|
||||
throw "Telegraf configuration validation failed: $validationDetail"
|
||||
}
|
||||
|
||||
Copy-Item -LiteralPath $configurationPath -Destination $backupPath
|
||||
Stop-Service -Name $serviceName
|
||||
try {
|
||||
Copy-Item -LiteralPath $ConfigurationTemplate -Destination $configurationPath
|
||||
Start-Service -Name $serviceName
|
||||
$service = Get-Service -Name $serviceName
|
||||
$service.WaitForStatus(
|
||||
[ServiceProcess.ServiceControllerStatus]::Running,
|
||||
[TimeSpan]::FromSeconds(20)
|
||||
)
|
||||
}
|
||||
catch {
|
||||
Copy-Item -LiteralPath $backupPath -Destination $configurationPath
|
||||
Start-Service -Name $serviceName
|
||||
throw
|
||||
}
|
||||
|
||||
[ordered]@{
|
||||
SchemaVersion = "missioncore.telemetry-agent-config-result/v1"
|
||||
ServiceName = $serviceName
|
||||
Status = (Get-Service -Name $serviceName).Status.ToString()
|
||||
Configuration = $configurationPath
|
||||
Backup = $backupPath
|
||||
} | ConvertTo-Json -Compress
|
||||
}
|
||||
finally {
|
||||
Remove-Item -LiteralPath $temporaryRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
[agent]
|
||||
interval = "2s"
|
||||
round_interval = true
|
||||
omit_hostname = false
|
||||
|
||||
[global_tags]
|
||||
node_id = "${MISSIONCORE_NODE_ID}"
|
||||
contour_id = "${MISSIONCORE_CONTOUR_ID}"
|
||||
agent_id = "${MISSIONCORE_AGENT_ID}"
|
||||
|
||||
[[inputs.cpu]]
|
||||
percpu = false
|
||||
totalcpu = true
|
||||
report_active = true
|
||||
|
||||
[[inputs.mem]]
|
||||
[[inputs.disk]]
|
||||
ignore_fs = ["tmpfs", "devtmpfs", "devfs"]
|
||||
[[inputs.net]]
|
||||
[[inputs.system]]
|
||||
[[inputs.nvidia_smi]]
|
||||
|
||||
[[inputs.docker]]
|
||||
endpoint = "unix:///var/run/docker.sock"
|
||||
container_name_include = []
|
||||
total = true
|
||||
|
||||
[[outputs.mqtt]]
|
||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||
password = "${MISSIONCORE_MQTT_PASSWORD}"
|
||||
qos = 1
|
||||
data_format = "json"
|
||||
json_timestamp_units = "1s"
|
||||
@@ -0,0 +1,57 @@
|
||||
[agent]
|
||||
interval = "2s"
|
||||
round_interval = true
|
||||
omit_hostname = false
|
||||
|
||||
[global_tags]
|
||||
node_id = "${MISSIONCORE_NODE_ID}"
|
||||
contour_id = "${MISSIONCORE_CONTOUR_ID}"
|
||||
agent_id = "${MISSIONCORE_AGENT_ID}"
|
||||
|
||||
[[inputs.cpu]]
|
||||
percpu = false
|
||||
totalcpu = true
|
||||
report_active = true
|
||||
|
||||
[[inputs.mem]]
|
||||
[[inputs.system]]
|
||||
[[inputs.disk]]
|
||||
ignore_fs = ["tmpfs", "devtmpfs", "devfs"]
|
||||
[[inputs.net]]
|
||||
[[inputs.win_perf_counters]]
|
||||
[[inputs.win_perf_counters.object]]
|
||||
ObjectName = "System"
|
||||
Counters = ["System Up Time"]
|
||||
Instances = ["------"]
|
||||
Measurement = "win_system"
|
||||
|
||||
[[inputs.nvidia_smi]]
|
||||
|
||||
[[inputs.docker]]
|
||||
endpoint = "npipe:////./pipe/docker_engine"
|
||||
container_name_include = []
|
||||
|
||||
[[inputs.http_response]]
|
||||
urls = ["http://127.0.0.1:8000/v2/health/ready"]
|
||||
response_timeout = "2s"
|
||||
response_status_code = 200
|
||||
name_override = "missioncore_triton_health"
|
||||
|
||||
[[inputs.prometheus]]
|
||||
urls = ["http://127.0.0.1:8002/metrics"]
|
||||
metric_version = 1
|
||||
response_timeout = "2s"
|
||||
namepass = [
|
||||
"nv_inference_request_success",
|
||||
"nv_inference_request_failure",
|
||||
"nv_inference_count",
|
||||
]
|
||||
|
||||
[[outputs.mqtt]]
|
||||
servers = ["tcp://${MISSIONCORE_MQTT_HOST}:${MISSIONCORE_MQTT_PORT}"]
|
||||
topic = "mission-core/v1/contours/${MISSIONCORE_CONTOUR_ID}/agents/${MISSIONCORE_AGENT_ID}/host"
|
||||
username = "${MISSIONCORE_MQTT_USERNAME}"
|
||||
password = "${MISSIONCORE_MQTT_PASSWORD}"
|
||||
qos = 1
|
||||
data_format = "json"
|
||||
json_timestamp_units = "1s"
|
||||
Reference in New Issue
Block a user