feat(telemetry): add canonical VPS host monitoring

2026-08-22 17:29:30 +03:00
parent a3e7a015de
commit 23ea96fbb2
33 changed files with 2541 additions and 51 deletions
@@ -0,0 +1,52 @@
+[agent]
+  interval = "2s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "2s"
+  flush_jitter = "0s"
+  precision = "1s"
+  debug = false
+  quiet = false
+  hostname = "koffyvngij"
+  omit_hostname = false
+
+[global_tags]
+  nodedc_host_key = "robot2b-b2-edge-vps"
+  nodedc_profile = "linux-host-telegraf-v1"
+
+[[inputs.cpu]]
+  percpu = false
+  totalcpu = true
+  collect_cpu_time = false
+  report_active = true
+
+[[inputs.mem]]
+
+[[inputs.swap]]
+
+[[inputs.system]]
+
+[[inputs.processes]]
+
+[[inputs.disk]]
+  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs", "nsfs"]
+
+[[inputs.diskio]]
+
+[[inputs.net]]
+  interfaces = ["*"]
+  ignore_protocol_stats = true
+
+[[inputs.systemd_units]]
+  pattern = "nodedc-*.service ssh.service systemd-networkd.service"
+  details = true
+
+[[outputs.http]]
+  url = "http://127.0.0.1:18223/internal/v1/host-telemetry"
+  method = "POST"
+  timeout = "10s"
+  data_format = "json"
+  use_batch_format = true
+  content_encoding = "identity"
@@ -13,6 +13,7 @@ import {
 import {
  createDeviceGatewayRuntime,
 } from "../../services/device-gateway/src/runtime.mjs";
+import { createHostTelemetryCollector } from "./host-telemetry-runtime.mjs";

 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  await main();
@@ -51,16 +52,26 @@ export async function main(environment = process.env) {
    onMessage: (message) => channel.submitAdapterMessage(message),
    onCommandStatus: (status) => channel.submitCommandStatus(status),
  });
-  const health = createCombinedHealthServer(channel, gateway, base.health);
+  const telemetry = createHostTelemetryCollector({
+    submit: (snapshot) => channel.submitHostTelemetry(snapshot),
+    hostKey: environment.NODEDC_INFRASTRUCTURE_HOST_KEY
+      ?? "robot2b-b2-edge-vps",
+    host: environment.NODEDC_HOST_TELEMETRY_HOST ?? "127.0.0.1",
+    port: environment.NODEDC_HOST_TELEMETRY_PORT ?? 18223,
+    agentVersion: environment.NODEDC_HOST_TELEMETRY_AGENT_VERSION ?? "1.38.4",
+  });
+  const health = createCombinedHealthServer(channel, gateway, telemetry, base.health);
  let stopping = false;

  try {
    await channel.start();
    await gateway.start();
+    await telemetry.start();
    await listen(health, base.health.port, base.health.host);
  } catch (error) {
    await Promise.allSettled([
      gateway.stop(),
+      telemetry.stop(),
      channel.stop(),
      closeServer(health),
    ]);
@@ -72,6 +83,7 @@ export async function main(environment = process.env) {
    channel: `${base.channel.host}:${base.channel.port}`,
    health: `${base.health.host}:${base.health.port}`,
    trackerIngress: `${tracker.tcpHost}:${tracker.tcpPort}`,
+    hostTelemetry: `${telemetry.status().host}:${telemetry.status().port}`,
    adapterProfile: tracker.protocolProfileRef,
    edgeRegistrationId: base.channel.edgeRegistrationId,
    channelGeneration: base.channel.channelGeneration,
@@ -87,6 +99,7 @@ export async function main(environment = process.env) {
    stopping = true;
    await Promise.allSettled([
      gateway.stop(),
+      telemetry.stop(),
      channel.stop(),
      closeServer(health),
    ]);
@@ -161,7 +174,7 @@ export function normalizeTrackerIngressConfiguration(environment = {}, edgeRef)
  });
 }

-function createCombinedHealthServer(channel, gateway, healthConfig) {
+function createCombinedHealthServer(channel, gateway, telemetry, healthConfig) {
  return createServer((request, response) => {
    response.setHeader("Content-Type", "application/json; charset=utf-8");
    response.setHeader("Cache-Control", "no-store");
@@ -179,6 +192,7 @@ function createCombinedHealthServer(channel, gateway, healthConfig) {
      ...channel.status(),
      trackerIngress: "telemetry-ingest",
      tracker: gateway.status(),
+      hostTelemetry: telemetry.status(),
      commandTransport: "typed-service-ping-v1",
    }));
  });
@@ -0,0 +1,161 @@
+import { createServer } from "node:http";
+import { arch, cpus, hostname, platform, release } from "node:os";
+
+import {
+  telegrafBatchToHostTelemetry,
+} from "../../packages/infrastructure-telemetry-contract/src/index.mjs";
+
+const MAX_BODY_BYTES = 512 * 1024;
+
+export function createHostTelemetryCollector(options = {}) {
+  const config = normalizeConfig(options);
+  let accepted = 0;
+  let rejected = 0;
+  let lastObservedAt = null;
+  let lastAcceptedAt = null;
+  let lastErrorCode = null;
+  let pending = false;
+
+  const server = createServer(async (request, response) => {
+    response.setHeader("Content-Type", "application/json; charset=utf-8");
+    response.setHeader("Cache-Control", "no-store");
+    response.setHeader("X-Content-Type-Options", "nosniff");
+    if (request.method !== "POST" || request.url !== "/internal/v1/host-telemetry") {
+      response.statusCode = 404;
+      response.end(JSON.stringify({ ok: false, error: "not_found" }));
+      return;
+    }
+    if (pending) {
+      rejected += 1;
+      response.statusCode = 429;
+      response.end(JSON.stringify({ ok: false, error: "host_telemetry_busy" }));
+      return;
+    }
+    pending = true;
+    try {
+      const body = await readJsonBody(request, MAX_BODY_BYTES);
+      const processors = cpus();
+      const snapshot = telegrafBatchToHostTelemetry(body, {
+        hostKey: config.hostKey,
+        hostname: hostname(),
+        architecture: arch(),
+        platform: platform(),
+        kernelRelease: release(),
+        cpuModel: processors[0]?.model ?? null,
+        logicalProcessors: processors.length || null,
+        agentVersion: config.agentVersion,
+      });
+      lastObservedAt = snapshot.observedAt;
+      await config.submit(snapshot);
+      accepted += 1;
+      lastAcceptedAt = new Date().toISOString();
+      lastErrorCode = null;
+      response.statusCode = 202;
+      response.end(JSON.stringify({ ok: true, accepted: true }));
+    } catch (error) {
+      rejected += 1;
+      lastErrorCode = safeErrorCode(error);
+      response.statusCode = lastErrorCode.includes("too_large") ? 413 : 503;
+      response.end(JSON.stringify({ ok: false, error: lastErrorCode }));
+    } finally {
+      pending = false;
+    }
+  });
+
+  return Object.freeze({
+    async start() {
+      await listen(server, config.port, config.host);
+      return server.address();
+    },
+    async stop() {
+      await closeServer(server);
+    },
+    status() {
+      return Object.freeze({
+        listening: server.listening,
+        host: config.host,
+        port: config.port,
+        profile: "linux-host-telegraf-v1",
+        accepted,
+        rejected,
+        pending,
+        lastObservedAt,
+        lastAcceptedAt,
+        lastErrorCode,
+      });
+    },
+  });
+}
+
+function normalizeConfig(options) {
+  if (typeof options.submit !== "function") {
+    throw new TypeError("host_telemetry_submit_required");
+  }
+  return Object.freeze({
+    submit: options.submit,
+    hostKey: normalizeRef(options.hostKey, "host_telemetry_host_key_invalid"),
+    agentVersion: String(options.agentVersion ?? "1.38.4"),
+    host: normalizeLoopbackHost(options.host ?? "127.0.0.1"),
+    port: normalizePort(options.port ?? 18223),
+  });
+}
+
+async function readJsonBody(request, maximumBytes) {
+  const chunks = [];
+  let total = 0;
+  for await (const chunk of request) {
+    total += chunk.length;
+    if (total > maximumBytes) throw new Error("host_telemetry_body_too_large");
+    chunks.push(chunk);
+  }
+  if (total < 2) throw new Error("host_telemetry_body_invalid");
+  try {
+    return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+  } catch {
+    throw new Error("host_telemetry_body_invalid");
+  }
+}
+
+function normalizeRef(value, errorCode) {
+  if (typeof value !== "string" || !/^[a-z][a-z0-9-]{1,62}$/.test(value)) {
+    throw new TypeError(errorCode);
+  }
+  return value;
+}
+
+function normalizeLoopbackHost(value) {
+  if (!["127.0.0.1", "::1"].includes(value)) {
+    throw new TypeError("host_telemetry_host_invalid");
+  }
+  return value;
+}
+
+function normalizePort(value) {
+  const normalized = Number(value);
+  if (!Number.isSafeInteger(normalized) || normalized < 0 || normalized > 65_535) {
+    throw new TypeError("host_telemetry_port_invalid");
+  }
+  return normalized;
+}
+
+function safeErrorCode(error) {
+  const value = String(error?.message || "host_telemetry_internal_error");
+  return /^[a-z0-9_.:-]{3,160}$/.test(value)
+    ? value
+    : "host_telemetry_internal_error";
+}
+
+function listen(server, port, host) {
+  return new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, host, () => {
+      server.off("error", reject);
+      resolve();
+    });
+  });
+}
+
+function closeServer(server) {
+  if (!server.listening) return Promise.resolve();
+  return new Promise((resolve) => server.close(() => resolve()));
+}
@@ -0,0 +1,37 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { createHostTelemetryCollector } from "./host-telemetry-runtime.mjs";
+
+test("accepts Telegraf only on the bounded local collector", async () => {
+  let submitted = null;
+  const collector = createHostTelemetryCollector({
+    hostKey: "robot2b-b2-edge-vps",
+    host: "127.0.0.1",
+    port: 0,
+    submit: async (value) => { submitted = value; },
+  });
+  const address = await collector.start();
+  try {
+    const response = await fetch(`http://127.0.0.1:${address.port}/internal/v1/host-telemetry`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ metrics: [
+        { name: "cpu", tags: { cpu: "cpu-total", host: "edge" }, fields: { usage_active: 9 }, timestamp: Math.floor(Date.now() / 1000) },
+      ] }),
+    });
+    assert.equal(response.status, 202);
+    assert.equal(submitted.hostKey, "robot2b-b2-edge-vps");
+    assert.equal(collector.status().accepted, 1);
+  } finally {
+    await collector.stop();
+  }
+});
+
+test("rejects non-loopback collector binds", () => {
+  assert.throws(() => createHostTelemetryCollector({
+    hostKey: "host-01",
+    host: "0.0.0.0",
+    submit: async () => {},
+  }), /host_telemetry_host_invalid/);
+});
@@ -28,6 +28,10 @@ Environment=DEVICE_GATEWAY_MAX_SESSIONS_PER_ADDRESS=16
 Environment=DEVICE_GATEWAY_MAX_CONNECTIONS_PER_MINUTE_PER_ADDRESS=60
 Environment=DEVICE_GATEWAY_MAX_TRACKED_SOURCE_ADDRESSES=2048
 Environment=DEVICE_GATEWAY_SESSION_TIMEOUT_MS=10000
+Environment=NODEDC_INFRASTRUCTURE_HOST_KEY=robot2b-b2-edge-vps
+Environment=NODEDC_HOST_TELEMETRY_HOST=127.0.0.1
+Environment=NODEDC_HOST_TELEMETRY_PORT=18223
+Environment=NODEDC_HOST_TELEMETRY_AGENT_VERSION=1.38.4
 Restart=always
 RestartSec=2
 TimeoutStartSec=20
@@ -0,0 +1,47 @@
+[Unit]
+Description=NODE.DC infrastructure host telemetry agent
+Documentation=https://docs.influxdata.com/telegraf/v1/
+After=network-online.target nodedc-device-edge-channel.service
+Wants=network-online.target
+Requires=nodedc-device-edge-channel.service
+
+[Service]
+Type=notify
+NotifyAccess=all
+User=nodedc-telemetry
+Group=nodedc-telemetry
+ExecStart=/opt/nodedc-b2-vps/runtime/telegraf/usr/bin/telegraf --config /opt/nodedc-b2-vps/vps/config/nodedc-host-telemetry-telegraf.conf
+Restart=always
+RestartSec=3
+TimeoutStartSec=30
+TimeoutStopSec=15
+NoNewPrivileges=yes
+PrivateTmp=yes
+PrivateDevices=yes
+ProtectSystem=strict
+ProtectHome=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+RestrictSUIDSGID=yes
+RestrictRealtime=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=no
+SystemCallArchitectures=native
+RestrictAddressFamilies=AF_UNIX AF_INET
+IPAddressDeny=any
+IPAddressAllow=localhost
+CapabilityBoundingSet=
+AmbientCapabilities=
+UMask=0077
+MemoryMax=96M
+MemorySwapMax=0
+CPUQuota=15%
+TasksMax=64
+LimitNOFILE=512
+
+[Install]
+WantedBy=multi-user.target