SILVER/NODEDC_DEVICE_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(telemetry): add canonical VPS host monitoring

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 17:29:30 +03:00
parent a3e7a015de
commit 23ea96fbb2
33 changed files with 2541 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-b2-vps-deploy
+272 -7
View File
@@ -46,6 +46,8 @@ RELAY_USER = "nodedc-relay"
RELAY_GROUP = "nodedc-relay"
CHANNEL_USER = "nodedc-channel"
CHANNEL_GROUP = "nodedc-channel"
TELEMETRY_USER = "nodedc-telemetry"
TELEMETRY_GROUP = "nodedc-telemetry"
TAILSCALE_REQUIRED_TAG = "tag:device-edge-vps"
MANAGEMENT_KEY_FINGERPRINT = (
"SHA256:DYYy1E3DaxIQGC0jnsW6SP7gXdBHUy3A1zn4pvgVUEw"
@@ -67,10 +69,19 @@ TAILSCALE_ARCHIVE_SHA256 = (
NODE_BIN_SHA256 = "3517c2df0b2f8cd7f422b4b8450ef81c6889f08eb03e281d6de9079b15e6a327"
TAILSCALE_BIN_SHA256 = "58b0fa0907677ea6afe0d3022cc3e99b1a03f39a7ed60144843ed38252e00c80"
TAILSCALED_BIN_SHA256 = "5f17b092bac92326325f6c4ffd9991fad3c073975abe412d02ee68721a500394"
TELEGRAF_VERSION = "1.38.4"
TELEGRAF_ARCHIVE = "telegraf-1.38.4_linux_amd64.tar.gz"
TELEGRAF_ARCHIVE_SHA256 = (
"81857e9745ebf26e058b6fdc27b9b2c210fd1fe61e57d7fad3d4bb9131f60041"
)
TELEGRAF_BIN_SHA256 = (
"0643b582546eb9c70d99a9646b3e49e25ccdea4f78ab06fd9096ed71dba1babb"
)
NODE_BIN = LIVE_ROOT / "runtime/node/bin/node"
TAILSCALE_BIN = LIVE_ROOT / "runtime/tailscale/tailscale"
TAILSCALED_BIN = LIVE_ROOT / "runtime/tailscale/tailscaled"
TELEGRAF_BIN = LIVE_ROOT / "runtime/telegraf/usr/bin/telegraf"
TAILSCALE_SOCKET = Path("/run/nodedc-b2-vps/tailscaled.sock")
TAILSCALE_STATE = Path("/var/lib/nodedc-b2-vps/tailscale/tailscaled.state")
TRUST_ROOT = Path("/var/lib/nodedc-b2-vps/trust")
@@ -102,6 +113,10 @@ CHANNEL_CORE_CERTIFICATE = CHANNEL_TRUST_ROOT / "core-certificate.pem"
CHANNEL_RUNTIME_CONFIG = CHANNEL_TRUST_ROOT / "runtime.json"
CHANNEL_HEALTH_PORT = 18222
CHANNEL_PUBLIC_PORT = 443
HOST_TELEMETRY_PORT = 18223
HOST_TELEMETRY_UNIT = Path(
"/etc/systemd/system/nodedc-host-telemetry-agent.service"
)
CORE_CHANNEL_ACCEPTED_PATCH = "device-edge-vps-core-channel-20260812-010"
CORE_CHANNEL_ACCEPTED_SHA256 = (
"c8ef3c4bb45850cad32e881eba081bc4c891c2886e5500d02cb94616d82353f3"
@@ -116,6 +131,12 @@ TRACKER_INGRESS_ACCEPTED_PATCH = "device-edge-vps-tracker-ingress-20260812-012"
TRACKER_INGRESS_ACCEPTED_SHA256 = (
"290acef118839c6b0c31aac864c47da1832a289537366af9322d4624a1dd81ec"
)
COMMAND_TRANSPORT_ACCEPTED_PATCH = (
"device-edge-vps-command-transport-20260812-013"
)
COMMAND_TRANSPORT_ACCEPTED_SHA256 = (
"c7486ec879681ddd706f229628c8556ca8c9ccc4f152a85debb409c302759ef"
)
FOUNDATION_ENTRIES = (
"vps/config/00-nodedc-b2-vps.conf",
@@ -181,6 +202,21 @@ COMMAND_TRANSPORT_ENTRIES = (
"vps/edge-process/device-edge-runtime.mjs",
"deployment/device-edge-vps-command-transport-v1.json",
)
HOST_TELEMETRY_ENTRIES = (
"packages/device-edge-channel-contract/package.json",
"packages/device-edge-channel-contract/src",
"packages/infrastructure-telemetry-contract/package.json",
"packages/infrastructure-telemetry-contract/src",
"services/device-edge-channel/package.json",
"services/device-edge-channel/src",
"vps/edge-process/device-edge-runtime.mjs",
"vps/edge-process/host-telemetry-runtime.mjs",
"vps/config/nodedc-host-telemetry-telegraf.conf",
"vps/systemd/nodedc-device-edge-runtime.service",
"vps/systemd/nodedc-host-telemetry-agent.service",
"deployment/device-edge-vps-host-telemetry-v1.json",
f"vendor/{TELEGRAF_ARCHIVE}",
)
PHASE_ENTRIES = {
"foundation": FOUNDATION_ENTRIES,
@@ -191,6 +227,7 @@ PHASE_ENTRIES = {
"tailscale-retirement": TAILSCALE_RETIREMENT_ENTRIES,
"tracker-ingress": TRACKER_INGRESS_ENTRIES,
"command-transport": COMMAND_TRANSPORT_ENTRIES,
"host-telemetry": HOST_TELEMETRY_ENTRIES,
}
SUPERSEDED_TRANSPORT_PHASES = frozenset({"backhaul", "relay"})
@@ -312,6 +349,35 @@ PHASE_FILE_SHA256 = {
"deployment/device-edge-vps-command-transport-v1.json":
"971166143fe954b9c5043cce9a464d17efbc87933da1405b2517a4693a7bed09",
},
"host-telemetry": {
"deployment/device-edge-vps-host-telemetry-v1.json":
"69d4ed7c7462d982e53fa2f0688d7a129952811b96dcb1bf4ae3721f235fc406",
"packages/device-edge-channel-contract/package.json":
"57d5349b5dcef2cacd4f3e4fad010359a65d59f5f903eff07d89f67c497f97c0",
"packages/device-edge-channel-contract/src/index.mjs":
"58a53836495dc891de191c6022cf7661a2198deb9fff7055a5cc23a36ddf49d2",
"packages/infrastructure-telemetry-contract/package.json":
"5ef70204acc9a2bee68be959347487dc2e8fb7fbe8bd88731033e7ab204acf34",
"packages/infrastructure-telemetry-contract/src/index.mjs":
"6d4b60b79e131380fcec403cf8842a3612614b5540c7052020e84d4fc8a9360f",
"services/device-edge-channel/package.json":
"bdf502be43b62bdd6db05b022a532d93ba954277ac5143d6058d2f27f6a2e9d2",
"services/device-edge-channel/src/runtime.mjs":
"4c0e874b2f1161910d3abde9a07f4a7744ffeece325303cc9985521a0eafb47b",
"services/device-edge-channel/src/server.mjs":
"a82057218bb368ab926404f90a19cc17c0359b57dc890f38a1324ab8c497c17b",
"vps/config/nodedc-host-telemetry-telegraf.conf":
"596e386d1e37b8178ccc660760567f5a8914ae7bef43b3603b31846c73154972",
"vps/edge-process/device-edge-runtime.mjs":
"0fc32e8c71a028777b0945ea6a0dbab22b0244caab1d7f616702c8a4a143c597",
"vps/edge-process/host-telemetry-runtime.mjs":
"ce1c6f368199d6c91e8a7496e0e8388e3c390018f2695107bc2877eced5e566d",
"vps/systemd/nodedc-device-edge-runtime.service":
"88cd8d34df254f8daa1d82f6bcd175fb061376b65491ee6ac90b296fce675dfb",
"vps/systemd/nodedc-host-telemetry-agent.service":
"0d3aa1644af528ebd4cca7a95fb1bb89fe14544a64fb9729fd469296c4343506",
f"vendor/{TELEGRAF_ARCHIVE}": TELEGRAF_ARCHIVE_SHA256,
},
}
# Exact immutable baselines from terminally accepted predecessor artifacts.
@@ -472,7 +538,11 @@ def validate_payload(payload: Path, phase: str):
descriptor.get("component") != COMPONENT
or descriptor.get("runtimeHost") != RUNTIME_HOST
or descriptor.get("commandTransport")
!= ("typed-service-ping-v1" if phase == "command-transport" else "disabled")
!= (
"typed-service-ping-v1"
if phase in {"command-transport", "host-telemetry"}
else "disabled"
)
or not str(descriptor.get("gelios", "")).startswith("untouched")
or not descriptor.get("rollback")
):
@@ -737,6 +807,25 @@ def current_phase_preflight(phase: str):
if (LIVE_ROOT / COMMAND_TRANSPORT_ENTRIES[-1]).exists():
die("VPS command transport target path already exists")
return {"predecessor": "accepted-tracker-ingress-012"}
if phase == "host-telemetry":
command_record = applied_phase_record("command-transport")
if (
command_record.get("patch") != COMMAND_TRANSPORT_ACCEPTED_PATCH
or command_record.get("sha256")
!= COMMAND_TRANSPORT_ACCEPTED_SHA256
):
die("VPS host telemetry command transport predecessor mismatch")
source_file_state("command-transport")
validate_command_transport_runtime()
if (
HOST_TELEMETRY_UNIT.exists()
or TELEGRAF_BIN.exists()
or user_exists(TELEMETRY_USER)
or (LIVE_ROOT / HOST_TELEMETRY_ENTRIES[-2]).exists()
):
die("VPS host telemetry target boundary already exists")
assert_port_closed(HOST_TELEMETRY_PORT)
return {"predecessor": "accepted-command-transport-013"}
validate_foundation_runtime(
require_running_tailnet=phase in {"backhaul", "relay"},
expected_key_user=BACKHAUL_USER if phase == "relay" else SERVICE_USER,
@@ -848,6 +937,12 @@ def backup_targets_for_phase(phase: str):
return common + [CHANNEL_UNIT, NFTABLES_CONFIG]
if phase == "command-transport":
return common + [CHANNEL_UNIT, NFTABLES_CONFIG]
if phase == "host-telemetry":
return common + [
CHANNEL_UNIT,
HOST_TELEMETRY_UNIT,
TELEGRAF_BIN.parent,
]
return common + [RELAY_UNIT, NFTABLES_CONFIG]
@@ -887,7 +982,13 @@ def create_backup(patch_id: str, phase: str):
"serviceUserExisted": user_exists(),
"serviceUsersExisted": {
name: user_exists(name)
for name in (SERVICE_USER, BACKHAUL_USER, RELAY_USER, CHANNEL_USER)
for name in (
SERVICE_USER,
BACKHAUL_USER,
RELAY_USER,
CHANNEL_USER,
TELEMETRY_USER,
)
},
"services": {
name: {
@@ -902,6 +1003,7 @@ def create_backup(patch_id: str, phase: str):
"nodedc-b2-backhaul.service",
"nodedc-b2-relay.service",
"nodedc-device-edge-channel.service",
"nodedc-host-telemetry-agent.service",
)
},
}
@@ -1336,6 +1438,32 @@ def apply_command_transport(_payload: Path):
validate_command_transport_runtime()
def apply_host_telemetry(_payload: Path):
ensure_service_user(
TELEMETRY_USER,
"/var/lib/nodedc-b2-vps/telemetry",
)
extract_vendor_binary(
LIVE_ROOT / f"vendor/{TELEGRAF_ARCHIVE}",
f"telegraf-{TELEGRAF_VERSION}/usr/bin/telegraf",
TELEGRAF_BIN,
)
install_file(
LIVE_ROOT / "vps/systemd/nodedc-device-edge-runtime.service",
CHANNEL_UNIT,
0o644,
)
install_file(
LIVE_ROOT / "vps/systemd/nodedc-host-telemetry-agent.service",
HOST_TELEMETRY_UNIT,
0o644,
)
systemctl("daemon-reload")
systemctl("restart", "nodedc-device-edge-channel.service")
systemctl("enable", "--now", "nodedc-host-telemetry-agent.service")
validate_host_telemetry_runtime()
def sshd_effective():
return run(["/usr/sbin/sshd", "-T"]).stdout.lower()
@@ -1777,6 +1905,126 @@ def validate_command_transport_runtime():
return health
def validate_host_telemetry_runtime():
source_file_state("foundation")
source_file_state("tailscale-retirement")
source_file_state("host-telemetry")
command_record = applied_phase_record("command-transport")
if (
command_record.get("patch") != COMMAND_TRANSPORT_ACCEPTED_PATCH
or command_record.get("sha256") != COMMAND_TRANSPORT_ACCEPTED_SHA256
):
die("host telemetry command transport identity mismatch")
binary = assert_regular_nonsymlink(
TELEGRAF_BIN,
"VPS Telegraf runtime",
)
if (
binary.st_uid != 0
or binary.st_gid != 0
or (binary.st_mode & 0o777) != 0o755
or sha256_file(TELEGRAF_BIN) != TELEGRAF_BIN_SHA256
):
die("VPS Telegraf runtime identity mismatch")
version = run([str(TELEGRAF_BIN), "version"]).stdout.strip()
if not version.startswith(f"Telegraf {TELEGRAF_VERSION}"):
die("VPS Telegraf version mismatch")
telemetry_account = pwd.getpwnam(TELEMETRY_USER)
channel_account = pwd.getpwnam(CHANNEL_USER)
if (
telemetry_account.pw_shell != "/usr/sbin/nologin"
or telemetry_account.pw_uid == channel_account.pw_uid
):
die("VPS telemetry runtime identity is not isolated")
if not service_active("nodedc-device-edge-channel.service"):
die("VPS Device Edge runtime is not active")
if not service_active("nodedc-host-telemetry-agent.service"):
die("VPS host telemetry agent is not active")
if (
systemctl(
"is-enabled",
"nodedc-host-telemetry-agent.service",
check=False,
).returncode != 0
):
die("VPS host telemetry agent is not enabled")
health = None
last_error = "no host telemetry acceptance"
for _attempt in range(60):
candidate = core_channel_health(require_accepted=True)
host_telemetry = candidate.get("hostTelemetry") or {}
if (
host_telemetry.get("listening") is True
and host_telemetry.get("host") == "127.0.0.1"
and host_telemetry.get("port") == HOST_TELEMETRY_PORT
and host_telemetry.get("profile") == "linux-host-telegraf-v1"
and int(host_telemetry.get("accepted") or 0) >= 1
and host_telemetry.get("lastErrorCode") is None
):
health = candidate
break
last_error = json.dumps(host_telemetry, sort_keys=True)
time.sleep(2)
if health is None:
die(f"VPS host telemetry acceptance timeout: {last_error}")
expected = {
"ok": True,
"service": "nodedc-device-edge-runtime",
"channel": "accepted",
"trackerIngress": "telemetry-ingest",
"commandTransport": "typed-service-ping-v1",
}
for key, value in expected.items():
if health.get(key) != value:
die(f"VPS host telemetry preserved contract mismatch: {key}")
if not port_is_open("127.0.0.1", HOST_TELEMETRY_PORT, timeout=5):
die("VPS host telemetry collector is unavailable")
if port_is_open(PUBLIC_IPV4, HOST_TELEMETRY_PORT, timeout=2):
die("VPS host telemetry collector became public")
for port in (1883, 8883):
if port_is_open("127.0.0.1", port) or port_is_open(PUBLIC_IPV4, port):
die(f"VPS forbidden MQTT listener became available: {port}")
for port in (22, CHANNEL_PUBLIC_PORT, 9921):
if not port_is_open(PUBLIC_IPV4, port, timeout=5):
die(f"VPS host telemetry preserved listener unavailable: {port}")
unit = run([
"/usr/bin/systemctl",
"show",
"nodedc-host-telemetry-agent.service",
"--property=User,Group,NoNewPrivileges,CapabilityBoundingSet,MemoryMax,MemorySwapMax,TasksMax,LimitNOFILE",
]).stdout
for required in (
"User=nodedc-telemetry",
"Group=nodedc-telemetry",
"NoNewPrivileges=yes",
"CapabilityBoundingSet=",
"MemoryMax=100663296",
"MemorySwapMax=0",
"TasksMax=64",
"LimitNOFILE=512",
):
if required not in unit:
die(f"VPS host telemetry resource boundary mismatch: {required}")
nft = run([
"/usr/sbin/nft",
"list",
"table",
"inet",
"nodedc_b2_vps",
]).stdout
for required in (
"policy drop",
"tcp dport 22",
"tcp dport 443",
"tcp dport 9921",
):
if required not in nft:
die(f"VPS host telemetry firewall contract mismatch: {required}")
if f"tcp dport {HOST_TELEMETRY_PORT}" in nft:
die("VPS host telemetry firewall exposed the collector")
return health
def validate_relay_runtime():
validate_backhaul_runtime()
source_file_state("relay")
@@ -1835,6 +2083,7 @@ def restore_service_enablement(metadata):
def rollback(backup: Path, phase: str):
for service in (
"nodedc-host-telemetry-agent.service",
"nodedc-b2-relay.service",
"nodedc-b2-backhaul.service",
"nodedc-device-edge-channel.service",
@@ -1866,6 +2115,9 @@ def rollback(backup: Path, phase: str):
if phase == "core-channel":
if not users_before.get(CHANNEL_USER, False) and user_exists(CHANNEL_USER):
run(["/usr/sbin/userdel", CHANNEL_USER], check=False)
if phase == "host-telemetry":
if not users_before.get(TELEMETRY_USER, False) and user_exists(TELEMETRY_USER):
run(["/usr/sbin/userdel", TELEMETRY_USER], check=False)
if phase == "foundation" and not metadata.get("serviceUserExisted"):
runtime_state_root = Path("/var/lib/nodedc-b2-vps")
if LIVE_ROOT.exists() and not LIVE_ROOT.is_symlink():
@@ -1955,7 +2207,7 @@ def plan_artifact(artifact_argument: str):
print("runtime_composition=single-non-root-process:core-channel+universal-gateway")
print("tailscale=preserved:absent")
print("services=recreate:nodedc-device-edge-channel")
else:
elif phase == "command-transport":
print("public_core_channel=preserved:155.212.211.15:443/tcp:tls13-mtls-h2")
print(f"health=127.0.0.1:{CHANNEL_HEALTH_PORT}:combined-edge-runtime")
print("public_b2_ingress=155.212.211.15:9921/tcp:bidirectional-session")
@@ -1964,16 +2216,27 @@ def plan_artifact(artifact_argument: str):
print("runtime_composition=single-non-root-process:core-channel+universal-gateway")
print("tailscale=preserved:absent")
print("services=recreate:nodedc-device-edge-channel")
else:
print("public_core_channel=preserved:155.212.211.15:443/tcp:tls13-mtls-h2")
print(f"health=127.0.0.1:{CHANNEL_HEALTH_PORT}:combined-edge-runtime")
print(f"host_telemetry_collector=127.0.0.1:{HOST_TELEMETRY_PORT}:loopback-only")
print(f"host_telemetry_agent=telegraf:{TELEGRAF_VERSION}:sha256:{TELEGRAF_ARCHIVE_SHA256}")
print("host_telemetry_interval=2s")
print("host_telemetry_transport=existing-pinned-mtls-core-channel")
print("host_telemetry_ontology=observation.observation=>infrastructure.host")
print("mqtt=disabled")
print("public_port_set=preserved:22,443,9921")
print("services=recreate:nodedc-device-edge-channel+create:nodedc-host-telemetry-agent")
print(
"command_transport=typed-service-ping-v1:allowlisted-adapter-only"
if phase == "command-transport"
if phase in {"command-transport", "host-telemetry"}
else "command_transport=disabled"
)
if phase == "command-transport":
if phase in {"command-transport", "host-telemetry"}:
print("command_catalog=allowlisted-adapter-typed-commands-only")
print(
"gelios=untouched-legacy-only"
if phase == "command-transport"
if phase in {"command-transport", "host-telemetry"}
else "gelios=untouched"
)
print("dns=unchanged")
@@ -2013,8 +2276,10 @@ def apply_artifact(artifact_argument: str):
apply_tailscale_retirement(loaded["payload"])
elif loaded["phase"] == "tracker-ingress":
apply_tracker_ingress(loaded["payload"])
else:
elif loaded["phase"] == "command-transport":
apply_command_transport(loaded["payload"])
else:
apply_host_telemetry(loaded["payload"])
archived = archive_artifact(loaded["artifact"], APPLIED_ROOT)
record = {