SILVER/NODEDC_DEVICE_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(telemetry): add canonical VPS host monitoring

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 17:29:30 +03:00
parent a3e7a015de
commit 23ea96fbb2
33 changed files with 2541 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/build-device-control-core-release-artifact.mjs
+22 -7
View File
@@ -35,20 +35,24 @@ if (
);
}
const isV3 = patchId.startsWith("device-control-core-release-v3-");
const isV2 = patchId.startsWith("device-control-core-release-v2-");
const expectedV2Predecessor = Object.freeze({
patchId: predecessorPatchId ?? "device-control-core-release-20260812-024",
artifactSha256: predecessorSha256 ?? "a289e909283109642e6bba3d9822a31f63423cfe0bbcd52705979681bd2bc793",
});
const descriptorPath = isV2
? "deployment/device-control-core-release-v2.json"
: "deployment/device-control-core-release-v1.json";
const descriptorPath = isV3
? "deployment/device-control-core-release-v3.json"
: isV2
? "deployment/device-control-core-release-v2.json"
: "deployment/device-control-core-release-v1.json";
const entries = [
".dockerignore",
"package.json",
"package-lock.json",
"packages/device-protocol-contract",
"packages/device-edge-channel-contract",
...(isV3 ? ["packages/infrastructure-telemetry-contract"] : []),
"services/device-control-core",
descriptorPath,
];
@@ -88,6 +92,7 @@ try {
"services/device-control-core/src/device-gateway-core-runtime.mjs",
"packages/device-protocol-contract/src/index.mjs",
"packages/device-edge-channel-contract/src/index.mjs",
...(isV3 ? ["packages/infrastructure-telemetry-contract/src/index.mjs"] : []),
]) {
const imported = spawnSync(
process.execPath,
@@ -101,7 +106,7 @@ try {
const descriptor = JSON.parse(await readFile(join(payload, descriptorPath), "utf8"));
if (
descriptor.schemaVersion !== `nodedc.device-plane.device-control-core-release.${isV2 ? "v2" : "v1"}`
descriptor.schemaVersion !== `nodedc.device-plane.device-control-core-release.${isV3 ? "v3" : isV2 ? "v2" : "v1"}`
|| descriptor.releaseId !== patchId
|| descriptor.action !== "upgrade"
|| descriptor.service !== "device-control-core"
@@ -113,11 +118,11 @@ try {
|| JSON.stringify(descriptor.coreNetworks) !== JSON.stringify(["device-plane-private", "device-plane-egress"])
|| descriptor.publicIngress !== "none-on-synology"
|| descriptor.edgeRegistrations !== "preserved"
|| descriptor.commandTransport !== (isV2 ? "typed-service-ping-v1" : "disabled")
|| descriptor.gelios !== (isV2 ? "untouched-legacy-only" : "untouched")
|| descriptor.commandTransport !== ((isV2 || isV3) ? "typed-service-ping-v1" : "disabled")
|| descriptor.gelios !== ((isV2 || isV3) ? "untouched-legacy-only" : "untouched")
|| descriptor.rollback !== "restore-preapply-source-and-core-runtime"
|| (
isV2
(isV2 || isV3)
&& (
descriptor.commandCatalog !== "allowlisted-adapter-typed-commands-only"
|| descriptor.credentialBoundary !== "transient-core-memory-then-single-pinned-mtls-command-envelope-to-edge-never-persisted-never-logged-never-returned"
@@ -125,6 +130,16 @@ try {
|| descriptor.predecessor?.artifactSha256 !== expectedV2Predecessor.artifactSha256
)
)
|| (
isV3
&& (
descriptor.telemetryTransport !== "edge-channel-host-telemetry-observed-v1"
|| descriptor.telemetryContract !== "nodedc.infrastructure.host-telemetry.v1"
|| descriptor.telemetryStorage !== "device-control-core-postgres-seven-day-retention"
|| descriptor.ontologyProjection !== "observation-observed-property-provenance-freshness-v1"
|| descriptor.telemetryFreshness !== "fifteen-seconds-missing-stale-not-unhealthy"
)
)
) {
throw new Error("device_control_core_release_contract_mismatch");
}
infra/deploy-runner/build-device-edge-vps-artifact.mjs
+61 -5
View File
@@ -38,11 +38,12 @@ if (
"tailscale-retirement",
"tracker-ingress",
"command-transport",
"host-telemetry",
].includes(phase)
|| !/^[A-Za-z0-9._-]{1,96}$/.test(patchId || "")
) {
throw new Error(
"usage: build-device-edge-vps-artifact.mjs <foundation|runtime-reconciliation|backhaul|relay|core-channel|tailscale-retirement|tracker-ingress|command-transport> <patch-id>",
"usage: build-device-edge-vps-artifact.mjs <foundation|runtime-reconciliation|backhaul|relay|core-channel|tailscale-retirement|tracker-ingress|command-transport|host-telemetry> <patch-id>",
);
}
@@ -53,16 +54,22 @@ if (
) {
throw new Error("vps_initiated_transport_frozen:ADR-0001");
}
const acceptedSharedSourcePhases = new Set(["core-channel", "tracker-ingress"]);
const acceptedSharedSourcePhases = new Set([
"core-channel",
"tracker-ingress",
"command-transport",
]);
if (acceptedSharedSourcePhases.has(phase)) {
throw new Error(`accepted_vps_phase_rebuild_frozen:${phase}:ADR-0001`);
}
const nodeArchive = "node-v22.23.2-linux-x64.tar.xz";
const tailscaleArchive = "tailscale_1.102.2_amd64.tgz";
const telegrafArchive = "telegraf-1.38.4_linux_amd64.tar.gz";
const runtimeDigests = new Map([
[nodeArchive, "d60acfe00a2932254bb0ad20e01b0d74397a0875595de719654b214f4b03f307"],
[tailscaleArchive, "ad2cde12f8de95f7b93a1e0401e652291c603d42b9d60a33fb1741eb38ab04d8"],
[telegrafArchive, "81857e9745ebf26e058b6fdc27b9b2c210fd1fe61e57d7fad3d4bb9131f60041"],
]);
const entriesByPhase = {
@@ -130,6 +137,21 @@ const entriesByPhase = {
"vps/edge-process/device-edge-runtime.mjs",
"deployment/device-edge-vps-command-transport-v1.json",
],
"host-telemetry": [
"packages/device-edge-channel-contract/package.json",
"packages/device-edge-channel-contract/src",
"packages/infrastructure-telemetry-contract/package.json",
"packages/infrastructure-telemetry-contract/src",
"services/device-edge-channel/package.json",
"services/device-edge-channel/src",
"vps/edge-process/device-edge-runtime.mjs",
"vps/edge-process/host-telemetry-runtime.mjs",
"vps/config/nodedc-host-telemetry-telegraf.conf",
"vps/systemd/nodedc-device-edge-runtime.service",
"vps/systemd/nodedc-host-telemetry-agent.service",
"deployment/device-edge-vps-host-telemetry-v1.json",
`vendor/${telegrafArchive}`,
],
};
const entries = entriesByPhase[phase];
const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]);
@@ -191,10 +213,10 @@ try {
? "tcp/9921"
: ["core-channel", "tailscale-retirement"].includes(phase)
? "tcp/443-mtls-only"
: ["tracker-ingress", "command-transport"].includes(phase)
: ["tracker-ingress", "command-transport", "host-telemetry"].includes(phase)
? "tcp/443-mtls+tcp/9921-telemetry"
: "disabled",
commandTransport: phase === "command-transport"
commandTransport: ["command-transport", "host-telemetry"].includes(phase)
? "typed-service-ping-v1"
: "disabled",
gelios: "untouched",
@@ -212,7 +234,7 @@ async function assertBoundary() {
if (
descriptor.component !== "device-edge-vps"
|| descriptor.runtimeHost !== "koffyvngij"
|| descriptor.commandTransport !== (phase === "command-transport"
|| descriptor.commandTransport !== (["command-transport", "host-telemetry"].includes(phase)
? "typed-service-ping-v1"
: "disabled")
|| !String(descriptor.gelios || "").startsWith("untouched")
@@ -419,6 +441,40 @@ async function assertBoundary() {
}
}
}
if (phase === "host-telemetry") {
for (const required of [
'"predecessorPatch": "device-edge-vps-command-transport-20260812-013"',
'"agent": "telegraf"',
'"agentVersion": "1.38.4"',
'"transport": "existing-core-initiated-pinned-mtls-channel"',
'"mqtt": "disabled-no-public-broker-no-wan-plaintext"',
"User=nodedc-telemetry",
"IPAddressDeny=any",
"IPAddressAllow=localhost",
"MemoryMax=96M",
"CPUQuota=15%",
'url = "http://127.0.0.1:18223/internal/v1/host-telemetry"',
'data_format = "json"',
"submitHostTelemetry",
"createHostTelemetryCollector",
]) {
if (!combined.includes(required)) {
throw new Error(`host_telemetry_boundary_missing:${required}`);
}
}
for (const forbidden of [
"mqtt://",
"tcp://",
"outputs.mqtt",
"PRIVATE KEY",
"TS_AUTHKEY",
"device.dc.ru",
]) {
if (combined.includes(forbidden)) {
throw new Error(`host_telemetry_boundary_violation:${forbidden}`);
}
}
}
}
function canonicalTarScript() {
infra/deploy-runner/build-device-manager-control-plane-artifact.mjs
+29 -5
View File
@@ -11,11 +11,13 @@ const platformRoot = resolve(scriptDir, "../..");
const devicePlaneRoot = platformRoot;
const managerRoot = resolve(platformRoot, "apps/device-manager");
const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"));
const [patchId = "device-manager-release-v8-20260822-039", ...extra] = process.argv.slice(2);
const [patchId = "device-manager-release-v9-20260822-041", ...extra] = process.argv.slice(2);
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) throw new Error("usage: build-device-manager-control-plane-artifact.mjs [patch-id]");
const descriptorPath = patchId.startsWith("device-manager-release-v8-")
? "deployment/device-manager-release-v8.json"
const descriptorPath = patchId.startsWith("device-manager-release-v9-")
? "deployment/device-manager-release-v9.json"
: patchId.startsWith("device-manager-release-v8-")
? "deployment/device-manager-release-v8.json"
: patchId.startsWith("device-manager-release-v7-")
? "deployment/device-manager-release-v7.json"
: patchId.startsWith("device-manager-release-v6-")
@@ -34,7 +36,8 @@ const isV5 = descriptorPath.endsWith("release-v5.json");
const isV6 = descriptorPath.endsWith("release-v6.json");
const isV7 = descriptorPath.endsWith("release-v7.json");
const isV8 = descriptorPath.endsWith("release-v8.json");
const isPersistent = isV4 || isV5 || isV6 || isV7 || isV8;
const isV9 = descriptorPath.endsWith("release-v9.json");
const isPersistent = isV4 || isV5 || isV6 || isV7 || isV8 || isV9;
const isManagerOnly = isV3 || isPersistent;
const composeSource = resolve(devicePlaneRoot, "docker-compose.device-manager.yml");
const composeSourceSha256 = createHash("sha256").update(await readFile(composeSource)).digest("hex");
@@ -173,7 +176,28 @@ try {
: "restore-preapply-snapshot")
);
if (commonContractInvalid) throw new Error("device_manager_activation_successor_contract_mismatch");
if (descriptorPath.endsWith("release-v8.json")) {
if (descriptorPath.endsWith("release-v9.json")) {
if (
descriptor.schemaVersion !== "nodedc.device-plane.device-manager-release.v9"
|| descriptor.predecessor?.kind !== "release"
|| descriptor.predecessor?.patchId !== "device-manager-release-v8-20260822-039"
|| descriptor.predecessor?.artifactSha256 !== "30a83d4c6b5c029c96c4af19fa558e0ae75e4b60bc897881457304bd17e0e465"
|| descriptor.controlCorePredecessor?.patchId !== "device-control-core-release-v3-20260822-040"
|| descriptor.controlCorePredecessor?.artifactSha256 !== "08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92"
|| descriptor.edgeChannelPredecessor?.patchId !== "device-edge-core-channel-upgrade-v4-20260812-023"
|| descriptor.edgeChannelPredecessor?.artifactSha256 !== "c10d5b6b7d55ab239f85b6c8130e34ce9f84985e3b46e6e5534733156c7982fc"
|| descriptor.telemetryWorkspace !== "mission-core-compute-module-parity-v1"
|| descriptor.telemetryNavigation !== "full-workspace-back-navigation-v1"
|| descriptor.telemetryPollInterval !== "three-seconds"
|| descriptor.telemetryFreshness !== "fifteen-seconds-missing-stale-not-unhealthy"
|| descriptor.telemetryOntologyProjection !== "observation-observed-property-provenance-freshness-v1"
|| descriptor.telemetryAgent !== "telegraf-host-observer-v1"
|| descriptor.interactiveShell !== "disabled-pending-managed-session-boundary"
|| descriptor.rollback !== "restore-preapply-snapshot-preserve-manager-data"
|| descriptor.gelios !== "untouched-legacy-only"
) throw new Error("device_manager_v9_host_telemetry_workspace_contract_mismatch");
await validateFaviconBundle(payload, "device_manager_v9");
} else if (descriptorPath.endsWith("release-v8.json")) {
if (
descriptor.schemaVersion !== "nodedc.device-plane.device-manager-release.v8"
|| descriptor.predecessor?.kind !== "release"
infra/deploy-runner/nodedc-b2-vps-deploy
+272 -7
View File
@@ -46,6 +46,8 @@ RELAY_USER = "nodedc-relay"
RELAY_GROUP = "nodedc-relay"
CHANNEL_USER = "nodedc-channel"
CHANNEL_GROUP = "nodedc-channel"
TELEMETRY_USER = "nodedc-telemetry"
TELEMETRY_GROUP = "nodedc-telemetry"
TAILSCALE_REQUIRED_TAG = "tag:device-edge-vps"
MANAGEMENT_KEY_FINGERPRINT = (
"SHA256:DYYy1E3DaxIQGC0jnsW6SP7gXdBHUy3A1zn4pvgVUEw"
@@ -67,10 +69,19 @@ TAILSCALE_ARCHIVE_SHA256 = (
NODE_BIN_SHA256 = "3517c2df0b2f8cd7f422b4b8450ef81c6889f08eb03e281d6de9079b15e6a327"
TAILSCALE_BIN_SHA256 = "58b0fa0907677ea6afe0d3022cc3e99b1a03f39a7ed60144843ed38252e00c80"
TAILSCALED_BIN_SHA256 = "5f17b092bac92326325f6c4ffd9991fad3c073975abe412d02ee68721a500394"
TELEGRAF_VERSION = "1.38.4"
TELEGRAF_ARCHIVE = "telegraf-1.38.4_linux_amd64.tar.gz"
TELEGRAF_ARCHIVE_SHA256 = (
"81857e9745ebf26e058b6fdc27b9b2c210fd1fe61e57d7fad3d4bb9131f60041"
)
TELEGRAF_BIN_SHA256 = (
"0643b582546eb9c70d99a9646b3e49e25ccdea4f78ab06fd9096ed71dba1babb"
)
NODE_BIN = LIVE_ROOT / "runtime/node/bin/node"
TAILSCALE_BIN = LIVE_ROOT / "runtime/tailscale/tailscale"
TAILSCALED_BIN = LIVE_ROOT / "runtime/tailscale/tailscaled"
TELEGRAF_BIN = LIVE_ROOT / "runtime/telegraf/usr/bin/telegraf"
TAILSCALE_SOCKET = Path("/run/nodedc-b2-vps/tailscaled.sock")
TAILSCALE_STATE = Path("/var/lib/nodedc-b2-vps/tailscale/tailscaled.state")
TRUST_ROOT = Path("/var/lib/nodedc-b2-vps/trust")
@@ -102,6 +113,10 @@ CHANNEL_CORE_CERTIFICATE = CHANNEL_TRUST_ROOT / "core-certificate.pem"
CHANNEL_RUNTIME_CONFIG = CHANNEL_TRUST_ROOT / "runtime.json"
CHANNEL_HEALTH_PORT = 18222
CHANNEL_PUBLIC_PORT = 443
HOST_TELEMETRY_PORT = 18223
HOST_TELEMETRY_UNIT = Path(
"/etc/systemd/system/nodedc-host-telemetry-agent.service"
)
CORE_CHANNEL_ACCEPTED_PATCH = "device-edge-vps-core-channel-20260812-010"
CORE_CHANNEL_ACCEPTED_SHA256 = (
"c8ef3c4bb45850cad32e881eba081bc4c891c2886e5500d02cb94616d82353f3"
@@ -116,6 +131,12 @@ TRACKER_INGRESS_ACCEPTED_PATCH = "device-edge-vps-tracker-ingress-20260812-012"
TRACKER_INGRESS_ACCEPTED_SHA256 = (
"290acef118839c6b0c31aac864c47da1832a289537366af9322d4624a1dd81ec"
)
COMMAND_TRANSPORT_ACCEPTED_PATCH = (
"device-edge-vps-command-transport-20260812-013"
)
COMMAND_TRANSPORT_ACCEPTED_SHA256 = (
"c7486ec879681ddd706f229628c8556ca8c9ccc4f152a85debb409c302759ef"
)
FOUNDATION_ENTRIES = (
"vps/config/00-nodedc-b2-vps.conf",
@@ -181,6 +202,21 @@ COMMAND_TRANSPORT_ENTRIES = (
"vps/edge-process/device-edge-runtime.mjs",
"deployment/device-edge-vps-command-transport-v1.json",
)
HOST_TELEMETRY_ENTRIES = (
"packages/device-edge-channel-contract/package.json",
"packages/device-edge-channel-contract/src",
"packages/infrastructure-telemetry-contract/package.json",
"packages/infrastructure-telemetry-contract/src",
"services/device-edge-channel/package.json",
"services/device-edge-channel/src",
"vps/edge-process/device-edge-runtime.mjs",
"vps/edge-process/host-telemetry-runtime.mjs",
"vps/config/nodedc-host-telemetry-telegraf.conf",
"vps/systemd/nodedc-device-edge-runtime.service",
"vps/systemd/nodedc-host-telemetry-agent.service",
"deployment/device-edge-vps-host-telemetry-v1.json",
f"vendor/{TELEGRAF_ARCHIVE}",
)
PHASE_ENTRIES = {
"foundation": FOUNDATION_ENTRIES,
@@ -191,6 +227,7 @@ PHASE_ENTRIES = {
"tailscale-retirement": TAILSCALE_RETIREMENT_ENTRIES,
"tracker-ingress": TRACKER_INGRESS_ENTRIES,
"command-transport": COMMAND_TRANSPORT_ENTRIES,
"host-telemetry": HOST_TELEMETRY_ENTRIES,
}
SUPERSEDED_TRANSPORT_PHASES = frozenset({"backhaul", "relay"})
@@ -312,6 +349,35 @@ PHASE_FILE_SHA256 = {
"deployment/device-edge-vps-command-transport-v1.json":
"971166143fe954b9c5043cce9a464d17efbc87933da1405b2517a4693a7bed09",
},
"host-telemetry": {
"deployment/device-edge-vps-host-telemetry-v1.json":
"69d4ed7c7462d982e53fa2f0688d7a129952811b96dcb1bf4ae3721f235fc406",
"packages/device-edge-channel-contract/package.json":
"57d5349b5dcef2cacd4f3e4fad010359a65d59f5f903eff07d89f67c497f97c0",
"packages/device-edge-channel-contract/src/index.mjs":
"58a53836495dc891de191c6022cf7661a2198deb9fff7055a5cc23a36ddf49d2",
"packages/infrastructure-telemetry-contract/package.json":
"5ef70204acc9a2bee68be959347487dc2e8fb7fbe8bd88731033e7ab204acf34",
"packages/infrastructure-telemetry-contract/src/index.mjs":
"6d4b60b79e131380fcec403cf8842a3612614b5540c7052020e84d4fc8a9360f",
"services/device-edge-channel/package.json":
"bdf502be43b62bdd6db05b022a532d93ba954277ac5143d6058d2f27f6a2e9d2",
"services/device-edge-channel/src/runtime.mjs":
"4c0e874b2f1161910d3abde9a07f4a7744ffeece325303cc9985521a0eafb47b",
"services/device-edge-channel/src/server.mjs":
"a82057218bb368ab926404f90a19cc17c0359b57dc890f38a1324ab8c497c17b",
"vps/config/nodedc-host-telemetry-telegraf.conf":
"596e386d1e37b8178ccc660760567f5a8914ae7bef43b3603b31846c73154972",
"vps/edge-process/device-edge-runtime.mjs":
"0fc32e8c71a028777b0945ea6a0dbab22b0244caab1d7f616702c8a4a143c597",
"vps/edge-process/host-telemetry-runtime.mjs":
"ce1c6f368199d6c91e8a7496e0e8388e3c390018f2695107bc2877eced5e566d",
"vps/systemd/nodedc-device-edge-runtime.service":
"88cd8d34df254f8daa1d82f6bcd175fb061376b65491ee6ac90b296fce675dfb",
"vps/systemd/nodedc-host-telemetry-agent.service":
"0d3aa1644af528ebd4cca7a95fb1bb89fe14544a64fb9729fd469296c4343506",
f"vendor/{TELEGRAF_ARCHIVE}": TELEGRAF_ARCHIVE_SHA256,
},
}
# Exact immutable baselines from terminally accepted predecessor artifacts.
@@ -472,7 +538,11 @@ def validate_payload(payload: Path, phase: str):
descriptor.get("component") != COMPONENT
or descriptor.get("runtimeHost") != RUNTIME_HOST
or descriptor.get("commandTransport")
!= ("typed-service-ping-v1" if phase == "command-transport" else "disabled")
!= (
"typed-service-ping-v1"
if phase in {"command-transport", "host-telemetry"}
else "disabled"
)
or not str(descriptor.get("gelios", "")).startswith("untouched")
or not descriptor.get("rollback")
):
@@ -737,6 +807,25 @@ def current_phase_preflight(phase: str):
if (LIVE_ROOT / COMMAND_TRANSPORT_ENTRIES[-1]).exists():
die("VPS command transport target path already exists")
return {"predecessor": "accepted-tracker-ingress-012"}
if phase == "host-telemetry":
command_record = applied_phase_record("command-transport")
if (
command_record.get("patch") != COMMAND_TRANSPORT_ACCEPTED_PATCH
or command_record.get("sha256")
!= COMMAND_TRANSPORT_ACCEPTED_SHA256
):
die("VPS host telemetry command transport predecessor mismatch")
source_file_state("command-transport")
validate_command_transport_runtime()
if (
HOST_TELEMETRY_UNIT.exists()
or TELEGRAF_BIN.exists()
or user_exists(TELEMETRY_USER)
or (LIVE_ROOT / HOST_TELEMETRY_ENTRIES[-2]).exists()
):
die("VPS host telemetry target boundary already exists")
assert_port_closed(HOST_TELEMETRY_PORT)
return {"predecessor": "accepted-command-transport-013"}
validate_foundation_runtime(
require_running_tailnet=phase in {"backhaul", "relay"},
expected_key_user=BACKHAUL_USER if phase == "relay" else SERVICE_USER,
@@ -848,6 +937,12 @@ def backup_targets_for_phase(phase: str):
return common + [CHANNEL_UNIT, NFTABLES_CONFIG]
if phase == "command-transport":
return common + [CHANNEL_UNIT, NFTABLES_CONFIG]
if phase == "host-telemetry":
return common + [
CHANNEL_UNIT,
HOST_TELEMETRY_UNIT,
TELEGRAF_BIN.parent,
]
return common + [RELAY_UNIT, NFTABLES_CONFIG]
@@ -887,7 +982,13 @@ def create_backup(patch_id: str, phase: str):
"serviceUserExisted": user_exists(),
"serviceUsersExisted": {
name: user_exists(name)
for name in (SERVICE_USER, BACKHAUL_USER, RELAY_USER, CHANNEL_USER)
for name in (
SERVICE_USER,
BACKHAUL_USER,
RELAY_USER,
CHANNEL_USER,
TELEMETRY_USER,
)
},
"services": {
name: {
@@ -902,6 +1003,7 @@ def create_backup(patch_id: str, phase: str):
"nodedc-b2-backhaul.service",
"nodedc-b2-relay.service",
"nodedc-device-edge-channel.service",
"nodedc-host-telemetry-agent.service",
)
},
}
@@ -1336,6 +1438,32 @@ def apply_command_transport(_payload: Path):
validate_command_transport_runtime()
def apply_host_telemetry(_payload: Path):
ensure_service_user(
TELEMETRY_USER,
"/var/lib/nodedc-b2-vps/telemetry",
)
extract_vendor_binary(
LIVE_ROOT / f"vendor/{TELEGRAF_ARCHIVE}",
f"telegraf-{TELEGRAF_VERSION}/usr/bin/telegraf",
TELEGRAF_BIN,
)
install_file(
LIVE_ROOT / "vps/systemd/nodedc-device-edge-runtime.service",
CHANNEL_UNIT,
0o644,
)
install_file(
LIVE_ROOT / "vps/systemd/nodedc-host-telemetry-agent.service",
HOST_TELEMETRY_UNIT,
0o644,
)
systemctl("daemon-reload")
systemctl("restart", "nodedc-device-edge-channel.service")
systemctl("enable", "--now", "nodedc-host-telemetry-agent.service")
validate_host_telemetry_runtime()
def sshd_effective():
return run(["/usr/sbin/sshd", "-T"]).stdout.lower()
@@ -1777,6 +1905,126 @@ def validate_command_transport_runtime():
return health
def validate_host_telemetry_runtime():
source_file_state("foundation")
source_file_state("tailscale-retirement")
source_file_state("host-telemetry")
command_record = applied_phase_record("command-transport")
if (
command_record.get("patch") != COMMAND_TRANSPORT_ACCEPTED_PATCH
or command_record.get("sha256") != COMMAND_TRANSPORT_ACCEPTED_SHA256
):
die("host telemetry command transport identity mismatch")
binary = assert_regular_nonsymlink(
TELEGRAF_BIN,
"VPS Telegraf runtime",
)
if (
binary.st_uid != 0
or binary.st_gid != 0
or (binary.st_mode & 0o777) != 0o755
or sha256_file(TELEGRAF_BIN) != TELEGRAF_BIN_SHA256
):
die("VPS Telegraf runtime identity mismatch")
version = run([str(TELEGRAF_BIN), "version"]).stdout.strip()
if not version.startswith(f"Telegraf {TELEGRAF_VERSION}"):
die("VPS Telegraf version mismatch")
telemetry_account = pwd.getpwnam(TELEMETRY_USER)
channel_account = pwd.getpwnam(CHANNEL_USER)
if (
telemetry_account.pw_shell != "/usr/sbin/nologin"
or telemetry_account.pw_uid == channel_account.pw_uid
):
die("VPS telemetry runtime identity is not isolated")
if not service_active("nodedc-device-edge-channel.service"):
die("VPS Device Edge runtime is not active")
if not service_active("nodedc-host-telemetry-agent.service"):
die("VPS host telemetry agent is not active")
if (
systemctl(
"is-enabled",
"nodedc-host-telemetry-agent.service",
check=False,
).returncode != 0
):
die("VPS host telemetry agent is not enabled")
health = None
last_error = "no host telemetry acceptance"
for _attempt in range(60):
candidate = core_channel_health(require_accepted=True)
host_telemetry = candidate.get("hostTelemetry") or {}
if (
host_telemetry.get("listening") is True
and host_telemetry.get("host") == "127.0.0.1"
and host_telemetry.get("port") == HOST_TELEMETRY_PORT
and host_telemetry.get("profile") == "linux-host-telegraf-v1"
and int(host_telemetry.get("accepted") or 0) >= 1
and host_telemetry.get("lastErrorCode") is None
):
health = candidate
break
last_error = json.dumps(host_telemetry, sort_keys=True)
time.sleep(2)
if health is None:
die(f"VPS host telemetry acceptance timeout: {last_error}")
expected = {
"ok": True,
"service": "nodedc-device-edge-runtime",
"channel": "accepted",
"trackerIngress": "telemetry-ingest",
"commandTransport": "typed-service-ping-v1",
}
for key, value in expected.items():
if health.get(key) != value:
die(f"VPS host telemetry preserved contract mismatch: {key}")
if not port_is_open("127.0.0.1", HOST_TELEMETRY_PORT, timeout=5):
die("VPS host telemetry collector is unavailable")
if port_is_open(PUBLIC_IPV4, HOST_TELEMETRY_PORT, timeout=2):
die("VPS host telemetry collector became public")
for port in (1883, 8883):
if port_is_open("127.0.0.1", port) or port_is_open(PUBLIC_IPV4, port):
die(f"VPS forbidden MQTT listener became available: {port}")
for port in (22, CHANNEL_PUBLIC_PORT, 9921):
if not port_is_open(PUBLIC_IPV4, port, timeout=5):
die(f"VPS host telemetry preserved listener unavailable: {port}")
unit = run([
"/usr/bin/systemctl",
"show",
"nodedc-host-telemetry-agent.service",
"--property=User,Group,NoNewPrivileges,CapabilityBoundingSet,MemoryMax,MemorySwapMax,TasksMax,LimitNOFILE",
]).stdout
for required in (
"User=nodedc-telemetry",
"Group=nodedc-telemetry",
"NoNewPrivileges=yes",
"CapabilityBoundingSet=",
"MemoryMax=100663296",
"MemorySwapMax=0",
"TasksMax=64",
"LimitNOFILE=512",
):
if required not in unit:
die(f"VPS host telemetry resource boundary mismatch: {required}")
nft = run([
"/usr/sbin/nft",
"list",
"table",
"inet",
"nodedc_b2_vps",
]).stdout
for required in (
"policy drop",
"tcp dport 22",
"tcp dport 443",
"tcp dport 9921",
):
if required not in nft:
die(f"VPS host telemetry firewall contract mismatch: {required}")
if f"tcp dport {HOST_TELEMETRY_PORT}" in nft:
die("VPS host telemetry firewall exposed the collector")
return health
def validate_relay_runtime():
validate_backhaul_runtime()
source_file_state("relay")
@@ -1835,6 +2083,7 @@ def restore_service_enablement(metadata):
def rollback(backup: Path, phase: str):
for service in (
"nodedc-host-telemetry-agent.service",
"nodedc-b2-relay.service",
"nodedc-b2-backhaul.service",
"nodedc-device-edge-channel.service",
@@ -1866,6 +2115,9 @@ def rollback(backup: Path, phase: str):
if phase == "core-channel":
if not users_before.get(CHANNEL_USER, False) and user_exists(CHANNEL_USER):
run(["/usr/sbin/userdel", CHANNEL_USER], check=False)
if phase == "host-telemetry":
if not users_before.get(TELEMETRY_USER, False) and user_exists(TELEMETRY_USER):
run(["/usr/sbin/userdel", TELEMETRY_USER], check=False)
if phase == "foundation" and not metadata.get("serviceUserExisted"):
runtime_state_root = Path("/var/lib/nodedc-b2-vps")
if LIVE_ROOT.exists() and not LIVE_ROOT.is_symlink():
@@ -1955,7 +2207,7 @@ def plan_artifact(artifact_argument: str):
print("runtime_composition=single-non-root-process:core-channel+universal-gateway")
print("tailscale=preserved:absent")
print("services=recreate:nodedc-device-edge-channel")
else:
elif phase == "command-transport":
print("public_core_channel=preserved:155.212.211.15:443/tcp:tls13-mtls-h2")
print(f"health=127.0.0.1:{CHANNEL_HEALTH_PORT}:combined-edge-runtime")
print("public_b2_ingress=155.212.211.15:9921/tcp:bidirectional-session")
@@ -1964,16 +2216,27 @@ def plan_artifact(artifact_argument: str):
print("runtime_composition=single-non-root-process:core-channel+universal-gateway")
print("tailscale=preserved:absent")
print("services=recreate:nodedc-device-edge-channel")
else:
print("public_core_channel=preserved:155.212.211.15:443/tcp:tls13-mtls-h2")
print(f"health=127.0.0.1:{CHANNEL_HEALTH_PORT}:combined-edge-runtime")
print(f"host_telemetry_collector=127.0.0.1:{HOST_TELEMETRY_PORT}:loopback-only")
print(f"host_telemetry_agent=telegraf:{TELEGRAF_VERSION}:sha256:{TELEGRAF_ARCHIVE_SHA256}")
print("host_telemetry_interval=2s")
print("host_telemetry_transport=existing-pinned-mtls-core-channel")
print("host_telemetry_ontology=observation.observation=>infrastructure.host")
print("mqtt=disabled")
print("public_port_set=preserved:22,443,9921")
print("services=recreate:nodedc-device-edge-channel+create:nodedc-host-telemetry-agent")
print(
"command_transport=typed-service-ping-v1:allowlisted-adapter-only"
if phase == "command-transport"
if phase in {"command-transport", "host-telemetry"}
else "command_transport=disabled"
)
if phase == "command-transport":
if phase in {"command-transport", "host-telemetry"}:
print("command_catalog=allowlisted-adapter-typed-commands-only")
print(
"gelios=untouched-legacy-only"
if phase == "command-transport"
if phase in {"command-transport", "host-telemetry"}
else "gelios=untouched"
)
print("dns=unchanged")
@@ -2013,8 +2276,10 @@ def apply_artifact(artifact_argument: str):
apply_tailscale_retirement(loaded["payload"])
elif loaded["phase"] == "tracker-ingress":
apply_tracker_ingress(loaded["payload"])
else:
elif loaded["phase"] == "command-transport":
apply_command_transport(loaded["payload"])
else:
apply_host_telemetry(loaded["payload"])
archived = archive_artifact(loaded["artifact"], APPLIED_ROOT)
record = {
infra/deploy-runner/test_device_edge_vps_artifact.py
+27 -15
View File
@@ -82,7 +82,7 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
RUNNER.preflight({"phase": phase})
def test_accepted_shared_source_phases_cannot_be_rebuilt(self):
for phase in ("core-channel", "tracker-ingress"):
for phase in ("core-channel", "tracker-ingress", "command-transport"):
with self.subTest(phase=phase), tempfile.TemporaryDirectory(
prefix=f"nodedc-vps-frozen-{phase}-"
) as directory:
@@ -102,6 +102,7 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
for name, digest in (
(RUNNER.NODE_ARCHIVE, RUNNER.NODE_ARCHIVE_SHA256),
(RUNNER.TAILSCALE_ARCHIVE, RUNNER.TAILSCALE_ARCHIVE_SHA256),
(RUNNER.TELEGRAF_ARCHIVE, RUNNER.TELEGRAF_ARCHIVE_SHA256),
):
path = DEFAULT_RUNTIME_CACHE / name
if not path.is_file():
@@ -114,10 +115,19 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
+ ", ".join(missing)
)
def require_telegraf_cache(self):
archive = DEFAULT_RUNTIME_CACHE / RUNNER.TELEGRAF_ARCHIVE
if not archive.is_file():
self.skipTest(f"immutable Telegraf runtime is not available: {archive}")
self.assertEqual(
hashlib.sha256(archive.read_bytes()).hexdigest(),
RUNNER.TELEGRAF_ARCHIVE_SHA256,
)
def test_builders_are_deterministic_narrow_and_secret_free(self):
self.require_runtime_cache()
self.require_telegraf_cache()
for phase in (
"command-transport",
"host-telemetry",
):
with self.subTest(phase=phase), tempfile.TemporaryDirectory(
prefix=f"nodedc-vps-{phase}-"
@@ -183,7 +193,7 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
self.assertIn("runtime_digest_mismatch", result.stderr)
def test_runner_loads_each_exact_phase(self):
self.require_runtime_cache()
self.require_telegraf_cache()
with tempfile.TemporaryDirectory(prefix="nodedc-vps-load-") as directory:
inbox = Path(directory) / "inbox"
inbox.mkdir()
@@ -191,7 +201,7 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
RUNNER.INBOX_ROOT = inbox
try:
for phase in (
"command-transport",
"host-telemetry",
):
result = self.build(
inbox,
@@ -416,14 +426,15 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
self.assertIn("command_transport=disabled", rendered)
self.assertIn("gelios=untouched", rendered)
def test_command_transport_plan_is_typed_single_process_and_bounded(self):
with tempfile.TemporaryDirectory(prefix="nodedc-vps-command-plan-") as directory:
def test_host_telemetry_plan_is_loopback_agent_over_existing_mtls(self):
self.require_telegraf_cache()
with tempfile.TemporaryDirectory(prefix="nodedc-vps-host-telemetry-plan-") as directory:
inbox = Path(directory) / "inbox"
inbox.mkdir()
result = self.build(
inbox,
"command-transport",
"device-edge-vps-command-transport-plan-001",
"host-telemetry",
"device-edge-vps-host-telemetry-plan-001",
)
self.assertEqual(result.returncode, 0, result.stderr)
artifact = Path(json.loads(result.stdout)["artifact"])
@@ -433,7 +444,7 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
with patch.object(RUNNER, "assert_root"), patch.object(
RUNNER,
"preflight",
return_value={"predecessor": "accepted-tracker-ingress-012"},
return_value={"predecessor": "accepted-command-transport-013"},
), patch("builtins.print") as output:
RUNNER.plan_artifact(str(artifact))
finally:
@@ -442,12 +453,13 @@ class DeviceEdgeVpsArtifactTest(unittest.TestCase):
" ".join(str(arg) for arg in call.args)
for call in output.call_args_list
)
self.assertIn("phase=command-transport", rendered)
self.assertIn("predecessor=accepted-tracker-ingress-012", rendered)
self.assertIn("phase=host-telemetry", rendered)
self.assertIn("predecessor=accepted-command-transport-013", rendered)
self.assertIn("host_telemetry_agent=telegraf:1.38.4", rendered)
self.assertIn("host_telemetry_collector=127.0.0.1:18223:loopback-only", rendered)
self.assertIn("host_telemetry_transport=existing-pinned-mtls-core-channel", rendered)
self.assertIn("mqtt=disabled", rendered)
self.assertIn("command_transport=typed-service-ping-v1", rendered)
self.assertIn("command_catalog=allowlisted-adapter-typed-commands-only", rendered)
self.assertIn("runtime_composition=single-non-root-process", rendered)
self.assertIn("public_b2_ingress=155.212.211.15:9921/tcp:bidirectional-session", rendered)
self.assertIn("gelios=untouched-legacy-only", rendered)
def test_publish_payload_preserves_unselected_executable_modes(self):