SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(deploy): publish device plane loopback health

Browse Source
This commit is contained in:
Codex
2026-07-26 00:09:55 +03:00
parent aca47c6143
commit 3c538ad98c
10 changed files with 1565 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+781 -2
View File
@@ -7,6 +7,7 @@ import os
import re
import secrets
import shutil
import socket
import stat
import subprocess
import sys
@@ -69,6 +70,13 @@ DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES = (
*DEVICE_PLANE_FOUNDATION_ENTRIES,
DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
)
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL = (
"deployment/device-plane-foundation-network-publication-v1.json"
)
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES = (
*DEVICE_PLANE_FOUNDATION_ENTRIES,
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
)
DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID = (
"device-plane-foundation-20260725-001"
)
@@ -116,6 +124,56 @@ DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS = {
"sha256:0001755f47e38bfd163788ccc543387b7f3da35a01b38a7c2ca1cd0277f06b5d"
),
}
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID = (
"device-plane-foundation-recovery-20260725-002"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT = (
"nodedc-device-plane-device-plane-foundation-recovery-20260725-002.tgz."
"20260725-232447"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256 = (
"9183cc385142584bfd12510bb0a3e6b833b2fd26607436f2486a564c628ea1bf"
)
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID = (
"device-plane-device-plane-foundation-recovery-20260725-002-"
"20260725-232447"
)
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_SHA256 = {
"manifest.env": (
"e1c31f28474d66c33f8b91e890c112e605ffe376ed3e7292d2f8504ea18fd9c1"
),
"files.txt": (
"bb0e3deec5952811dd413afdf24d116e5bf3fd3e041f78bfe5cc5ca87166b8ff"
),
"existing-files.txt": (
"f2e2c6e7501d3a383cd7e909a2e4e4cfbb3b0cda132f508f54e478bfb455585b"
),
"missing-files.txt": (
"e57cda977037cabbe01de04ece1a1e9f15cb9eb3c0ad2996811d6ea3d4f4b3d2"
),
"runtime-before.json": (
"f500964078c5814d2130fe57b5c67fb89dea000355591836e1b72088b37170ef"
),
"source-before.tgz": (
"eed474b04608e206f09ebe68ca80da675a2318fa3efaf3be90f2d983429f9d9f"
),
}
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256 = (
"47d4d153d27bc6f418c1f787ea5ff0153b1ee082f96b92e28486805cb46b3f15"
)
DEVICE_PLANE_PRIVATE_NETWORK = "nodedc-device-plane-private"
DEVICE_PLANE_CONTROL_NETWORK = "nodedc-device-plane-control"
DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS = {
"device-control-core": (
"ccdaeee71472a557ccecb485e81b31f5a4ee37b1b3f3cb4d5edbcf12d02bb08b"
),
"device-gateway": (
"d59258eb86ae8ec728ceaaed5b4976dd1359bdcecd6abc7950e73ffabf1d8eba"
),
"device-postgres": (
"44702b995b4397131646ed8076144efd116cb38d00d9a3d641252b7ad635d7a1"
),
}
DEVICE_PLANE_RUNTIME_SERVICES = (
"device-control-core",
"device-gateway",
@@ -2806,6 +2864,7 @@ def allowed_payload_path(component, rel):
"docker-compose.device-plane.yml",
DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
@@ -7835,6 +7894,13 @@ def load_artifact(artifact, work_dir):
entries,
):
validate_device_plane_foundation_recovery_payload(payload_dir)
if is_device_plane_foundation_network_publication_slice(
manifest["component"],
entries,
):
validate_device_plane_foundation_network_publication_payload(
payload_dir
)
if manifest["component"] == "n8n-private-extension":
validate_n8n_private_extension_release(payload_dir, entries)
if manifest["component"] == "engine":
@@ -8118,6 +8184,16 @@ def reject_terminal_device_plane_foundation_artifact(manifest, sha256):
"Device Plane foundation 001 is terminal failed; "
"use the exact registered foundation recovery successor"
)
if (
manifest.get("id")
== DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
or sha256
== DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
):
die(
"Device Plane foundation recovery 002 is terminal failed; "
"use the exact registered network-publication successor"
)
class DeployLock:
@@ -8153,6 +8229,15 @@ def is_device_plane_foundation_recovery_slice(component, entries):
)
def is_device_plane_foundation_network_publication_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries)
== DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES
)
def expected_device_plane_foundation_recovery_descriptor():
return {
"schemaVersion": "nodedc.device-plane.foundation-recovery.v1",
@@ -8185,6 +8270,65 @@ def validate_device_plane_foundation_recovery_payload(payload_dir):
return descriptor
def expected_device_plane_foundation_network_publication_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane.foundation-network-publication.v1"
),
"mode": "failed-foundation-network-publication-correction",
"failedRecoveryPatchId": (
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
),
"failedRecoveryArtifactSha256": (
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
),
"failedRecoveryBackupId": (
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID
),
"sourceAction": "publish-network-corrected-foundation-source",
"runtimeAction": "recreate-stateless-services-no-build",
"selectedServices": [
"device-control-core",
"device-gateway",
],
"preservedServices": ["device-postgres"],
"privateNetwork": DEVICE_PLANE_PRIVATE_NETWORK,
"controlNetwork": DEVICE_PLANE_CONTROL_NETWORK,
"publishedLoopbackPorts": [
"127.0.0.1:18120:18120",
"127.0.0.1:18121:18121",
],
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
"rollback": (
"restore-partial-source-and-internal-only-stateless-runtime"
),
}
def validate_device_plane_foundation_network_publication_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
"Device Plane foundation network-publication descriptor",
max_bytes=16 * 1024,
)
expected = (
expected_device_plane_foundation_network_publication_descriptor()
)
if descriptor != expected:
die(
"Device Plane foundation network-publication descriptor mismatch"
)
compose = payload_dir / "docker-compose.device-plane.yml"
if (
compose.is_symlink()
or not compose.is_file()
or sha256_file(compose)
!= DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256
):
die("Device Plane network-publication Compose mismatch")
return descriptor
def device_plane_service_container_ids(service):
if service not in DEVICE_PLANE_RUNTIME_SERVICES:
die(f"Device Plane runtime service is not registered: {service}")
@@ -8464,6 +8608,263 @@ def validate_device_plane_foundation_recovery_evidence(payload_dir):
}
def inspect_device_plane_network_optional(name):
result = subprocess.run(
[str(DOCKER), "network", "inspect", name],
check=False,
capture_output=True,
text=True,
)
if result.returncode == 1:
return None
if result.returncode != 0:
die(f"Device Plane network inspect failed: {name}")
try:
value = json.loads(result.stdout)
except json.JSONDecodeError:
die(f"Device Plane network inspect returned invalid JSON: {name}")
if (
not isinstance(value, list)
or len(value) != 1
or not isinstance(value[0], dict)
):
die(f"Device Plane network inspect shape mismatch: {name}")
return value[0]
def validate_device_plane_network_contract(
name,
*,
internal,
expected_container_ids,
):
network = inspect_device_plane_network_optional(name)
if network is None:
die(f"Device Plane network is missing: {name}")
options = network.get("Options") or {}
if (
network.get("Name") != name
or network.get("Driver") != "bridge"
or network.get("Internal") is not internal
or set((network.get("Containers") or {}).keys())
!= set(expected_container_ids)
):
die(f"Device Plane network boundary mismatch: {name}")
if (
name == DEVICE_PLANE_CONTROL_NETWORK
and options.get(
"com.docker.network.bridge.enable_ip_masquerade"
)
!= "false"
):
die("Device Plane control network masquerade boundary mismatch")
return network
def validate_device_plane_foundation_network_publication_evidence(
payload_dir,
):
validate_device_plane_foundation_network_publication_payload(payload_dir)
backup_dir = (
BACKUPS_DIR
/ DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID
)
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Device Plane failed recovery backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Device Plane failed recovery backup is unsafe")
backup_names = {child.name for child in backup_dir.iterdir()}
expected_backup = (
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_SHA256
)
if backup_names != set(expected_backup):
die("Device Plane failed recovery backup file set mismatch")
for name, expected_sha256 in expected_backup.items():
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected_sha256
):
die(
"Device Plane failed recovery backup drift detected: "
f"{name}"
)
failed_artifact = (
FAILED_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT
)
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Plane failed recovery artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
):
die("Device Plane failed recovery artifact evidence mismatch")
try:
state_stat = FAILED_STATE_FILE.lstat()
state_lines = FAILED_STATE_FILE.read_text(
encoding="utf-8"
).splitlines()
except (FileNotFoundError, OSError, UnicodeDecodeError):
die("Device Plane failed recovery journal is unreadable")
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
state_stat.st_mode
):
die("Device Plane failed recovery journal is unsafe")
records = []
for line in state_lines:
try:
value = json.loads(line)
except json.JSONDecodeError:
die("Device Plane failed recovery journal contains invalid JSON")
if (
isinstance(value, dict)
and value.get("id")
== DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
):
records.append(value)
if len(records) != 1:
die("Device Plane failed recovery journal evidence count mismatch")
record = records[0]
if (
record.get("artifact")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT
or record.get("backup_id")
!= DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID
or record.get("component") != "device-plane"
or record.get("sha256")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status")
!= (
"ok:device-plane-overlay:"
"source-restored-runtime-unchanged:9"
)
or record.get("status") != "failed"
or record.get("message")
!= (
"healthcheck failed for http://127.0.0.1:18120/healthz: "
"<urlopen error [Errno 111] Connection refused>"
)
):
die("Device Plane failed recovery journal evidence mismatch")
comparable_entries = tuple(
rel
for rel in DEVICE_PLANE_FOUNDATION_ENTRIES
if rel != "docker-compose.device-plane.yml"
)
with tempfile.TemporaryDirectory(
prefix="device-plane-failed-recovery-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES
):
die("Device Plane failed recovery artifact contract mismatch")
failed_source = collect_exact_files(
failed_payload,
comparable_entries,
"Device Plane failed recovery source",
)
candidate_source = collect_exact_files(
payload_dir,
comparable_entries,
"Device Plane network-publication candidate source",
)
if candidate_source != failed_source:
die(
"Device Plane network-publication candidate source drift "
"detected"
)
root = component_root("device-plane")
compose_path = root / "docker-compose.device-plane.yml"
if (
compose_path.is_symlink()
or not compose_path.is_file()
or sha256_file(compose_path)
!= DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256
):
die("Device Plane network-publication source predecessor mismatch")
for rel in DEVICE_PLANE_FOUNDATION_ENTRIES:
if rel == "docker-compose.device-plane.yml":
continue
path = root / rel
if path.exists() or path.is_symlink():
die(
"Device Plane network-publication predecessor retained "
f"source: {rel}"
)
for descriptor_rel in (
DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
):
descriptor_path = root / descriptor_rel
if descriptor_path.exists() or descriptor_path.is_symlink():
die(
"Device Plane network-publication descriptor predecessor "
"mismatch"
)
bootstrap_descriptor = root / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL
if (
bootstrap_descriptor.is_symlink()
or not bootstrap_descriptor.is_file()
or sha256_file(bootstrap_descriptor)
!= DEVICE_PLANE_POSTGRES_BOOTSTRAP_DESCRIPTOR_SHA256
):
die("Device Plane PostgreSQL bootstrap descriptor drift detected")
runtime = validate_device_plane_foundation_runtime()
for service, expected_id in (
DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS.items()
):
if runtime[service]["containerId"] != expected_id:
die(
"Device Plane network-publication runtime predecessor "
f"generation mismatch: {service}"
)
expected_network_ids = {
item["containerId"]
for item in runtime.values()
}
validate_device_plane_network_contract(
DEVICE_PLANE_PRIVATE_NETWORK,
internal=True,
expected_container_ids=expected_network_ids,
)
if inspect_device_plane_network_optional(DEVICE_PLANE_CONTROL_NETWORK):
die("Device Plane control network already exists")
return {
"mode": "failed-foundation-network-publication-correction",
"backup": backup_dir,
"failedArtifact": failed_artifact,
"runtime": runtime,
}
def validate_device_plane_postgres_bootstrap_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
@@ -8537,7 +8938,7 @@ def device_plane_postgres_plan_selection(postgres_preflight):
return "bootstrap-selected:create-if-absent"
def validate_device_plane_foundation_runtime():
def validate_device_plane_foundation_runtime(network_publication=False):
validate_device_plane_runtime_secret_metadata()
expected = {
"device-control-core": {
@@ -8552,6 +8953,24 @@ def validate_device_plane_foundation_runtime():
"HostPort": "18120",
}],
},
"actualPorts": (
{
"18120/tcp": [{
"HostIp": "127.0.0.1",
"HostPort": "18120",
}],
}
if network_publication
else {}
),
"networks": (
{
DEVICE_PLANE_PRIVATE_NETWORK,
DEVICE_PLANE_CONTROL_NETWORK,
}
if network_publication
else {DEVICE_PLANE_PRIVATE_NETWORK}
),
},
"device-gateway": {
"image": DEVICE_PLANE_GATEWAY_IMAGE,
@@ -8565,6 +8984,24 @@ def validate_device_plane_foundation_runtime():
"HostPort": "18121",
}],
},
"actualPorts": (
{
"18121/tcp": [{
"HostIp": "127.0.0.1",
"HostPort": "18121",
}],
}
if network_publication
else {}
),
"networks": (
{
DEVICE_PLANE_PRIVATE_NETWORK,
DEVICE_PLANE_CONTROL_NETWORK,
}
if network_publication
else {DEVICE_PLANE_PRIVATE_NETWORK}
),
},
"device-postgres": {
"image": "postgres:16-alpine",
@@ -8573,6 +9010,8 @@ def validate_device_plane_foundation_runtime():
],
"user": "",
"ports": {},
"actualPorts": {},
"networks": {DEVICE_PLANE_PRIVATE_NETWORK},
},
}
accepted = {}
@@ -8591,6 +9030,9 @@ def validate_device_plane_foundation_runtime():
networks = (container.get("NetworkSettings") or {}).get(
"Networks"
) or {}
actual_ports = (container.get("NetworkSettings") or {}).get(
"Ports"
) or {}
if (
state.get("Status") != "running"
or state.get("Running") is not True
@@ -8614,7 +9056,8 @@ def validate_device_plane_foundation_runtime():
or labels.get("com.docker.compose.project")
!= "nodedc-device-plane"
or labels.get("com.docker.compose.service") != service
or set(networks) != {"nodedc-device-plane-private"}
or set(networks) != contract["networks"]
or actual_ports != contract["actualPorts"]
):
die(
"Device Plane foundation runtime boundary mismatch: "
@@ -8627,6 +9070,24 @@ def validate_device_plane_foundation_runtime():
"restartCount": 0,
}
if network_publication:
for service in ("device-control-core", "device-gateway"):
if (
accepted[service]["containerId"]
== DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[service]
):
die(
"Device Plane stateless generation was not recreated: "
f"{service}"
)
if (
accepted["device-postgres"]["containerId"]
!= DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[
"device-postgres"
]
):
die("Device Plane PostgreSQL generation changed")
core = inspect_device_plane_container(
device_plane_service_container_ids("device-control-core")[0]
)
@@ -8752,6 +9213,23 @@ def validate_device_plane_foundation_runtime():
or postgres_volume[0].get("RW") is not True
):
die("Device Plane PostgreSQL preserved mount mismatch")
if network_publication:
validate_device_plane_network_contract(
DEVICE_PLANE_PRIVATE_NETWORK,
internal=True,
expected_container_ids={
accepted[service]["containerId"]
for service in DEVICE_PLANE_RUNTIME_SERVICES
},
)
validate_device_plane_network_contract(
DEVICE_PLANE_CONTROL_NETWORK,
internal=False,
expected_container_ids={
accepted["device-control-core"]["containerId"],
accepted["device-gateway"]["containerId"],
},
)
return accepted
@@ -8834,6 +9312,75 @@ def validate_device_plane_foundation_installed_source():
return actual
def validate_device_plane_foundation_network_publication_installed_source():
failed_artifact = (
FAILED_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT
)
comparable_entries = tuple(
rel
for rel in DEVICE_PLANE_FOUNDATION_ENTRIES
if rel != "docker-compose.device-plane.yml"
)
with tempfile.TemporaryDirectory(
prefix="device-plane-installed-network-publication-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
or tuple(failed_entries)
!= DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES
):
die("Device Plane failed recovery source contract mismatch")
expected = collect_exact_files(
failed_payload,
comparable_entries,
"Device Plane failed recovery source",
)
actual = collect_exact_files(
component_root("device-plane"),
comparable_entries,
"Device Plane installed network-publication source",
)
if actual != expected:
die("Device Plane installed network-publication source mismatch")
compose = (
component_root("device-plane")
/ "docker-compose.device-plane.yml"
)
if (
compose.is_symlink()
or not compose.is_file()
or sha256_file(compose)
!= DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256
):
die("Device Plane installed network-publication Compose mismatch")
descriptor = read_strict_json(
component_root("device-plane")
/ DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
"installed Device Plane network-publication descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_foundation_network_publication_descriptor()
):
die(
"installed Device Plane network-publication descriptor mismatch"
)
recovery_descriptor = (
component_root("device-plane")
/ DEVICE_PLANE_FOUNDATION_RECOVERY_REL
)
if recovery_descriptor.exists() or recovery_descriptor.is_symlink():
die("terminal Device Plane recovery descriptor is installed")
return actual
def component_compose_root(component):
return COMPONENTS[component].get("compose_root", component_root(component))
@@ -10366,6 +10913,14 @@ def component_services(component, entries=None):
# runtime without build/recreate/restart.
return ()
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
# Recreate only the two stateless services from their exact existing
# image IDs. PostgreSQL is a preserved prerequisite, never selected.
return ("device-control-core", "device-gateway")
if is_engine_l2_closed_loop_slice(component, entries):
# The failed 030 apply published both the backend source and the built
# UI before Compose rejected the descriptor/source mismatch. The exact
@@ -10703,6 +11258,12 @@ def component_builds(component, entries=None):
if is_device_plane_foundation_recovery_slice(component, entries):
return ()
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
return ()
if component == "device-plane" and entries is not None:
selected_services = component_services(component, entries)
builds = []
@@ -12675,6 +13236,7 @@ def plan_artifact(artifact):
mcp_autonomy_provider_v5_preflight = None
l2_closed_loop_preflight = None
device_plane_foundation_recovery_preflight = None
device_plane_network_publication_preflight = None
device_plane_runtime_before = None
composite_provider_v4_preflight = None
provider_rotating_slot_preflight = None
@@ -12849,6 +13411,15 @@ def plan_artifact(artifact):
payload_dir
)
)
if is_device_plane_foundation_network_publication_slice(
manifest["component"],
entries,
):
device_plane_network_publication_preflight = (
validate_device_plane_foundation_network_publication_evidence(
payload_dir
)
)
component = manifest["component"]
root = component_root(component)
@@ -14257,6 +14828,53 @@ def plan_artifact(artifact):
)
print("device_plane_source_action=publish-exact-failed-source")
print("device_plane_rollback=source-only-runtime-unchanged")
if device_plane_network_publication_preflight is not None:
print(
"device_plane_transition="
f"{device_plane_network_publication_preflight['mode']}"
)
print(
"failed_recovery_patch="
f"{DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID}"
)
print(
"failed_recovery_artifact_sha256="
f"{DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256}"
)
print(
"failed_recovery_backup="
f"{device_plane_network_publication_preflight['backup'].name}"
)
print("device_plane_build=none")
print(
"device_plane_runtime_mutation="
"recreate:device-control-core,device-gateway"
)
print(
"device_plane_runtime_services="
"preserved:device-postgres"
)
print(
"device_plane_private_network="
"preserved:internal:true"
)
print(
"device_plane_control_network="
"create:internal:false:masquerade:false"
)
print(
"device_plane_actual_ports="
"required:127.0.0.1:18120,127.0.0.1:18121"
)
print("device_gateway_tcp_9921=disabled:unpublished")
print(
"device_plane_source_action="
"publish-network-corrected-foundation-source"
)
print(
"device_plane_rollback="
"partial-source+internal-only-stateless-runtime"
)
if device_plane_postgres_preflight is not None:
print(
"device_postgres_bootstrap="
@@ -14545,6 +15163,83 @@ def rollback_platform_apply(root, backup_dir, entries, current_stamp, runtime_st
return f"source+runtime-restored:{restored_count}"
def remove_device_plane_control_network_if_unused():
network = inspect_device_plane_network_optional(
DEVICE_PLANE_CONTROL_NETWORK
)
if network is None:
return "absent"
if (
network.get("Name") != DEVICE_PLANE_CONTROL_NETWORK
or network.get("Driver") != "bridge"
or network.get("Internal") is not False
or (network.get("Containers") or {})
or (network.get("Options") or {}).get(
"com.docker.network.bridge.enable_ip_masquerade"
)
!= "false"
):
die("Device Plane control network is unsafe to remove")
subprocess.run(
[
str(DOCKER),
"network",
"rm",
DEVICE_PLANE_CONTROL_NETWORK,
],
check=True,
)
if inspect_device_plane_network_optional(DEVICE_PLANE_CONTROL_NETWORK):
die("Device Plane control network removal failed")
return "removed"
def rollback_device_plane_network_publication_apply(
root,
backup_dir,
entries,
current_stamp,
runtime_started,
applied_services,
):
if tuple(applied_services) != (
"device-control-core",
"device-gateway",
):
die("Device Plane network-publication rollback service mismatch")
if runtime_started:
stop_and_remove_compose_services(
"device-plane",
applied_services,
)
restored_count = restore_platform_overlay(
root,
backup_dir,
entries,
current_stamp,
)
remove_device_plane_control_network_if_unused()
run_compose(
"device-plane",
applied_services,
("docker-compose.device-plane.yml",),
)
for service in DEVICE_PLANE_RUNTIME_SERVICES:
healthcheck_compose_service("device-plane", service)
runtime = validate_device_plane_foundation_runtime()
if (
runtime["device-postgres"]["containerId"]
!= DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[
"device-postgres"
]
):
die("Device Plane rollback changed PostgreSQL generation")
assert_loopback_tcp_port_closed(18120)
assert_loopback_tcp_port_closed(18121)
assert_loopback_tcp_port_closed(9921)
return f"source+internal-runtime-restored:{restored_count}"
def rollback_device_plane_apply(
root,
backup_dir,
@@ -15313,6 +16008,13 @@ def run_component_runtime(component, entries, services):
return
if is_device_plane_foundation_recovery_slice(component, entries):
return
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
prepare_component_runtime(component, entries)
run_compose(component, services, entries)
return
run_build(component, entries)
prepare_component_runtime(component, entries)
if is_engine_node_intelligence_transition(component, entries):
@@ -15365,6 +16067,15 @@ def healthcheck_url(check):
die(f"healthcheck failed for {url}: {last_error}")
def assert_loopback_tcp_port_closed(port):
try:
connection = socket.create_connection(("127.0.0.1", port), timeout=3)
except OSError:
return
connection.close()
die(f"unexpected loopback TCP listener is open: {port}")
def external_data_plane_healthcheck(require_managed=True):
expected_json = {
"ok": True,
@@ -15896,6 +16607,31 @@ def run_healthchecks(component, entries=None, services=None):
validate_device_plane_foundation_installed_source()
validate_device_plane_foundation_runtime()
return
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
if tuple(services or ()) != (
"device-control-core",
"device-gateway",
):
die(
"Device Plane network-publication service set mismatch"
)
for service in (
"device-control-core",
"device-gateway",
"device-postgres",
):
healthcheck_compose_service("device-plane", service)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
validate_device_plane_foundation_network_publication_installed_source()
validate_device_plane_foundation_runtime(
network_publication=True
)
assert_loopback_tcp_port_closed(9921)
return
if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries):
return
if is_engine_l2_closed_loop_slice(component, entries):
@@ -16706,6 +17442,13 @@ def apply_artifact(artifact):
payload_dir
)
)
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
validate_device_plane_foundation_network_publication_evidence(
payload_dir
)
if not root.is_dir():
if bootstrap_root:
root.mkdir(parents=True, exist_ok=True)
@@ -17289,6 +18032,42 @@ def apply_artifact(artifact):
except Exception as rollback_exc:
rollback_status = f"failed:{type(rollback_exc).__name__}"
print("platform-automatic-rollback=failed", file=sys.stderr)
elif (
is_device_plane_foundation_network_publication_slice(
component,
entries,
)
and services is not None
):
try:
restored_state = (
rollback_device_plane_network_publication_apply(
root,
backup_dir,
entries,
current_stamp,
runtime_started,
services,
)
)
rollback_status = (
"ok:device-plane-network-publication:"
f"{restored_state}"
)
print(
"device-plane-network-publication-"
f"automatic-rollback={rollback_status}",
file=sys.stderr,
)
except Exception as rollback_exc:
rollback_status = (
f"failed:{type(rollback_exc).__name__}"
)
print(
"device-plane-network-publication-"
"automatic-rollback=failed",
file=sys.stderr,
)
elif (
component == "device-plane"
and entries is not None