SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(device-plane): reconcile failed manager rollout

Browse Source
This commit is contained in:
Codex
2026-08-11 00:32:35 +03:00
parent 4374a9b4e7
commit 04ba3a9f99
5 changed files with 918 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+495 -12
View File
@@ -230,6 +230,61 @@ DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES = (
"services/device-manager",
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
)
DEVICE_PLANE_MANAGER_RECONCILIATION_REL = (
"deployment/device-manager-control-plane-reconciliation-v1.json"
)
DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES = (
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
)
DEVICE_PLANE_MANAGER_FAILED_PATCH_ID = (
"device-manager-control-plane-20260810-001"
)
DEVICE_PLANE_MANAGER_FAILED_ARTIFACT = (
"nodedc-device-plane-device-manager-control-plane-20260810-001.tgz."
"20260811-000321"
)
DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256 = (
"50e275c1085286bcb3bb2b273aefc8bbba70f446ca2c7bd464dc745710a291a6"
)
DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID = (
"device-plane-device-manager-control-plane-20260810-001-"
"20260811-000321"
)
DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_SHA256 = {
"manifest.env": (
"70c8e6514f0a41cdd8892e29ca74c311c68e31bb74a7653d9b37e790db968526"
),
"files.txt": (
"2101d48cfb832f50e6112e443b4b61813e9e6d94c8171ca66fb23ed7f51bc575"
),
"existing-files.txt": (
"b605a86ce5ed6c0adb7b4cbeb5758795fe4219367df2c18db9411825f232e308"
),
"missing-files.txt": (
"84c546b7ddb0cd8c80b3de29f583ec48678bc9bb47ad0460860dc0022a6b87e9"
),
"source-before.tgz": (
"df6f29d173a56b0a067358d4b763c2de280450cf597957d1e79b8ae168e57735"
),
"runtime-before.json": (
"590310c9ad923516a714c7d47ba26eee2d82113cb800a5a7bff29a7de94f165d"
),
}
DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING = (
".dockerignore",
"package.json",
"package-lock.json",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway/package.json",
"services/device-edge-relay/package.json",
)
DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING = (
DEVICE_PLANE_MANAGER_COMPOSE_REL,
"services/device-manager",
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
)
DEVICE_PLANE_FOUNDATION_RECOVERY_REL = (
"deployment/device-plane-foundation-recovery-v1.json"
)
@@ -3115,6 +3170,7 @@ def allowed_payload_path(component, rel):
DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL,
DEVICE_PLANE_BACKHAUL_TARGET_REL,
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
@@ -8160,6 +8216,11 @@ def load_artifact(artifact, work_dir):
entries,
):
validate_device_plane_manager_control_plane_payload(payload_dir)
if is_device_plane_manager_reconciliation_slice(
manifest["component"],
entries,
):
validate_device_plane_manager_reconciliation_payload(payload_dir)
if is_device_plane_postgres_bootstrap_slice(
manifest["component"],
entries,
@@ -8494,6 +8555,17 @@ def reject_terminal_device_plane_foundation_artifact(manifest, sha256):
)
def reject_terminal_device_plane_manager_artifact(manifest, sha256):
if (
manifest.get("id") == DEVICE_PLANE_MANAGER_FAILED_PATCH_ID
or sha256 == DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
):
die(
"Device Manager control-plane 001 is terminal failed; "
"use the exact registered reconciliation successor"
)
def reject_terminal_device_plane_backhaul_artifact(manifest, sha256):
if (
manifest.get("id") == DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID
@@ -8555,6 +8627,14 @@ def is_device_plane_manager_control_plane_slice(component, entries):
)
def is_device_plane_manager_reconciliation_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries) == DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES
)
def expected_platform_device_core_hub_trust_descriptor():
return {
"schemaVersion": "nodedc.platform.device-core-hub-trust.v1",
@@ -8592,6 +8672,35 @@ def expected_device_plane_manager_control_plane_descriptor():
}
def expected_device_plane_manager_reconciliation_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane.device-manager-control-plane-"
"reconciliation.v1"
),
"mode": "failed-control-plane-baseline-adoption",
"failedPatchId": DEVICE_PLANE_MANAGER_FAILED_PATCH_ID,
"failedArtifactSha256": (
DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
),
"backupId": DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID,
"sourceAction": "publish-reconciliation-marker-only",
"runtimeAction": "read-only-acceptance",
"preservedServices": [
"device-control-core",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
"absentService": "device-manager",
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
"publicIngress": "disabled",
"commandTransport": "disabled",
"gelios": "untouched",
"rollback": "marker-only-runtime-unchanged",
}
def validate_platform_device_core_hub_trust_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
@@ -8673,6 +8782,17 @@ def validate_device_plane_manager_control_plane_payload(payload_dir):
return descriptor
def validate_device_plane_manager_reconciliation_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
"Device Manager control-plane reconciliation descriptor",
max_bytes=16 * 1024,
)
if descriptor != expected_device_plane_manager_reconciliation_descriptor():
die("Device Manager control-plane reconciliation descriptor mismatch")
return descriptor
def is_device_plane_postgres_bootstrap_slice(component, entries):
return (
component == "device-plane"
@@ -9895,6 +10015,207 @@ def validate_device_plane_b2_discovery_rollback_recovery_evidence(
}
def validate_device_plane_manager_reconciliation_backup():
backup_dir = (
BACKUPS_DIR / DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID
)
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Device Manager failed-apply backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Device Manager failed-apply backup is unsafe")
expected = DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_SHA256
if {child.name for child in backup_dir.iterdir()} != set(expected):
die("Device Manager failed-apply backup file set mismatch")
for name, expected_sha256 in expected.items():
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected_sha256
):
die(
"Device Manager failed-apply backup drift detected: "
f"{name}"
)
existing = tuple(read_backup_path_list(
backup_dir / "existing-files.txt"
))
missing = tuple(read_backup_path_list(
backup_dir / "missing-files.txt"
))
validate_backup_partition(
DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES,
existing,
missing,
"Device Manager reconciliation",
)
if (
existing != DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING
or missing != DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING
):
die("Device Manager failed-apply backup partition mismatch")
return backup_dir
def validate_device_plane_manager_reconciled_baseline(
backup_dir,
*,
marker_installed,
):
root = component_root("device-plane")
with tempfile.TemporaryDirectory(
prefix="device-manager-reconciled-baseline-",
dir=TMP_DIR,
) as directory:
backup_root = Path(directory)
materialize_backup_tree(
backup_dir / "source-before.tgz",
backup_root,
set(DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING),
)
backup_source = collect_exact_files(
backup_root,
DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING,
"Device Manager pre-apply source",
)
live_source = collect_exact_files(
root,
DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING,
"Device Manager reconciled live source",
)
if live_source != backup_source:
die("Device Manager rollback source does not match backup")
for rel in DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING:
path = root / rel
if path.exists() or path.is_symlink():
die(
"Device Manager candidate-only source remains installed: "
f"{rel}"
)
marker = root / DEVICE_PLANE_MANAGER_RECONCILIATION_REL
if marker_installed:
descriptor = read_strict_json(
marker,
"installed Device Manager reconciliation descriptor",
max_bytes=16 * 1024,
)
if descriptor != expected_device_plane_manager_reconciliation_descriptor():
die("installed Device Manager reconciliation descriptor mismatch")
elif marker.exists() or marker.is_symlink():
die("Device Manager reconciliation descriptor already installed")
if device_plane_service_container_ids("device-manager"):
die("Device Manager candidate container remains installed")
runtime_before = read_strict_json(
backup_dir / "runtime-before.json",
"Device Manager pre-apply runtime inventory",
max_bytes=64 * 1024,
)
runtime = validate_device_plane_b2_discovery_ingress_runtime(
runtime_before,
preserved_stateless_services=("device-gateway",),
)
core_ids = device_plane_service_container_ids("device-control-core")
if len(core_ids) != 1:
die("Device Manager reconciled Core service count mismatch")
core = inspect_device_plane_container(core_ids[0])
core_environment = container_environment(
core,
"Device Manager reconciled Control Core",
)
if (
"DEVICE_MANAGEMENT_API_ENABLED" in core_environment
or "DEVICE_MANAGEMENT_CORE_TOKEN_FILE" in core_environment
or any(
mount.get("Destination")
== "/run/nodedc-secrets/management-core-token"
for mount in core.get("Mounts") or []
)
):
die("Device Manager management boundary remains active after rollback")
return runtime
def validate_device_plane_manager_reconciliation_evidence(payload_dir):
descriptor = validate_device_plane_manager_reconciliation_payload(
payload_dir
)
backup_dir = validate_device_plane_manager_reconciliation_backup()
failed_artifact = FAILED_DIR / DEVICE_PLANE_MANAGER_FAILED_ARTIFACT
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Manager failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
):
die("Device Manager failed artifact evidence mismatch")
records = [
value
for value in load_state(FAILED_STATE_FILE)
if value.get("id") == DEVICE_PLANE_MANAGER_FAILED_PATCH_ID
]
if len(records) != 1:
die("Device Manager failed journal evidence count mismatch")
record = records[0]
if (
record.get("artifact") != DEVICE_PLANE_MANAGER_FAILED_ARTIFACT
or record.get("backup_id")
!= DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID
or record.get("component") != "device-plane"
or record.get("sha256")
!= DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status") != "failed:DeployError"
or record.get("status") != "failed"
or record.get("message")
!= (
"container healthcheck failed for "
"500bc061b97d9007a33dd51d35ca1168eae7f6b019165b143d3bd908220656c0: "
"unhealthy"
)
):
die("Device Manager failed journal evidence mismatch")
with tempfile.TemporaryDirectory(
prefix="device-manager-failed-artifact-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, _failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id") != DEVICE_PLANE_MANAGER_FAILED_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES
):
die("Device Manager failed artifact contract mismatch")
runtime = validate_device_plane_manager_reconciled_baseline(
backup_dir,
marker_installed=False,
)
return {
"mode": descriptor["mode"],
"backup": backup_dir,
"failedArtifact": failed_artifact,
"runtime": runtime,
}
def device_plane_service_container_ids(service):
if service not in (
*DEVICE_PLANE_RUNTIME_SERVICES,
@@ -10803,8 +11124,17 @@ def validate_device_plane_foundation_runtime(network_publication=False):
return accepted
def validate_device_plane_b2_discovery_ingress_runtime(runtime_before):
def validate_device_plane_b2_discovery_ingress_runtime(
runtime_before,
preserved_stateless_services=(),
):
validate_device_plane_runtime_secret_metadata()
preserved_stateless = set(preserved_stateless_services)
if not preserved_stateless.issubset({
"device-control-core",
"device-gateway",
}):
die("Device Plane B2 preserved stateless service set mismatch")
before_names = device_plane_inventory_service_names(runtime_before)
if set(before_names) != set(DEVICE_PLANE_RUNTIME_SERVICES):
die("Device Plane B2 ingress predecessor inventory mismatch")
@@ -10903,11 +11233,20 @@ def validate_device_plane_b2_discovery_ingress_runtime(runtime_before):
):
die("Device Plane B2 ingress changed PostgreSQL generation")
for service in ("device-control-core", "device-gateway"):
if (
same_container = (
accepted[service]["containerId"]
== before[service]["containerId"]
or accepted[service]["imageId"] == before[service]["imageId"]
):
)
same_image = (
accepted[service]["imageId"] == before[service]["imageId"]
)
if service in preserved_stateless:
if not (same_container and same_image):
die(
"Device Plane B2 stateless generation was not preserved: "
f"{service}"
)
elif same_container or same_image:
die(
"Device Plane B2 ingress stateless generation not replaced: "
f"{service}"
@@ -13061,6 +13400,11 @@ def component_services(component, entries=None):
# failed archive, journal, backup, restored source and live runtime.
return ()
if is_device_plane_manager_reconciliation_slice(component, entries):
# Reconciliation publishes one marker after proving the failed
# evidence and the already-restored baseline. Runtime is read-only.
return ()
if is_device_plane_postgres_bootstrap_slice(component, entries):
# This exact one-time transition is the only Device Plane artifact that
# may select durable state. Its preflight requires both container and
@@ -13476,6 +13820,9 @@ def component_builds(component, entries=None):
),
)
if is_device_plane_manager_reconciliation_slice(component, entries):
return ()
if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
return ()
@@ -15498,6 +15845,7 @@ def plan_artifact(artifact):
device_plane_network_publication_preflight = None
device_plane_b2_ingress_preflight = None
device_plane_b2_recovery_preflight = None
device_plane_manager_reconciliation_preflight = None
device_plane_backhaul_preflight = None
device_plane_backhaul_vps_enrollment_preflight = None
device_plane_runtime_before = None
@@ -15525,6 +15873,7 @@ def plan_artifact(artifact):
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
reject_terminal_engine_l2_failed_artifact(manifest, sha)
reject_terminal_device_plane_foundation_artifact(manifest, sha)
reject_terminal_device_plane_manager_artifact(manifest, sha)
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
transition_descriptor = None
transition_preflight = None
@@ -15702,6 +16051,15 @@ def plan_artifact(artifact):
payload_dir
)
)
if is_device_plane_manager_reconciliation_slice(
manifest["component"],
entries,
):
device_plane_manager_reconciliation_preflight = (
validate_device_plane_manager_reconciliation_evidence(
payload_dir
)
)
if is_device_plane_backhaul_target_slice(
manifest["component"],
entries,
@@ -17255,6 +17613,38 @@ def plan_artifact(artifact):
)
print("device_gateway_tcp_9921=disabled:unpublished")
print("device_plane_rollback=marker-only-runtime-unchanged")
if device_plane_manager_reconciliation_preflight is not None:
print(
"device_plane_transition="
f"{device_plane_manager_reconciliation_preflight['mode']}"
)
print(
"failed_patch="
f"{DEVICE_PLANE_MANAGER_FAILED_PATCH_ID}"
)
print(
"failed_artifact_sha256="
f"{DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256}"
)
print(
"recovery_backup="
f"{device_plane_manager_reconciliation_preflight['backup'].name}"
)
print("device_plane_build=none")
print("device_plane_runtime_mutation=none")
print(
"device_plane_runtime_services="
"preserved:device-control-core,device-gateway,"
"device-postgres,device-backhaul-target"
)
print("device_manager=absent")
print(
"device_plane_source_action="
"publish-reconciliation-marker-only"
)
print("device_gateway_tcp_9921=preserved:loopback-only")
print("device_manager_public_route=unchanged:absent")
print("device_plane_rollback=marker-only-runtime-unchanged")
if device_plane_backhaul_vps_enrollment_preflight is not None:
print(
"device_plane_transition="
@@ -17867,11 +18257,24 @@ def rollback_device_plane_apply(
baseline_entries,
baseline_services,
)
run_healthchecks(
"device-plane",
baseline_entries,
baseline_services,
)
if is_device_plane_manager_control_plane_slice("device-plane", entries):
for service in baseline_services:
healthcheck_compose_service_with_grace(
"device-plane",
service,
)
for check in component_healthchecks(
"device-plane",
baseline_entries,
baseline_services,
):
healthcheck_url(check)
else:
run_healthchecks(
"device-plane",
baseline_entries,
baseline_services,
)
return f"source+runtime-restored:{restored_count}"
@@ -18755,6 +19158,8 @@ def run_component_runtime(component, entries, services):
entries,
):
return
if is_device_plane_manager_reconciliation_slice(component, entries):
return
if is_device_plane_foundation_recovery_slice(component, entries):
return
if is_device_plane_foundation_network_publication_slice(
@@ -18782,8 +19187,9 @@ def run_device_plane_runtime_for_apply(
"device-plane",
entries,
) or is_device_plane_foundation_recovery_slice(
"device-plane",
entries,
"device-plane", entries
) or is_device_plane_manager_reconciliation_slice(
"device-plane", entries
):
return
# Build and runtime preparation are pre-runtime phases. A failure here
@@ -19187,6 +19593,46 @@ def healthcheck_container(container_name):
die(f"container healthcheck failed for {container_name}: {last_status}")
def healthcheck_container_with_grace(container_name):
"""Wait through bounded transient unhealthy/exited states.
Device Manager activation and its baseline rollback both recreate a Core
generation while Docker health probes and restart policy converge. The
ordinary helper deliberately fails on the first terminal-looking state;
this exact transition instead requires a healthy result within the same
bounded five-minute window.
"""
last_status = "unknown"
for attempt in range(1, 61):
result = subprocess.run(
[
str(DOCKER),
"inspect",
"--format",
"{{if .State.Health}}{{.State.Health.Status}}"
"{{else}}{{.State.Status}}{{end}}",
container_name,
],
check=False,
capture_output=True,
text=True,
)
status_text = result.stdout.strip().lower()
last_status = (
status_text
or result.stderr.strip()
or f"inspect-exit-{result.returncode}"
)
if status_text in ("healthy", "running"):
return
if attempt < 60:
time.sleep(5)
die(
"container healthcheck grace exhausted for "
f"{container_name}: {last_status}"
)
def compose_service_container_id(component, service):
result = subprocess.run(
[*compose_base_cmd(component), "ps", "-q", service],
@@ -19209,6 +19655,12 @@ def healthcheck_compose_service(component, service):
healthcheck_container(compose_service_container_id(component, service))
def healthcheck_compose_service_with_grace(component, service):
healthcheck_container_with_grace(
compose_service_container_id(component, service)
)
def container_environment(container, label):
environment = {}
for raw in (container.get("Config") or {}).get("Env") or []:
@@ -19467,6 +19919,23 @@ process.stdout.write('engine-l2-closed-loop:0.7.0:cas+safe-profile+external-plan
def run_healthchecks(component, entries=None, services=None):
if is_device_plane_manager_reconciliation_slice(component, entries):
if tuple(services or ()) != ():
die("Device Manager reconciliation service set mismatch")
for service in DEVICE_PLANE_RUNTIME_SERVICES:
healthcheck_compose_service_with_grace(
"device-plane",
service,
)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
backup_dir = validate_device_plane_manager_reconciliation_backup()
validate_device_plane_manager_reconciled_baseline(
backup_dir,
marker_installed=True,
)
return
if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
if tuple(services or ()) != (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,):
die("Device Plane VPS enrollment service set mismatch")
@@ -19583,7 +20052,10 @@ def run_healthchecks(component, entries=None, services=None):
):
die("Device Manager control-plane service set mismatch")
for service in services:
healthcheck_compose_service("device-plane", service)
healthcheck_compose_service_with_grace(
"device-plane",
service,
)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
validate_device_manager_control_plane_runtime()
@@ -20374,6 +20846,10 @@ def apply_artifact(artifact):
manifest,
sha,
)
reject_terminal_device_plane_manager_artifact(
manifest,
sha,
)
reject_terminal_device_plane_backhaul_artifact(
manifest,
sha,
@@ -20425,6 +20901,13 @@ def apply_artifact(artifact):
validate_device_plane_b2_discovery_rollback_recovery_evidence(
payload_dir
)
if is_device_plane_manager_reconciliation_slice(
component,
entries,
):
validate_device_plane_manager_reconciliation_evidence(
payload_dir
)
if is_device_plane_backhaul_target_slice(
component,
entries,