From 04ba3a9f99b7de9e9bb4ac8b3db131b72065bd66 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 11 Aug 2026 00:32:35 +0300 Subject: [PATCH] fix(device-plane): reconcile failed manager rollout --- ...nager-control-plane-reconciliation-v1.json | 21 + ...-control-plane-reconciliation-artifact.mjs | 111 ++++ infra/deploy-runner/nodedc-deploy | 507 +++++++++++++++++- ..._device_manager_control_plane_artifacts.py | 22 +- ...r_control_plane_reconciliation_artifact.py | 273 ++++++++++ 5 files changed, 918 insertions(+), 16 deletions(-) create mode 100644 device-plane/deployment/device-manager-control-plane-reconciliation-v1.json create mode 100644 infra/deploy-runner/build-device-manager-control-plane-reconciliation-artifact.mjs create mode 100644 infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py diff --git a/device-plane/deployment/device-manager-control-plane-reconciliation-v1.json b/device-plane/deployment/device-manager-control-plane-reconciliation-v1.json new file mode 100644 index 0000000..d34ac3f --- /dev/null +++ b/device-plane/deployment/device-manager-control-plane-reconciliation-v1.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": "nodedc.device-plane.device-manager-control-plane-reconciliation.v1", + "mode": "failed-control-plane-baseline-adoption", + "failedPatchId": "device-manager-control-plane-20260810-001", + "failedArtifactSha256": "50e275c1085286bcb3bb2b273aefc8bbba70f446ca2c7bd464dc745710a291a6", + "backupId": "device-plane-device-manager-control-plane-20260810-001-20260811-000321", + "sourceAction": "publish-reconciliation-marker-only", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres", + "device-backhaul-target" + ], + "absentService": "device-manager", + "databaseVolume": "nodedc-device-plane-postgres-data", + "publicIngress": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "rollback": "marker-only-runtime-unchanged" +} diff --git a/infra/deploy-runner/build-device-manager-control-plane-reconciliation-artifact.mjs b/infra/deploy-runner/build-device-manager-control-plane-reconciliation-artifact.mjs new file mode 100644 index 0000000..ab7edcb --- /dev/null +++ b/infra/deploy-runner/build-device-manager-control-plane-reconciliation-artifact.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const sourceRoot = resolve(platformRoot, "device-plane"); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-manager-control-plane-reconciliation-20260811-002", + ...extra +] = process.argv.slice(2); +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-manager-control-plane-reconciliation-artifact.mjs [patch-id]", + ); +} + +const entry = "deployment/device-manager-control-plane-reconciliation-v1.json"; +const stage = await mkdtemp(join(tmpdir(), "nodedc-device-manager-reconciliation-")); +const payload = join(stage, "payload"); +const target = resolve( + artifactDir, + `nodedc-device-plane-${patchId}.tgz`, +); + +try { + const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entry), "utf8")); + const expected = { + schemaVersion: "nodedc.device-plane.device-manager-control-plane-reconciliation.v1", + mode: "failed-control-plane-baseline-adoption", + failedPatchId: "device-manager-control-plane-20260810-001", + failedArtifactSha256: + "50e275c1085286bcb3bb2b273aefc8bbba70f446ca2c7bd464dc745710a291a6", + backupId: + "device-plane-device-manager-control-plane-20260810-001-20260811-000321", + sourceAction: "publish-reconciliation-marker-only", + runtimeAction: "read-only-acceptance", + preservedServices: [ + "device-control-core", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + absentService: "device-manager", + databaseVolume: "nodedc-device-plane-postgres-data", + publicIngress: "disabled", + commandTransport: "disabled", + gelios: "untouched", + rollback: "marker-only-runtime-unchanged", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error("device_manager_reconciliation_descriptor_mismatch"); + } + + await mkdir(dirname(join(payload, entry)), { recursive: true }); + await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true }); + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${entry}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, + ); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + const sha256 = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + component: "device-plane", + artifact: target, + sha256, + entries: [entry], + build: [], + services: [], + transition: "failed-control-plane-baseline-adoption", + runtimeAction: "read-only-acceptance", + sourceAction: "publish-reconciliation-marker-only", + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index a4eae80..bd01218 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -230,6 +230,61 @@ DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES = ( "services/device-manager", DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL, ) +DEVICE_PLANE_MANAGER_RECONCILIATION_REL = ( + "deployment/device-manager-control-plane-reconciliation-v1.json" +) +DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES = ( + DEVICE_PLANE_MANAGER_RECONCILIATION_REL, +) +DEVICE_PLANE_MANAGER_FAILED_PATCH_ID = ( + "device-manager-control-plane-20260810-001" +) +DEVICE_PLANE_MANAGER_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-manager-control-plane-20260810-001.tgz." + "20260811-000321" +) +DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256 = ( + "50e275c1085286bcb3bb2b273aefc8bbba70f446ca2c7bd464dc745710a291a6" +) +DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID = ( + "device-plane-device-manager-control-plane-20260810-001-" + "20260811-000321" +) +DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_SHA256 = { + "manifest.env": ( + "70c8e6514f0a41cdd8892e29ca74c311c68e31bb74a7653d9b37e790db968526" + ), + "files.txt": ( + "2101d48cfb832f50e6112e443b4b61813e9e6d94c8171ca66fb23ed7f51bc575" + ), + "existing-files.txt": ( + "b605a86ce5ed6c0adb7b4cbeb5758795fe4219367df2c18db9411825f232e308" + ), + "missing-files.txt": ( + "84c546b7ddb0cd8c80b3de29f583ec48678bc9bb47ad0460860dc0022a6b87e9" + ), + "source-before.tgz": ( + "df6f29d173a56b0a067358d4b763c2de280450cf597957d1e79b8ae168e57735" + ), + "runtime-before.json": ( + "590310c9ad923516a714c7d47ba26eee2d82113cb800a5a7bff29a7de94f165d" + ), +} +DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING = ( + ".dockerignore", + "package.json", + "package-lock.json", + "packages/device-protocol-contract", + "packages/arusnavi-b2-adapter", + "services/device-control-core", + "services/device-gateway/package.json", + "services/device-edge-relay/package.json", +) +DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING = ( + DEVICE_PLANE_MANAGER_COMPOSE_REL, + "services/device-manager", + DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL, +) DEVICE_PLANE_FOUNDATION_RECOVERY_REL = ( "deployment/device-plane-foundation-recovery-v1.json" ) @@ -3115,6 +3170,7 @@ def allowed_payload_path(component, rel): DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL, DEVICE_PLANE_BACKHAUL_TARGET_REL, DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL, + DEVICE_PLANE_MANAGER_RECONCILIATION_REL, "packages/device-protocol-contract", "packages/arusnavi-b2-adapter", "services/device-control-core", @@ -8160,6 +8216,11 @@ def load_artifact(artifact, work_dir): entries, ): validate_device_plane_manager_control_plane_payload(payload_dir) + if is_device_plane_manager_reconciliation_slice( + manifest["component"], + entries, + ): + validate_device_plane_manager_reconciliation_payload(payload_dir) if is_device_plane_postgres_bootstrap_slice( manifest["component"], entries, @@ -8494,6 +8555,17 @@ def reject_terminal_device_plane_foundation_artifact(manifest, sha256): ) +def reject_terminal_device_plane_manager_artifact(manifest, sha256): + if ( + manifest.get("id") == DEVICE_PLANE_MANAGER_FAILED_PATCH_ID + or sha256 == DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256 + ): + die( + "Device Manager control-plane 001 is terminal failed; " + "use the exact registered reconciliation successor" + ) + + def reject_terminal_device_plane_backhaul_artifact(manifest, sha256): if ( manifest.get("id") == DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID @@ -8555,6 +8627,14 @@ def is_device_plane_manager_control_plane_slice(component, entries): ) +def is_device_plane_manager_reconciliation_slice(component, entries): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) == DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES + ) + + def expected_platform_device_core_hub_trust_descriptor(): return { "schemaVersion": "nodedc.platform.device-core-hub-trust.v1", @@ -8592,6 +8672,35 @@ def expected_device_plane_manager_control_plane_descriptor(): } +def expected_device_plane_manager_reconciliation_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane.device-manager-control-plane-" + "reconciliation.v1" + ), + "mode": "failed-control-plane-baseline-adoption", + "failedPatchId": DEVICE_PLANE_MANAGER_FAILED_PATCH_ID, + "failedArtifactSha256": ( + DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256 + ), + "backupId": DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID, + "sourceAction": "publish-reconciliation-marker-only", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + "absentService": "device-manager", + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "publicIngress": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "rollback": "marker-only-runtime-unchanged", + } + + def validate_platform_device_core_hub_trust_payload(payload_dir): descriptor = read_strict_json( payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL, @@ -8673,6 +8782,17 @@ def validate_device_plane_manager_control_plane_payload(payload_dir): return descriptor +def validate_device_plane_manager_reconciliation_payload(payload_dir): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_MANAGER_RECONCILIATION_REL, + "Device Manager control-plane reconciliation descriptor", + max_bytes=16 * 1024, + ) + if descriptor != expected_device_plane_manager_reconciliation_descriptor(): + die("Device Manager control-plane reconciliation descriptor mismatch") + return descriptor + + def is_device_plane_postgres_bootstrap_slice(component, entries): return ( component == "device-plane" @@ -9895,6 +10015,207 @@ def validate_device_plane_b2_discovery_rollback_recovery_evidence( } +def validate_device_plane_manager_reconciliation_backup(): + backup_dir = ( + BACKUPS_DIR / DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID + ) + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Manager failed-apply backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Manager failed-apply backup is unsafe") + expected = DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_SHA256 + if {child.name for child in backup_dir.iterdir()} != set(expected): + die("Device Manager failed-apply backup file set mismatch") + for name, expected_sha256 in expected.items(): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Device Manager failed-apply backup drift detected: " + f"{name}" + ) + existing = tuple(read_backup_path_list( + backup_dir / "existing-files.txt" + )) + missing = tuple(read_backup_path_list( + backup_dir / "missing-files.txt" + )) + validate_backup_partition( + DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES, + existing, + missing, + "Device Manager reconciliation", + ) + if ( + existing != DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING + or missing != DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING + ): + die("Device Manager failed-apply backup partition mismatch") + return backup_dir + + +def validate_device_plane_manager_reconciled_baseline( + backup_dir, + *, + marker_installed, +): + root = component_root("device-plane") + with tempfile.TemporaryDirectory( + prefix="device-manager-reconciled-baseline-", + dir=TMP_DIR, + ) as directory: + backup_root = Path(directory) + materialize_backup_tree( + backup_dir / "source-before.tgz", + backup_root, + set(DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING), + ) + backup_source = collect_exact_files( + backup_root, + DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING, + "Device Manager pre-apply source", + ) + live_source = collect_exact_files( + root, + DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING, + "Device Manager reconciled live source", + ) + if live_source != backup_source: + die("Device Manager rollback source does not match backup") + for rel in DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING: + path = root / rel + if path.exists() or path.is_symlink(): + die( + "Device Manager candidate-only source remains installed: " + f"{rel}" + ) + + marker = root / DEVICE_PLANE_MANAGER_RECONCILIATION_REL + if marker_installed: + descriptor = read_strict_json( + marker, + "installed Device Manager reconciliation descriptor", + max_bytes=16 * 1024, + ) + if descriptor != expected_device_plane_manager_reconciliation_descriptor(): + die("installed Device Manager reconciliation descriptor mismatch") + elif marker.exists() or marker.is_symlink(): + die("Device Manager reconciliation descriptor already installed") + + if device_plane_service_container_ids("device-manager"): + die("Device Manager candidate container remains installed") + runtime_before = read_strict_json( + backup_dir / "runtime-before.json", + "Device Manager pre-apply runtime inventory", + max_bytes=64 * 1024, + ) + runtime = validate_device_plane_b2_discovery_ingress_runtime( + runtime_before, + preserved_stateless_services=("device-gateway",), + ) + core_ids = device_plane_service_container_ids("device-control-core") + if len(core_ids) != 1: + die("Device Manager reconciled Core service count mismatch") + core = inspect_device_plane_container(core_ids[0]) + core_environment = container_environment( + core, + "Device Manager reconciled Control Core", + ) + if ( + "DEVICE_MANAGEMENT_API_ENABLED" in core_environment + or "DEVICE_MANAGEMENT_CORE_TOKEN_FILE" in core_environment + or any( + mount.get("Destination") + == "/run/nodedc-secrets/management-core-token" + for mount in core.get("Mounts") or [] + ) + ): + die("Device Manager management boundary remains active after rollback") + return runtime + + +def validate_device_plane_manager_reconciliation_evidence(payload_dir): + descriptor = validate_device_plane_manager_reconciliation_payload( + payload_dir + ) + backup_dir = validate_device_plane_manager_reconciliation_backup() + failed_artifact = FAILED_DIR / DEVICE_PLANE_MANAGER_FAILED_ARTIFACT + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Manager failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256 + ): + die("Device Manager failed artifact evidence mismatch") + + records = [ + value + for value in load_state(FAILED_STATE_FILE) + if value.get("id") == DEVICE_PLANE_MANAGER_FAILED_PATCH_ID + ] + if len(records) != 1: + die("Device Manager failed journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") != DEVICE_PLANE_MANAGER_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "failed:DeployError" + or record.get("status") != "failed" + or record.get("message") + != ( + "container healthcheck failed for " + "500bc061b97d9007a33dd51d35ca1168eae7f6b019165b143d3bd908220656c0: " + "unhealthy" + ) + ): + die("Device Manager failed journal evidence mismatch") + + with tempfile.TemporaryDirectory( + prefix="device-manager-failed-artifact-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, _failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") != DEVICE_PLANE_MANAGER_FAILED_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES + ): + die("Device Manager failed artifact contract mismatch") + + runtime = validate_device_plane_manager_reconciled_baseline( + backup_dir, + marker_installed=False, + ) + return { + "mode": descriptor["mode"], + "backup": backup_dir, + "failedArtifact": failed_artifact, + "runtime": runtime, + } + + def device_plane_service_container_ids(service): if service not in ( *DEVICE_PLANE_RUNTIME_SERVICES, @@ -10803,8 +11124,17 @@ def validate_device_plane_foundation_runtime(network_publication=False): return accepted -def validate_device_plane_b2_discovery_ingress_runtime(runtime_before): +def validate_device_plane_b2_discovery_ingress_runtime( + runtime_before, + preserved_stateless_services=(), +): validate_device_plane_runtime_secret_metadata() + preserved_stateless = set(preserved_stateless_services) + if not preserved_stateless.issubset({ + "device-control-core", + "device-gateway", + }): + die("Device Plane B2 preserved stateless service set mismatch") before_names = device_plane_inventory_service_names(runtime_before) if set(before_names) != set(DEVICE_PLANE_RUNTIME_SERVICES): die("Device Plane B2 ingress predecessor inventory mismatch") @@ -10903,11 +11233,20 @@ def validate_device_plane_b2_discovery_ingress_runtime(runtime_before): ): die("Device Plane B2 ingress changed PostgreSQL generation") for service in ("device-control-core", "device-gateway"): - if ( + same_container = ( accepted[service]["containerId"] == before[service]["containerId"] - or accepted[service]["imageId"] == before[service]["imageId"] - ): + ) + same_image = ( + accepted[service]["imageId"] == before[service]["imageId"] + ) + if service in preserved_stateless: + if not (same_container and same_image): + die( + "Device Plane B2 stateless generation was not preserved: " + f"{service}" + ) + elif same_container or same_image: die( "Device Plane B2 ingress stateless generation not replaced: " f"{service}" @@ -13061,6 +13400,11 @@ def component_services(component, entries=None): # failed archive, journal, backup, restored source and live runtime. return () + if is_device_plane_manager_reconciliation_slice(component, entries): + # Reconciliation publishes one marker after proving the failed + # evidence and the already-restored baseline. Runtime is read-only. + return () + if is_device_plane_postgres_bootstrap_slice(component, entries): # This exact one-time transition is the only Device Plane artifact that # may select durable state. Its preflight requires both container and @@ -13476,6 +13820,9 @@ def component_builds(component, entries=None): ), ) + if is_device_plane_manager_reconciliation_slice(component, entries): + return () + if is_device_plane_backhaul_vps_enrollment_slice(component, entries): return () @@ -15498,6 +15845,7 @@ def plan_artifact(artifact): device_plane_network_publication_preflight = None device_plane_b2_ingress_preflight = None device_plane_b2_recovery_preflight = None + device_plane_manager_reconciliation_preflight = None device_plane_backhaul_preflight = None device_plane_backhaul_vps_enrollment_preflight = None device_plane_runtime_before = None @@ -15525,6 +15873,7 @@ def plan_artifact(artifact): manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_device_plane_foundation_artifact(manifest, sha) + reject_terminal_device_plane_manager_artifact(manifest, sha) reject_terminal_device_plane_backhaul_artifact(manifest, sha) transition_descriptor = None transition_preflight = None @@ -15702,6 +16051,15 @@ def plan_artifact(artifact): payload_dir ) ) + if is_device_plane_manager_reconciliation_slice( + manifest["component"], + entries, + ): + device_plane_manager_reconciliation_preflight = ( + validate_device_plane_manager_reconciliation_evidence( + payload_dir + ) + ) if is_device_plane_backhaul_target_slice( manifest["component"], entries, @@ -17255,6 +17613,38 @@ def plan_artifact(artifact): ) print("device_gateway_tcp_9921=disabled:unpublished") print("device_plane_rollback=marker-only-runtime-unchanged") + if device_plane_manager_reconciliation_preflight is not None: + print( + "device_plane_transition=" + f"{device_plane_manager_reconciliation_preflight['mode']}" + ) + print( + "failed_patch=" + f"{DEVICE_PLANE_MANAGER_FAILED_PATCH_ID}" + ) + print( + "failed_artifact_sha256=" + f"{DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256}" + ) + print( + "recovery_backup=" + f"{device_plane_manager_reconciliation_preflight['backup'].name}" + ) + print("device_plane_build=none") + print("device_plane_runtime_mutation=none") + print( + "device_plane_runtime_services=" + "preserved:device-control-core,device-gateway," + "device-postgres,device-backhaul-target" + ) + print("device_manager=absent") + print( + "device_plane_source_action=" + "publish-reconciliation-marker-only" + ) + print("device_gateway_tcp_9921=preserved:loopback-only") + print("device_manager_public_route=unchanged:absent") + print("device_plane_rollback=marker-only-runtime-unchanged") if device_plane_backhaul_vps_enrollment_preflight is not None: print( "device_plane_transition=" @@ -17867,11 +18257,24 @@ def rollback_device_plane_apply( baseline_entries, baseline_services, ) - run_healthchecks( - "device-plane", - baseline_entries, - baseline_services, - ) + if is_device_plane_manager_control_plane_slice("device-plane", entries): + for service in baseline_services: + healthcheck_compose_service_with_grace( + "device-plane", + service, + ) + for check in component_healthchecks( + "device-plane", + baseline_entries, + baseline_services, + ): + healthcheck_url(check) + else: + run_healthchecks( + "device-plane", + baseline_entries, + baseline_services, + ) return f"source+runtime-restored:{restored_count}" @@ -18755,6 +19158,8 @@ def run_component_runtime(component, entries, services): entries, ): return + if is_device_plane_manager_reconciliation_slice(component, entries): + return if is_device_plane_foundation_recovery_slice(component, entries): return if is_device_plane_foundation_network_publication_slice( @@ -18782,8 +19187,9 @@ def run_device_plane_runtime_for_apply( "device-plane", entries, ) or is_device_plane_foundation_recovery_slice( - "device-plane", - entries, + "device-plane", entries + ) or is_device_plane_manager_reconciliation_slice( + "device-plane", entries ): return # Build and runtime preparation are pre-runtime phases. A failure here @@ -19187,6 +19593,46 @@ def healthcheck_container(container_name): die(f"container healthcheck failed for {container_name}: {last_status}") +def healthcheck_container_with_grace(container_name): + """Wait through bounded transient unhealthy/exited states. + + Device Manager activation and its baseline rollback both recreate a Core + generation while Docker health probes and restart policy converge. The + ordinary helper deliberately fails on the first terminal-looking state; + this exact transition instead requires a healthy result within the same + bounded five-minute window. + """ + last_status = "unknown" + for attempt in range(1, 61): + result = subprocess.run( + [ + str(DOCKER), + "inspect", + "--format", + "{{if .State.Health}}{{.State.Health.Status}}" + "{{else}}{{.State.Status}}{{end}}", + container_name, + ], + check=False, + capture_output=True, + text=True, + ) + status_text = result.stdout.strip().lower() + last_status = ( + status_text + or result.stderr.strip() + or f"inspect-exit-{result.returncode}" + ) + if status_text in ("healthy", "running"): + return + if attempt < 60: + time.sleep(5) + die( + "container healthcheck grace exhausted for " + f"{container_name}: {last_status}" + ) + + def compose_service_container_id(component, service): result = subprocess.run( [*compose_base_cmd(component), "ps", "-q", service], @@ -19209,6 +19655,12 @@ def healthcheck_compose_service(component, service): healthcheck_container(compose_service_container_id(component, service)) +def healthcheck_compose_service_with_grace(component, service): + healthcheck_container_with_grace( + compose_service_container_id(component, service) + ) + + def container_environment(container, label): environment = {} for raw in (container.get("Config") or {}).get("Env") or []: @@ -19467,6 +19919,23 @@ process.stdout.write('engine-l2-closed-loop:0.7.0:cas+safe-profile+external-plan def run_healthchecks(component, entries=None, services=None): + if is_device_plane_manager_reconciliation_slice(component, entries): + if tuple(services or ()) != (): + die("Device Manager reconciliation service set mismatch") + for service in DEVICE_PLANE_RUNTIME_SERVICES: + healthcheck_compose_service_with_grace( + "device-plane", + service, + ) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + backup_dir = validate_device_plane_manager_reconciliation_backup() + validate_device_plane_manager_reconciled_baseline( + backup_dir, + marker_installed=True, + ) + return + if is_device_plane_backhaul_vps_enrollment_slice(component, entries): if tuple(services or ()) != (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,): die("Device Plane VPS enrollment service set mismatch") @@ -19583,7 +20052,10 @@ def run_healthchecks(component, entries=None, services=None): ): die("Device Manager control-plane service set mismatch") for service in services: - healthcheck_compose_service("device-plane", service) + healthcheck_compose_service_with_grace( + "device-plane", + service, + ) for check in component_healthchecks(component, entries, services): healthcheck_url(check) validate_device_manager_control_plane_runtime() @@ -20374,6 +20846,10 @@ def apply_artifact(artifact): manifest, sha, ) + reject_terminal_device_plane_manager_artifact( + manifest, + sha, + ) reject_terminal_device_plane_backhaul_artifact( manifest, sha, @@ -20425,6 +20901,13 @@ def apply_artifact(artifact): validate_device_plane_b2_discovery_rollback_recovery_evidence( payload_dir ) + if is_device_plane_manager_reconciliation_slice( + component, + entries, + ): + validate_device_plane_manager_reconciliation_evidence( + payload_dir + ) if is_device_plane_backhaul_target_slice( component, entries, diff --git a/infra/deploy-runner/test_device_manager_control_plane_artifacts.py b/infra/deploy-runner/test_device_manager_control_plane_artifacts.py index 33b2a7b..576f293 100644 --- a/infra/deploy-runner/test_device_manager_control_plane_artifacts.py +++ b/infra/deploy-runner/test_device_manager_control_plane_artifacts.py @@ -209,7 +209,7 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): with ( mock.patch.object( RUNNER, - "healthcheck_compose_service", + "healthcheck_compose_service_with_grace", ) as service_health, mock.patch.object(RUNNER, "healthcheck_url") as url_health, mock.patch.object( @@ -277,8 +277,21 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): ) as restore_runtime, mock.patch.object( RUNNER, - "run_healthchecks", + "healthcheck_compose_service_with_grace", ) as restore_health, + mock.patch.object( + RUNNER, + "component_healthchecks", + return_value=("core-health",), + ), + mock.patch.object( + RUNNER, + "healthcheck_url", + ) as restore_url_health, + mock.patch.object( + RUNNER, + "run_healthchecks", + ) as generic_health, ): result = RUNNER.rollback_device_plane_apply( Path(directory) / "live", @@ -298,9 +311,10 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): ) restore_health.assert_called_once_with( "device-plane", - existing, - ("device-control-core",), + "device-control-core", ) + restore_url_health.assert_called_once_with("core-health") + generic_health.assert_not_called() self.assertEqual( result, f"source+runtime-restored:{len(entries)}", diff --git a/infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py b/infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py new file mode 100644 index 0000000..52ee2c9 --- /dev/null +++ b/infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +BUILDER = ( + SCRIPT_DIR + / "build-device-manager-control-plane-reconciliation-artifact.mjs" +) + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_device_manager_reconciliation_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase): + def build(self, artifact_dir, patch_id): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + ["node", str(BUILDER), patch_id], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_artifact_is_marker_only_exact_and_deterministic(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-manager-reconciliation-artifact-", + ) as directory: + artifact_dir = Path(directory) + first = self.build( + artifact_dir, + "device-manager-control-plane-reconciliation-unit-002", + ) + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build( + artifact_dir, + "device-manager-control-plane-reconciliation-unit-002", + ) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + self.assertEqual(first["component"], "device-plane") + self.assertEqual( + first["entries"], + list(RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES), + ) + self.assertEqual(first["build"], []) + self.assertEqual(first["services"], []) + self.assertEqual( + first["runtimeAction"], + "read-only-acceptance", + ) + + with tarfile.open(first["artifact"], "r:gz") as archive: + names = [member.name for member in archive.getmembers()] + descriptor = json.loads( + archive.extractfile( + "payload/" + + RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_REL + ).read().decode("utf-8") + ) + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_manager_reconciliation_descriptor(), + ) + self.assertEqual( + names, + [ + "manifest.env", + "files.txt", + "payload", + "payload/deployment", + "payload/" + + RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_REL, + ], + ) + self.assertEqual( + RUNNER.component_services( + "device-plane", + RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES, + ), + (), + ) + self.assertEqual( + RUNNER.component_builds( + "device-plane", + RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES, + ), + (), + ) + + def test_failed_control_plane_is_terminal(self): + with self.assertRaisesRegex( + RUNNER.DeployError, + "exact registered reconciliation successor", + ): + RUNNER.reject_terminal_device_plane_manager_artifact( + {"id": RUNNER.DEVICE_PLANE_MANAGER_FAILED_PATCH_ID}, + "0" * 64, + ) + with self.assertRaisesRegex( + RUNNER.DeployError, + "exact registered reconciliation successor", + ): + RUNNER.reject_terminal_device_plane_manager_artifact( + {"id": "different"}, + RUNNER.DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256, + ) + + def test_health_grace_waits_through_unhealthy_and_exited(self): + results = [ + mock.Mock(stdout="unhealthy\n", stderr="", returncode=0), + mock.Mock(stdout="exited\n", stderr="", returncode=0), + mock.Mock(stdout="starting\n", stderr="", returncode=0), + mock.Mock(stdout="healthy\n", stderr="", returncode=0), + ] + with ( + mock.patch.object( + RUNNER.subprocess, + "run", + side_effect=results, + ) as inspect, + mock.patch.object(RUNNER.time, "sleep") as sleep, + ): + RUNNER.healthcheck_container_with_grace("container-id") + self.assertEqual(inspect.call_count, 4) + self.assertEqual(sleep.call_count, 3) + + def test_manager_rollback_uses_bounded_grace_for_restored_core(self): + entries = RUNNER.DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES + runtime_before = { + "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [ + { + "service": "device-control-core", + "containerId": "a" * 64, + "imageId": "sha256:" + "b" * 64, + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + }, + { + "service": "device-gateway", + "containerId": "c" * 64, + "imageId": "sha256:" + "d" * 64, + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + }, + { + "service": "device-postgres", + "containerId": "e" * 64, + "imageId": "sha256:" + "f" * 64, + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + }, + ], + } + existing = list(RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING) + missing = list(RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING) + with ( + mock.patch.object( + RUNNER, + "read_backup_path_list", + side_effect=[existing, missing], + ), + mock.patch.object( + RUNNER, + "read_strict_json", + return_value=runtime_before, + ), + mock.patch.object( + RUNNER, + "stop_and_remove_compose_services", + ) as stop, + mock.patch.object( + RUNNER, + "restore_platform_overlay", + return_value=len(entries), + ), + mock.patch.object(RUNNER, "run_component_runtime") as runtime, + mock.patch.object( + RUNNER, + "healthcheck_compose_service_with_grace", + ) as health, + mock.patch.object( + RUNNER, + "component_healthchecks", + return_value=("core",), + ), + mock.patch.object(RUNNER, "healthcheck_url") as url_health, + mock.patch.object(RUNNER, "run_healthchecks") as generic_health, + ): + restored = RUNNER.rollback_device_plane_apply( + Path("/live"), + Path("/backup"), + entries, + "stamp", + True, + ("device-control-core", "device-manager"), + ) + + self.assertEqual( + restored, + f"source+runtime-restored:{len(entries)}", + ) + stop.assert_called_once_with("device-plane", ("device-manager",)) + runtime.assert_called_once_with( + "device-plane", + existing, + ("device-control-core",), + ) + health.assert_called_once_with( + "device-plane", + "device-control-core", + ) + url_health.assert_called_once_with("core") + generic_health.assert_not_called() + + def test_reconciliation_runtime_phase_is_read_only(self): + entries = RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES + with ( + mock.patch.object(RUNNER, "run_build") as build, + mock.patch.object(RUNNER, "prepare_component_runtime") as prepare, + mock.patch.object(RUNNER, "run_compose") as compose, + ): + RUNNER.run_component_runtime("device-plane", entries, ()) + RUNNER.run_device_plane_runtime_for_apply( + entries, + (), + mock.Mock(), + ) + build.assert_not_called() + prepare.assert_not_called() + compose.assert_not_called() + + +if __name__ == "__main__": + unittest.main(verbosity=2)