diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 9e21203..37385da 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -383,6 +383,64 @@ DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES = ( "services/device-control-core", DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL, ) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL = ( + "deployment/device-control-core-release-v3-reconciliation-v1.json" +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES = ( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID = ( + "device-control-core-release-v3-reconciliation-20260822-042" +) +DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID = ( + "device-control-core-release-v3-20260822-040" +) +DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-control-core-release-v3-20260822-040.tgz." + "20260822-184245" +) +DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 = ( + "08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92" +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID = ( + "device-plane-device-control-core-release-v3-20260822-040-" + "20260822-184245" +) +DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID = ( + "sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a" +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_SHA256 = { + "manifest.env": ( + "eeb6ba544cde0d5d45a7bf16e921ca8969f0e5307872a53a5448d6a1279f76db" + ), + "files.txt": ( + "be1bda7d2afff11cfaeacd607a63261f99c17d6c04c8c762424669585ad9a75f" + ), + "existing-files.txt": ( + "a491b4a4ad7bc8dd8e8fc5b008f992663ef64e569b016efe7de1f4f14e5629ad" + ), + "missing-files.txt": ( + "d04dd94a0f5a1fd6a7809219e953ade89a43ec5a2e29016cb666f684a77323c0" + ), + "source-before.tgz": ( + "1c119875cccb52b761d1e2b42a6d666c9f9287782b424283b20180ea502237c4" + ), + "runtime-before.json": ( + "b694a183345181e32a3335bb50d73c4b8df9ce45e54072f3a8dc06789684fa89" + ), +} +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING = ( + ".dockerignore", + "package.json", + "package-lock.json", + "packages/device-protocol-contract", + "packages/device-edge-channel-contract", + "services/device-control-core", +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING = ( + "packages/infrastructure-telemetry-contract", + DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL, +) DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_PATCH_ID = ( "device-edge-core-channel-upgrade-v4-20260812-023" ) @@ -4713,6 +4771,7 @@ def allowed_payload_path(component, rel): DEVICE_PLANE_CONTROL_CORE_RELEASE_REL, DEVICE_PLANE_CONTROL_CORE_RELEASE_V2_REL, DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL, + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, DEVICE_PLANE_MANAGER_RECONCILIATION_REL, DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, "packages/device-protocol-contract", @@ -11167,6 +11226,13 @@ def load_artifact(artifact, work_dir): entries, ): validate_device_plane_manager_v2_reconciliation_payload(payload_dir) + if is_device_plane_control_core_v3_reconciliation_slice( + manifest["component"], + entries, + ): + validate_device_plane_control_core_v3_reconciliation_payload( + payload_dir + ) if is_device_plane_postgres_bootstrap_slice( manifest["component"], entries, @@ -11807,6 +11873,15 @@ def is_device_plane_control_core_release_v3_slice(component, entries): ) +def is_device_plane_control_core_v3_reconciliation_slice(component, entries): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) + == DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES + ) + + def is_device_plane_manager_v2_control_plane_slice(component, entries): return ( component == "device-plane" @@ -12458,6 +12533,43 @@ def expected_device_plane_manager_v2_reconciliation_descriptor(): } +def expected_device_plane_control_core_v3_reconciliation_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane.device-control-core-release-v3-" + "reconciliation.v1" + ), + "mode": "failed-release-v3-exact-preapply-image-restore", + "failedPatchId": DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID, + "failedArtifactSha256": ( + DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 + ), + "failedArtifact": DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT, + "backupId": DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID, + "predecessorPatchId": "device-control-core-release-v2-20260822-038", + "predecessorArtifactSha256": ( + "e2d062b82b022dba662522b5d6e192026ac964d78950d903295ca3cbbc95ab28" + ), + "preapplyImageId": DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "sourceAction": "accept-byte-exact-restored-preapply-source", + "runtimeAction": ( + "retag-exact-preapply-image+recreate-device-control-core-only" + ), + "preservedServices": [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "publicIngress": "disabled", + "edgeChannel": "core-initiated-pinned-mtls-registered-edges-only", + "commandTransport": "typed-service-ping-v1", + "gelios": "untouched-legacy-only", + "rollback": "marker+exact-preapply-image-runtime", + } + + def validate_platform_device_core_hub_trust_payload(payload_dir): descriptor = read_strict_json( payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL, @@ -13335,6 +13447,20 @@ def validate_device_plane_manager_v2_reconciliation_payload(payload_dir): return descriptor +def validate_device_plane_control_core_v3_reconciliation_payload(payload_dir): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, + "Device Control Core v3 reconciliation descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_control_core_v3_reconciliation_descriptor() + ): + die("Device Control Core v3 reconciliation descriptor mismatch") + return descriptor + + def is_device_plane_postgres_bootstrap_slice(component, entries): return ( component == "device-plane" @@ -15935,6 +16061,247 @@ def validate_device_plane_manager_v2_reconciliation_evidence(payload_dir): } +def validate_device_plane_control_core_v3_reconciliation_backup(): + backup_dir = ( + BACKUPS_DIR / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID + ) + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Control Core v3 failed-apply backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Control Core v3 failed-apply backup is unsafe") + expected = DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_SHA256 + if {child.name for child in backup_dir.iterdir()} != set(expected): + die("Device Control Core v3 failed-apply backup file set mismatch") + for name, expected_sha256 in expected.items(): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Device Control Core v3 failed-apply backup drift detected: " + f"{name}" + ) + existing = tuple(read_backup_path_list( + backup_dir / "existing-files.txt" + )) + missing = tuple(read_backup_path_list( + backup_dir / "missing-files.txt" + )) + validate_backup_partition( + DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES, + existing, + missing, + "Device Control Core v3 reconciliation", + ) + if ( + existing != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING + or missing != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING + ): + die("Device Control Core v3 failed-apply backup partition mismatch") + return backup_dir + + +def validate_device_plane_control_core_v3_restored_source( + backup_dir, + *, + marker_installed, +): + root = component_root("device-plane") + with tempfile.TemporaryDirectory( + prefix="device-control-core-v3-restored-source-", + dir=TMP_DIR, + ) as directory: + backup_root = Path(directory) + materialize_backup_tree( + backup_dir / "source-before.tgz", + backup_root, + set(DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING), + ) + backup_source = collect_exact_files( + backup_root, + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING, + "Device Control Core v3 pre-apply source", + ) + live_source = collect_exact_files( + root, + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING, + "Device Control Core v3 restored live source", + ) + if live_source != backup_source: + die("Device Control Core v3 rollback source does not match backup") + for rel in DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING: + path = root / rel + if path.exists() or path.is_symlink(): + die( + "Device Control Core v3 candidate-only source remains installed: " + f"{rel}" + ) + marker = root / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL + if marker_installed: + descriptor = read_strict_json( + marker, + "installed Device Control Core v3 reconciliation descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_control_core_v3_reconciliation_descriptor() + ): + die("installed Device Control Core v3 reconciliation mismatch") + elif marker.exists() or marker.is_symlink(): + die("Device Control Core v3 reconciliation already installed") + + +def validate_device_plane_control_core_v3_reconciliation_runtime( + backup_dir, + *, + require_recovered, +): + runtime_before = read_strict_json( + backup_dir / "runtime-before.json", + "Device Control Core v3 pre-apply runtime inventory", + max_bytes=64 * 1024, + ) + expected_names = { + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + } + if set(device_plane_inventory_service_names(runtime_before)) != expected_names: + die("Device Control Core v3 pre-apply runtime inventory mismatch") + before = {item["service"]: item for item in runtime_before["services"]} + if ( + before["device-control-core"]["imageId"] + != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID + or before["device-control-core"]["health"] != "healthy" + or before["device-control-core"]["running"] is not True + ): + die("Device Control Core v3 pre-apply image evidence mismatch") + current = device_plane_runtime_inventory(tuple(sorted(expected_names))) + if set(device_plane_inventory_service_names(current)) != expected_names: + die("Device Control Core v3 current runtime inventory mismatch") + observed = {item["service"]: item for item in current["services"]} + for service in expected_names - {"device-control-core"}: + expected_item = before[service] + actual_item = observed[service] + if ( + actual_item["containerId"] != expected_item["containerId"] + or actual_item["imageId"] != expected_item["imageId"] + or actual_item["running"] is not True + or actual_item["health"] != "healthy" + ): + die( + "Device Control Core v3 reconciliation changed preserved service: " + f"{service}" + ) + image_id = inspect_optional_local_image( + DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "Device Control Core v3 exact pre-apply image", + ) + if image_id != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID: + die("Device Control Core v3 exact pre-apply image is unavailable") + core = observed["device-control-core"] + if require_recovered: + if ( + core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID + or core["running"] is not True + or core["health"] != "healthy" + ): + die("Device Control Core v3 exact-image recovery did not converge") + validate_device_manager_control_plane_runtime( + require_edge_channel=True, + core_network_mode="private-egress", + ) + return { + "before": runtime_before, + "current": current, + "core": core, + } + + +def validate_device_plane_control_core_v3_reconciliation_evidence(payload_dir): + descriptor = validate_device_plane_control_core_v3_reconciliation_payload( + payload_dir + ) + backup_dir = validate_device_plane_control_core_v3_reconciliation_backup() + failed_artifact = FAILED_DIR / DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Control Core v3 failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 + ): + die("Device Control Core v3 failed artifact evidence mismatch") + records = [ + value + for value in load_state(FAILED_STATE_FILE) + if value.get("id") == DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID + ] + if len(records) != 1: + die("Device Control Core v3 failed journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "failed:DeployError" + or record.get("status") != "failed" + or record.get("message") + != ( + "container healthcheck grace exhausted for " + "97513c9a1027ae84f5dad6d0bc4792a67a4b0a1f32665d6d498093f43ee7987b: " + "unhealthy" + ) + ): + die("Device Control Core v3 failed journal evidence mismatch") + with tempfile.TemporaryDirectory( + prefix="device-control-core-v3-failed-artifact-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, _failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) != DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES + ): + die("Device Control Core v3 failed artifact contract mismatch") + validate_device_plane_control_core_v3_restored_source( + backup_dir, + marker_installed=False, + ) + runtime = validate_device_plane_control_core_v3_reconciliation_runtime( + backup_dir, + require_recovered=False, + ) + return { + "mode": descriptor["mode"], + "backup": backup_dir, + "failedArtifact": failed_artifact, + "runtime": runtime, + } + + def device_plane_service_container_ids(service): if service not in ( *DEVICE_PLANE_RUNTIME_SERVICES, @@ -19322,6 +19689,12 @@ def component_services(component, entries=None): if is_device_plane_control_core_release_slice(component, entries): return ("device-control-core",) + if is_device_plane_control_core_v3_reconciliation_slice( + component, + entries, + ): + return ("device-control-core",) + if is_device_plane_manager_only_release_slice(component, entries): return ("device-manager",) @@ -19922,6 +20295,12 @@ def component_builds(component, entries=None): ), ),) + if is_device_plane_control_core_v3_reconciliation_slice( + component, + entries, + ): + return () + if is_device_plane_manager_only_release_slice(component, entries): return (( DEVICE_PLANE_ROOT / "services/device-manager", @@ -28114,6 +28493,7 @@ def plan_artifact(artifact): device_plane_control_core_release_preflight = None device_plane_manager_reconciliation_preflight = None device_plane_manager_v2_reconciliation_preflight = None + device_plane_control_core_v3_reconciliation_preflight = None device_plane_backhaul_preflight = None device_plane_backhaul_vps_enrollment_preflight = None device_plane_runtime_before = None @@ -28377,6 +28757,15 @@ def plan_artifact(artifact): payload_dir ) ) + if is_device_plane_control_core_v3_reconciliation_slice( + manifest["component"], + entries, + ): + device_plane_control_core_v3_reconciliation_preflight = ( + validate_device_plane_control_core_v3_reconciliation_evidence( + payload_dir + ) + ) if is_device_plane_backhaul_target_slice( manifest["component"], entries, @@ -30509,6 +30898,46 @@ def plan_artifact(artifact): print("device_gateway_tcp_9921=preserved:loopback-only") print("device_manager_public_route=unchanged:absent") print("device_plane_rollback=marker-only-runtime-unchanged") + if device_plane_control_core_v3_reconciliation_preflight is not None: + recovery = device_plane_control_core_v3_reconciliation_preflight + current_core = recovery["runtime"]["core"] + print(f"device_plane_transition={recovery['mode']}") + print( + "failed_patch=" + f"{DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID}" + ) + print( + "failed_artifact_sha256=" + f"{DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256}" + ) + print(f"recovery_backup={recovery['backup'].name}") + print( + "device_control_core_current_runtime=" + f"{current_core['status']}:{current_core['health']}" + ) + print( + "device_control_core_recovery_image=" + f"{DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID}" + ) + print("device_plane_build=none") + print( + "device_plane_runtime_mutation=" + "retag-exact-preapply-image+recreate:device-control-core" + ) + print( + "device_plane_runtime_services=" + "preserved:device-manager,device-gateway,device-postgres," + "device-backhaul-target" + ) + print("device_postgres=preserved:not-selected") + print( + "device_postgres_volume=" + "preserved:nodedc-device-plane-postgres-data" + ) + print("device_gateway_public_ingress=disabled") + print("device_gateway_tcp_9921=preserved:loopback-only") + print("gelios=untouched-legacy-only") + print("device_plane_rollback=marker+exact-preapply-image-runtime") if device_plane_backhaul_vps_enrollment_preflight is not None: print( "device_plane_transition=" @@ -31116,15 +31545,32 @@ def rollback_device_plane_apply( if not runtime_started or not baseline_services: return f"source-restored-runtime-unchanged:{restored_count}" - run_component_runtime( - "device-plane", - baseline_entries, - baseline_services, - ) - if is_device_plane_control_core_release_slice( + control_core_release_rollback = is_device_plane_control_core_release_slice( "device-plane", entries, - ): + ) + if control_core_release_rollback: + before = { + item["service"]: item + for item in runtime_inventory["services"] + } + selected_before = before.get("device-control-core") or {} + image_id = selected_before.get("imageId") + if not isinstance(image_id, str): + die("Device Control Core rollback image evidence is missing") + retag_device_plane_control_core_image( + image_id, + "Device Control Core exact pre-apply rollback image", + ) + prepare_component_runtime("device-plane", baseline_entries) + run_compose("device-plane", baseline_services, baseline_entries) + else: + run_component_runtime( + "device-plane", + baseline_entries, + baseline_services, + ) + if control_core_release_rollback: accept_device_plane_control_core_rollback_runtime( runtime_inventory ) @@ -32360,12 +32806,59 @@ def run_component_runtime(component, entries, services): run_compose(component, services, entries) +def retag_device_plane_control_core_image(image_id, label): + image_id = inspect_optional_local_image( + image_id, + label, + ) + if image_id is None: + die(f"{label} is unavailable") + result = subprocess.run( + [ + str(DOCKER), + "image", + "tag", + image_id, + DEVICE_PLANE_CONTROL_CORE_IMAGE, + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + die(f"{label} retag failed") + tagged = inspect_optional_local_image( + DEVICE_PLANE_CONTROL_CORE_IMAGE, + f"{label} recovered tag", + ) + if tagged != image_id: + die(f"{label} recovered tag mismatch") + + +def restore_device_plane_control_core_v3_preapply_image(): + retag_device_plane_control_core_image( + DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "Device Control Core v3 exact pre-apply image", + ) + + def run_device_plane_runtime_for_apply( entries, services, mark_runtime_started, backhaul_serve_before=None, ): + if is_device_plane_control_core_v3_reconciliation_slice( + "device-plane", + entries, + ): + if tuple(services or ()) != ("device-control-core",): + die("Device Control Core v3 reconciliation service set mismatch") + prepare_component_runtime("device-plane", entries) + mark_runtime_started() + restore_device_plane_control_core_v3_preapply_image() + run_compose("device-plane", services, entries) + return if is_device_plane_b2_discovery_rollback_recovery_slice( "device-plane", entries, @@ -33663,6 +34156,38 @@ def run_healthchecks(component, entries=None, services=None): accept_gitea_fresh_install_runtime() return + if is_device_plane_control_core_v3_reconciliation_slice( + component, + entries, + ): + if tuple(services or ()) != ("device-control-core",): + die("Device Control Core v3 reconciliation service set mismatch") + for service in ( + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ): + healthcheck_compose_service_with_grace( + "device-plane", + service, + ) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + recovery_backup = ( + validate_device_plane_control_core_v3_reconciliation_backup() + ) + validate_device_plane_control_core_v3_restored_source( + recovery_backup, + marker_installed=True, + ) + validate_device_plane_control_core_v3_reconciliation_runtime( + recovery_backup, + require_recovered=True, + ) + return + if is_device_plane_control_core_release_slice(component, entries): if tuple(services or ()) != ("device-control-core",): die("Device Control Core release service set mismatch") @@ -34793,6 +35318,13 @@ def apply_artifact(artifact): validate_device_plane_manager_v2_reconciliation_evidence( payload_dir ) + if is_device_plane_control_core_v3_reconciliation_slice( + component, + entries, + ): + validate_device_plane_control_core_v3_reconciliation_evidence( + payload_dir + ) if is_device_plane_backhaul_target_slice( component, entries, @@ -35111,6 +35643,10 @@ def apply_artifact(artifact): component, entries, ) + or is_device_plane_control_core_v3_reconciliation_slice( + component, + entries, + ) ): inventory_services = ( *inventory_services,