diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index d0b2081..f085a74 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -12877,7 +12877,35 @@ def validate_device_plane_edge_core_channel_upgrade_v4_predecessor( } -def validate_device_plane_control_core_release_predecessor(payload_dir): +def device_plane_control_core_preserved_runtime_health_services(descriptor): + services = tuple(descriptor.get("preservedServices") or ()) + expected = ( + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ) + if services != expected: + die("Device Control Core preserved service set mismatch") + return services + + +def validate_device_plane_control_core_preserved_runtime_health(descriptor): + services = device_plane_control_core_preserved_runtime_health_services( + descriptor + ) + for service in services: + healthcheck_compose_service("device-plane", service) + return services + + +def validate_device_plane_control_core_release_predecessor( + payload_dir, + *, + preflight_phase, +): + if preflight_phase not in ("plan", "apply"): + die("Device Control Core predecessor preflight phase is invalid") descriptor = validate_device_plane_control_core_release_payload( payload_dir ) @@ -12981,13 +13009,16 @@ def validate_device_plane_control_core_release_predecessor(payload_dir): identity_state = inspect_device_edge_channel_core_identity_state() if identity_state != "valid-reuse-at-apply": die("Device Control Core release requires the valid active identity") - for service in ( - "device-control-core", - "device-manager", - "device-gateway", - "device-postgres", - ): - healthcheck_compose_service("device-plane", service) + selected_runtime = ( + validate_device_plane_control_core_selected_predecessor_runtime() + ) + preserved_runtime_health = "deferred-to-apply" + if preflight_phase == "apply": + preserved_runtime_health = ( + validate_device_plane_control_core_preserved_runtime_health( + descriptor + ) + ) validate_device_manager_control_plane_runtime( require_edge_channel=True, core_network_mode="private-egress", @@ -12997,6 +13028,8 @@ def validate_device_plane_control_core_release_predecessor(payload_dir): "descriptor": descriptor, "identityState": identity_state, "predecessorArtifact": artifact, + "selectedRuntime": selected_runtime, + "preservedRuntimeHealth": preserved_runtime_health, } @@ -13287,6 +13320,7 @@ def device_plane_inventory_service_names(inventory): or item.get("service") not in ( *DEVICE_PLANE_RUNTIME_SERVICES, "device-manager", + DEVICE_PLANE_BACKHAUL_TARGET_SERVICE, ) or not isinstance(item.get("containerId"), str) or not re.fullmatch(r"[a-f0-9]{64}", item["containerId"]) @@ -13311,6 +13345,80 @@ def device_plane_inventory_service_names(inventory): return tuple(names) +def validate_device_plane_control_core_selected_predecessor_runtime(): + inventory = device_plane_runtime_inventory(("device-control-core",)) + names = device_plane_inventory_service_names(inventory) + if names != ("device-control-core",): + die("Device Control Core selected predecessor runtime is missing") + selected = inventory["services"][0] + if ( + selected["status"] != "running" + or selected["running"] is not True + or selected["health"] not in ("healthy", "unhealthy") + ): + die("Device Control Core selected predecessor state is invalid") + return selected + + +def accept_device_plane_control_core_rollback_runtime(runtime_before): + expected_services = ( + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ) + names = device_plane_inventory_service_names(runtime_before) + if set(names) != set(expected_services): + die("Device Control Core rollback predecessor inventory mismatch") + before = { + item["service"]: item + for item in runtime_before["services"] + } + selected_before = before["device-control-core"] + if ( + selected_before["status"] != "running" + or selected_before["running"] is not True + or selected_before["health"] not in ("healthy", "unhealthy") + ): + die("Device Control Core rollback predecessor state is invalid") + + for service in expected_services[1:]: + healthcheck_compose_service_with_grace("device-plane", service) + + container_id = compose_service_container_id( + "device-plane", + "device-control-core", + ) + if container_id == selected_before["containerId"]: + die("Device Control Core rollback generation was not recreated") + accepted_health = {"healthy"} + if selected_before["health"] == "unhealthy": + accepted_health.add("unhealthy") + last_status = "unknown" + for attempt in range(1, 61): + container = inspect_device_plane_container(container_id) + state = container.get("State") or {} + health = (state.get("Health") or {}).get("Status") + last_status = health or state.get("Status") or "unknown" + if ( + state.get("Status") == "running" + and state.get("Running") is True + and health in accepted_health + ): + return { + "containerId": container_id, + "health": health, + "predecessorHealth": selected_before["health"], + } + if attempt < 60: + time.sleep(5) + die( + "Device Control Core rollback state did not converge: " + f"{last_status}" + ) + + def validate_device_plane_foundation_recovery_evidence(payload_dir): backup_dir = BACKUPS_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID try: @@ -19354,7 +19462,8 @@ def plan_artifact(artifact): ): device_plane_control_core_release_preflight = ( validate_device_plane_control_core_release_predecessor( - payload_dir + payload_dir, + preflight_phase="plan", ) ) if is_device_plane_manager_reconciliation_slice( @@ -21821,16 +21930,9 @@ def rollback_device_plane_apply( "device-plane", entries, ): - for service in ( - "device-control-core", - "device-manager", - "device-gateway", - "device-postgres", - ): - healthcheck_compose_service_with_grace( - "device-plane", - service, - ) + accept_device_plane_control_core_rollback_runtime( + runtime_inventory + ) validate_device_manager_control_plane_runtime( require_edge_channel=True, core_network_mode="private-egress", @@ -23595,6 +23697,7 @@ def run_healthchecks(component, entries=None, services=None): "device-manager", "device-gateway", "device-postgres", + "device-backhaul-target", ): healthcheck_compose_service_with_grace( "device-plane", @@ -24659,7 +24762,8 @@ def apply_artifact(artifact): entries, ): validate_device_plane_control_core_release_predecessor( - payload_dir + payload_dir, + preflight_phase="apply", ) if is_device_plane_manager_reconciliation_slice( component, @@ -24986,11 +25090,16 @@ def apply_artifact(artifact): component, entries, ) - or is_device_plane_edge_core_channel_bootstrap_slice( + ): + inventory_services = ( + *inventory_services, + "device-manager", + "device-backhaul-target", + ) + elif is_device_plane_edge_core_channel_bootstrap_slice( component, entries, - ) - ): + ): inventory_services = ( *inventory_services, "device-manager", diff --git a/infra/deploy-runner/test_device_plane_registry.py b/infra/deploy-runner/test_device_plane_registry.py index 83cc8e6..74cccbf 100644 --- a/infra/deploy-runner/test_device_plane_registry.py +++ b/infra/deploy-runner/test_device_plane_registry.py @@ -28,6 +28,197 @@ RUNNER = load_runner() class DevicePlaneRegistryTest(unittest.TestCase): + def test_control_core_predecessor_health_is_phase_scoped(self): + with self.assertRaisesRegex( + RUNNER.DeployError, + "preflight phase is invalid", + ): + RUNNER.validate_device_plane_control_core_release_predecessor( + Path("/not-used"), + preflight_phase="unknown", + ) + + plan_source = inspect.getsource(RUNNER.plan_artifact) + self.assertIn( + 'validate_device_plane_control_core_release_predecessor(\n' + ' payload_dir,\n' + ' preflight_phase="plan",', + plan_source, + ) + apply_source = inspect.getsource(RUNNER.apply_artifact) + self.assertIn( + 'validate_device_plane_control_core_release_predecessor(\n' + ' payload_dir,\n' + ' preflight_phase="apply",', + apply_source, + ) + + def test_control_core_preflight_health_excludes_selected_target(self): + descriptor = { + "preservedServices": [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + } + with mock.patch.object( + RUNNER, + "healthcheck_compose_service", + ) as healthcheck: + services = ( + RUNNER.validate_device_plane_control_core_preserved_runtime_health( + descriptor + ) + ) + + self.assertEqual( + services, + ( + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ), + ) + self.assertEqual( + healthcheck.call_args_list, + [ + mock.call("device-plane", "device-manager"), + mock.call("device-plane", "device-gateway"), + mock.call("device-plane", "device-postgres"), + mock.call("device-plane", "device-backhaul-target"), + ], + ) + self.assertNotIn( + mock.call("device-plane", "device-control-core"), + healthcheck.call_args_list, + ) + + def test_control_core_selected_predecessor_may_be_unhealthy(self): + inventory = { + "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [{ + "service": "device-control-core", + "containerId": "a" * 64, + "imageId": "sha256:" + "b" * 64, + "status": "running", + "running": True, + "health": "unhealthy", + "restartCount": 4, + }], + } + with mock.patch.object( + RUNNER, + "device_plane_runtime_inventory", + return_value=inventory, + ): + selected = ( + RUNNER.validate_device_plane_control_core_selected_predecessor_runtime() + ) + self.assertEqual(selected["health"], "unhealthy") + + def test_control_core_rollback_accepts_restored_unhealthy_boundary(self): + service_names = ( + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ) + inventory = { + "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [ + { + "service": service, + "containerId": chr(97 + index) * 64, + "imageId": "sha256:" + str(index + 1) * 64, + "status": "running", + "running": True, + "health": "unhealthy" if index == 0 else "healthy", + "restartCount": index, + } + for index, service in enumerate(service_names) + ], + } + restored_id = "f" * 64 + restored = { + "Id": restored_id, + "State": { + "Status": "running", + "Running": True, + "Health": {"Status": "unhealthy"}, + }, + } + with ( + mock.patch.object( + RUNNER, + "healthcheck_compose_service_with_grace", + ) as preserved_health, + mock.patch.object( + RUNNER, + "compose_service_container_id", + return_value=restored_id, + ), + mock.patch.object( + RUNNER, + "inspect_device_plane_container", + return_value=restored, + ), + ): + accepted = ( + RUNNER.accept_device_plane_control_core_rollback_runtime( + inventory + ) + ) + + self.assertEqual(accepted["health"], "unhealthy") + self.assertEqual(accepted["predecessorHealth"], "unhealthy") + self.assertEqual( + preserved_health.call_args_list, + [ + mock.call("device-plane", "device-manager"), + mock.call("device-plane", "device-gateway"), + mock.call("device-plane", "device-postgres"), + mock.call("device-plane", "device-backhaul-target"), + ], + ) + + def test_control_core_post_apply_health_includes_backhaul(self): + with ( + mock.patch.object( + RUNNER, + "healthcheck_compose_service_with_grace", + ) as healthcheck, + mock.patch.object( + RUNNER, + "component_healthchecks", + return_value=(), + ), + mock.patch.object( + RUNNER, + "validate_device_manager_control_plane_runtime", + ), + ): + RUNNER.run_healthchecks( + "device-plane", + RUNNER.DEVICE_PLANE_CONTROL_CORE_RELEASE_V2_ENTRIES, + ("device-control-core",), + ) + + self.assertEqual( + healthcheck.call_args_list, + [ + mock.call("device-plane", "device-control-core"), + mock.call("device-plane", "device-manager"), + mock.call("device-plane", "device-gateway"), + mock.call("device-plane", "device-postgres"), + mock.call("device-plane", "device-backhaul-target"), + ], + ) + def test_manager_predecessor_health_is_phase_scoped(self): with self.assertRaisesRegex( RUNNER.DeployError,