fix(deploy): allow unhealthy selected Core recovery

infra/deploy-runner/nodedc-deploy

@@ -12877,7 +12877,35 @@ def validate_device_plane_edge_core_channel_upgrade_v4_predecessor(
     }
 
 
-def validate_device_plane_control_core_release_predecessor(payload_dir):
+def device_plane_control_core_preserved_runtime_health_services(descriptor):
+    services = tuple(descriptor.get("preservedServices") or ())
+    expected = (
+        "device-manager",
+        "device-gateway",
+        "device-postgres",
+        "device-backhaul-target",
+    )
+    if services != expected:
+        die("Device Control Core preserved service set mismatch")
+    return services
+
+
+def validate_device_plane_control_core_preserved_runtime_health(descriptor):
+    services = device_plane_control_core_preserved_runtime_health_services(
+        descriptor
+    )
+    for service in services:
+        healthcheck_compose_service("device-plane", service)
+    return services
+
+
+def validate_device_plane_control_core_release_predecessor(
+    payload_dir,
+    *,
+    preflight_phase,
+):
+    if preflight_phase not in ("plan", "apply"):
+        die("Device Control Core predecessor preflight phase is invalid")
     descriptor = validate_device_plane_control_core_release_payload(
         payload_dir
     )
@@ -12981,13 +13009,16 @@ def validate_device_plane_control_core_release_predecessor(payload_dir):
     identity_state = inspect_device_edge_channel_core_identity_state()
     if identity_state != "valid-reuse-at-apply":
         die("Device Control Core release requires the valid active identity")
-    for service in (
+    selected_runtime = (
-        "device-control-core",
+        validate_device_plane_control_core_selected_predecessor_runtime()
-        "device-manager",
+    )
-        "device-gateway",
+    preserved_runtime_health = "deferred-to-apply"
-        "device-postgres",
+    if preflight_phase == "apply":
-    ):
+        preserved_runtime_health = (
-        healthcheck_compose_service("device-plane", service)
+            validate_device_plane_control_core_preserved_runtime_health(
+                descriptor
+            )
+        )
     validate_device_manager_control_plane_runtime(
         require_edge_channel=True,
         core_network_mode="private-egress",
@@ -12997,6 +13028,8 @@ def validate_device_plane_control_core_release_predecessor(payload_dir):
         "descriptor": descriptor,
         "identityState": identity_state,
         "predecessorArtifact": artifact,
+        "selectedRuntime": selected_runtime,
+        "preservedRuntimeHealth": preserved_runtime_health,
     }
 
 
@@ -13287,6 +13320,7 @@ def device_plane_inventory_service_names(inventory):
             or item.get("service") not in (
                 *DEVICE_PLANE_RUNTIME_SERVICES,
                 "device-manager",
+                DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,
             )
             or not isinstance(item.get("containerId"), str)
             or not re.fullmatch(r"[a-f0-9]{64}", item["containerId"])
@@ -13311,6 +13345,80 @@ def device_plane_inventory_service_names(inventory):
     return tuple(names)
 
 
+def validate_device_plane_control_core_selected_predecessor_runtime():
+    inventory = device_plane_runtime_inventory(("device-control-core",))
+    names = device_plane_inventory_service_names(inventory)
+    if names != ("device-control-core",):
+        die("Device Control Core selected predecessor runtime is missing")
+    selected = inventory["services"][0]
+    if (
+        selected["status"] != "running"
+        or selected["running"] is not True
+        or selected["health"] not in ("healthy", "unhealthy")
+    ):
+        die("Device Control Core selected predecessor state is invalid")
+    return selected
+
+
+def accept_device_plane_control_core_rollback_runtime(runtime_before):
+    expected_services = (
+        "device-control-core",
+        "device-manager",
+        "device-gateway",
+        "device-postgres",
+        "device-backhaul-target",
+    )
+    names = device_plane_inventory_service_names(runtime_before)
+    if set(names) != set(expected_services):
+        die("Device Control Core rollback predecessor inventory mismatch")
+    before = {
+        item["service"]: item
+        for item in runtime_before["services"]
+    }
+    selected_before = before["device-control-core"]
+    if (
+        selected_before["status"] != "running"
+        or selected_before["running"] is not True
+        or selected_before["health"] not in ("healthy", "unhealthy")
+    ):
+        die("Device Control Core rollback predecessor state is invalid")
+
+    for service in expected_services[1:]:
+        healthcheck_compose_service_with_grace("device-plane", service)
+
+    container_id = compose_service_container_id(
+        "device-plane",
+        "device-control-core",
+    )
+    if container_id == selected_before["containerId"]:
+        die("Device Control Core rollback generation was not recreated")
+    accepted_health = {"healthy"}
+    if selected_before["health"] == "unhealthy":
+        accepted_health.add("unhealthy")
+    last_status = "unknown"
+    for attempt in range(1, 61):
+        container = inspect_device_plane_container(container_id)
+        state = container.get("State") or {}
+        health = (state.get("Health") or {}).get("Status")
+        last_status = health or state.get("Status") or "unknown"
+        if (
+            state.get("Status") == "running"
+            and state.get("Running") is True
+            and health in accepted_health
+        ):
+            return {
+                "containerId": container_id,
+                "health": health,
+                "predecessorHealth": selected_before["health"],
+            }
+        if attempt < 60:
+            time.sleep(5)
+    die(
+        "Device Control Core rollback state did not converge: "
+        f"{last_status}"
+    )
+
+
 def validate_device_plane_foundation_recovery_evidence(payload_dir):
     backup_dir = BACKUPS_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID
     try:
@@ -19354,7 +19462,8 @@ def plan_artifact(artifact):
         ):
             device_plane_control_core_release_preflight = (
                 validate_device_plane_control_core_release_predecessor(
-                    payload_dir
+                    payload_dir,
+                    preflight_phase="plan",
                 )
             )
         if is_device_plane_manager_reconciliation_slice(
@@ -21821,16 +21930,9 @@ def rollback_device_plane_apply(
         "device-plane",
         entries,
     ):
-        for service in (
+        accept_device_plane_control_core_rollback_runtime(
-            "device-control-core",
+            runtime_inventory
-            "device-manager",
+        )
-            "device-gateway",
-            "device-postgres",
-        ):
-            healthcheck_compose_service_with_grace(
-                "device-plane",
-                service,
-            )
         validate_device_manager_control_plane_runtime(
             require_edge_channel=True,
             core_network_mode="private-egress",
@@ -23595,6 +23697,7 @@ def run_healthchecks(component, entries=None, services=None):
             "device-manager",
             "device-gateway",
             "device-postgres",
+            "device-backhaul-target",
         ):
             healthcheck_compose_service_with_grace(
                 "device-plane",
@@ -24659,7 +24762,8 @@ def apply_artifact(artifact):
                     entries,
                 ):
                     validate_device_plane_control_core_release_predecessor(
-                        payload_dir
+                        payload_dir,
+                        preflight_phase="apply",
                     )
                 if is_device_plane_manager_reconciliation_slice(
                     component,
@@ -24986,11 +25090,16 @@ def apply_artifact(artifact):
                             component,
                             entries,
                         )
-                        or is_device_plane_edge_core_channel_bootstrap_slice(
+                    ):
+                        inventory_services = (
+                            *inventory_services,
+                            "device-manager",
+                            "device-backhaul-target",
+                        )
+                    elif is_device_plane_edge_core_channel_bootstrap_slice(
                             component,
                             entries,
-                        )
+                        ):
-                    ):
                         inventory_services = (
                             *inventory_services,
                             "device-manager",

infra/deploy-runner/test_device_plane_registry.py

@@ -28,6 +28,197 @@ RUNNER = load_runner()
 
 
 class DevicePlaneRegistryTest(unittest.TestCase):
+    def test_control_core_predecessor_health_is_phase_scoped(self):
+        with self.assertRaisesRegex(
+            RUNNER.DeployError,
+            "preflight phase is invalid",
+        ):
+            RUNNER.validate_device_plane_control_core_release_predecessor(
+                Path("/not-used"),
+                preflight_phase="unknown",
+            )
+
+        plan_source = inspect.getsource(RUNNER.plan_artifact)
+        self.assertIn(
+            'validate_device_plane_control_core_release_predecessor(\n'
+            '                    payload_dir,\n'
+            '                    preflight_phase="plan",',
+            plan_source,
+        )
+        apply_source = inspect.getsource(RUNNER.apply_artifact)
+        self.assertIn(
+            'validate_device_plane_control_core_release_predecessor(\n'
+            '                        payload_dir,\n'
+            '                        preflight_phase="apply",',
+            apply_source,
+        )
+
+    def test_control_core_preflight_health_excludes_selected_target(self):
+        descriptor = {
+            "preservedServices": [
+                "device-manager",
+                "device-gateway",
+                "device-postgres",
+                "device-backhaul-target",
+            ],
+        }
+        with mock.patch.object(
+            RUNNER,
+            "healthcheck_compose_service",
+        ) as healthcheck:
+            services = (
+                RUNNER.validate_device_plane_control_core_preserved_runtime_health(
+                    descriptor
+                )
+            )
+
+        self.assertEqual(
+            services,
+            (
+                "device-manager",
+                "device-gateway",
+                "device-postgres",
+                "device-backhaul-target",
+            ),
+        )
+        self.assertEqual(
+            healthcheck.call_args_list,
+            [
+                mock.call("device-plane", "device-manager"),
+                mock.call("device-plane", "device-gateway"),
+                mock.call("device-plane", "device-postgres"),
+                mock.call("device-plane", "device-backhaul-target"),
+            ],
+        )
+        self.assertNotIn(
+            mock.call("device-plane", "device-control-core"),
+            healthcheck.call_args_list,
+        )
+
+    def test_control_core_selected_predecessor_may_be_unhealthy(self):
+        inventory = {
+            "schemaVersion": "nodedc.device-plane.runtime-inventory.v1",
+            "composeProject": "nodedc-device-plane",
+            "services": [{
+                "service": "device-control-core",
+                "containerId": "a" * 64,
+                "imageId": "sha256:" + "b" * 64,
+                "status": "running",
+                "running": True,
+                "health": "unhealthy",
+                "restartCount": 4,
+            }],
+        }
+        with mock.patch.object(
+            RUNNER,
+            "device_plane_runtime_inventory",
+            return_value=inventory,
+        ):
+            selected = (
+                RUNNER.validate_device_plane_control_core_selected_predecessor_runtime()
+            )
+        self.assertEqual(selected["health"], "unhealthy")
+
+    def test_control_core_rollback_accepts_restored_unhealthy_boundary(self):
+        service_names = (
+            "device-control-core",
+            "device-manager",
+            "device-gateway",
+            "device-postgres",
+            "device-backhaul-target",
+        )
+        inventory = {
+            "schemaVersion": "nodedc.device-plane.runtime-inventory.v1",
+            "composeProject": "nodedc-device-plane",
+            "services": [
+                {
+                    "service": service,
+                    "containerId": chr(97 + index) * 64,
+                    "imageId": "sha256:" + str(index + 1) * 64,
+                    "status": "running",
+                    "running": True,
+                    "health": "unhealthy" if index == 0 else "healthy",
+                    "restartCount": index,
+                }
+                for index, service in enumerate(service_names)
+            ],
+        }
+        restored_id = "f" * 64
+        restored = {
+            "Id": restored_id,
+            "State": {
+                "Status": "running",
+                "Running": True,
+                "Health": {"Status": "unhealthy"},
+            },
+        }
+        with (
+            mock.patch.object(
+                RUNNER,
+                "healthcheck_compose_service_with_grace",
+            ) as preserved_health,
+            mock.patch.object(
+                RUNNER,
+                "compose_service_container_id",
+                return_value=restored_id,
+            ),
+            mock.patch.object(
+                RUNNER,
+                "inspect_device_plane_container",
+                return_value=restored,
+            ),
+        ):
+            accepted = (
+                RUNNER.accept_device_plane_control_core_rollback_runtime(
+                    inventory
+                )
+            )
+
+        self.assertEqual(accepted["health"], "unhealthy")
+        self.assertEqual(accepted["predecessorHealth"], "unhealthy")
+        self.assertEqual(
+            preserved_health.call_args_list,
+            [
+                mock.call("device-plane", "device-manager"),
+                mock.call("device-plane", "device-gateway"),
+                mock.call("device-plane", "device-postgres"),
+                mock.call("device-plane", "device-backhaul-target"),
+            ],
+        )
+
+    def test_control_core_post_apply_health_includes_backhaul(self):
+        with (
+            mock.patch.object(
+                RUNNER,
+                "healthcheck_compose_service_with_grace",
+            ) as healthcheck,
+            mock.patch.object(
+                RUNNER,
+                "component_healthchecks",
+                return_value=(),
+            ),
+            mock.patch.object(
+                RUNNER,
+                "validate_device_manager_control_plane_runtime",
+            ),
+        ):
+            RUNNER.run_healthchecks(
+                "device-plane",
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_RELEASE_V2_ENTRIES,
+                ("device-control-core",),
+            )
+
+        self.assertEqual(
+            healthcheck.call_args_list,
+            [
+                mock.call("device-plane", "device-control-core"),
+                mock.call("device-plane", "device-manager"),
+                mock.call("device-plane", "device-gateway"),
+                mock.call("device-plane", "device-postgres"),
+                mock.call("device-plane", "device-backhaul-target"),
+            ],
+        )
+
     def test_manager_predecessor_health_is_phase_scoped(self):
         with self.assertRaisesRegex(
             RUNNER.DeployError,