diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index f085a74..2db792c 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -13350,14 +13350,10 @@ def validate_device_plane_control_core_selected_predecessor_runtime(): names = device_plane_inventory_service_names(inventory) if names != ("device-control-core",): die("Device Control Core selected predecessor runtime is missing") - selected = inventory["services"][0] - if ( - selected["status"] != "running" - or selected["running"] is not True - or selected["health"] not in ("healthy", "unhealthy") - ): - die("Device Control Core selected predecessor state is invalid") - return selected + # This is the selected repair target, not a preserved dependency. Its + # observed Docker state is evidence for rollback only; health and state + # acceptance belongs to the new generation after recreate. + return inventory["services"][0] def accept_device_plane_control_core_rollback_runtime(runtime_before): @@ -13376,12 +13372,11 @@ def accept_device_plane_control_core_rollback_runtime(runtime_before): for item in runtime_before["services"] } selected_before = before["device-control-core"] - if ( - selected_before["status"] != "running" - or selected_before["running"] is not True - or selected_before["health"] not in ("healthy", "unhealthy") - ): - die("Device Control Core rollback predecessor state is invalid") + predecessor_was_healthy = ( + selected_before["status"] == "running" + and selected_before["running"] is True + and selected_before["health"] == "healthy" + ) for service in expected_services[1:]: healthcheck_compose_service_with_grace("device-plane", service) @@ -13392,24 +13387,38 @@ def accept_device_plane_control_core_rollback_runtime(runtime_before): ) if container_id == selected_before["containerId"]: die("Device Control Core rollback generation was not recreated") - accepted_health = {"healthy"} - if selected_before["health"] == "unhealthy": - accepted_health.add("unhealthy") last_status = "unknown" for attempt in range(1, 61): container = inspect_device_plane_container(container_id) state = container.get("State") or {} health = (state.get("Health") or {}).get("Status") last_status = health or state.get("Status") or "unknown" - if ( + restored_healthy = ( state.get("Status") == "running" and state.get("Running") is True - and health in accepted_health + and health == "healthy" + ) + restored_repair_boundary = ( + state.get("Status") in ( + "created", + "running", + "paused", + "restarting", + "exited", + ) + and isinstance(state.get("Running"), bool) + and health in (None, "starting", "healthy", "unhealthy") + ) + if restored_healthy or ( + not predecessor_was_healthy + and restored_repair_boundary ): return { "containerId": container_id, "health": health, + "status": state.get("Status"), "predecessorHealth": selected_before["health"], + "predecessorStatus": selected_before["status"], } if attempt < 60: time.sleep(5) diff --git a/infra/deploy-runner/test_device_plane_registry.py b/infra/deploy-runner/test_device_plane_registry.py index 74cccbf..cbecfdd 100644 --- a/infra/deploy-runner/test_device_plane_registry.py +++ b/infra/deploy-runner/test_device_plane_registry.py @@ -119,6 +119,31 @@ class DevicePlaneRegistryTest(unittest.TestCase): ) self.assertEqual(selected["health"], "unhealthy") + def test_control_core_selected_predecessor_may_be_restarting(self): + inventory = { + "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [{ + "service": "device-control-core", + "containerId": "a" * 64, + "imageId": "sha256:" + "b" * 64, + "status": "restarting", + "running": True, + "health": "starting", + "restartCount": 5, + }], + } + with mock.patch.object( + RUNNER, + "device_plane_runtime_inventory", + return_value=inventory, + ): + selected = ( + RUNNER.validate_device_plane_control_core_selected_predecessor_runtime() + ) + self.assertEqual(selected["status"], "restarting") + self.assertEqual(selected["health"], "starting") + def test_control_core_rollback_accepts_restored_unhealthy_boundary(self): service_names = ( "device-control-core", @@ -175,6 +200,7 @@ class DevicePlaneRegistryTest(unittest.TestCase): ) self.assertEqual(accepted["health"], "unhealthy") + self.assertEqual(accepted["status"], "running") self.assertEqual(accepted["predecessorHealth"], "unhealthy") self.assertEqual( preserved_health.call_args_list,