SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(deploy): allow unhealthy selected Core recovery

Browse Source
This commit is contained in:
Codex
2026-08-22 01:27:00 +03:00
parent 26dbcfd262
commit c1c9818e30
2 changed files with 323 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+132 -23
View File
@@ -12877,7 +12877,35 @@ def validate_device_plane_edge_core_channel_upgrade_v4_predecessor(
}
def validate_device_plane_control_core_release_predecessor(payload_dir):
def device_plane_control_core_preserved_runtime_health_services(descriptor):
services = tuple(descriptor.get("preservedServices") or ())
expected = (
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
)
if services != expected:
die("Device Control Core preserved service set mismatch")
return services
def validate_device_plane_control_core_preserved_runtime_health(descriptor):
services = device_plane_control_core_preserved_runtime_health_services(
descriptor
)
for service in services:
healthcheck_compose_service("device-plane", service)
return services
def validate_device_plane_control_core_release_predecessor(
payload_dir,
*,
preflight_phase,
):
if preflight_phase not in ("plan", "apply"):
die("Device Control Core predecessor preflight phase is invalid")
descriptor = validate_device_plane_control_core_release_payload(
payload_dir
)
@@ -12981,13 +13009,16 @@ def validate_device_plane_control_core_release_predecessor(payload_dir):
identity_state = inspect_device_edge_channel_core_identity_state()
if identity_state != "valid-reuse-at-apply":
die("Device Control Core release requires the valid active identity")
for service in (
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
):
healthcheck_compose_service("device-plane", service)
selected_runtime = (
validate_device_plane_control_core_selected_predecessor_runtime()
)
preserved_runtime_health = "deferred-to-apply"
if preflight_phase == "apply":
preserved_runtime_health = (
validate_device_plane_control_core_preserved_runtime_health(
descriptor
)
)
validate_device_manager_control_plane_runtime(
require_edge_channel=True,
core_network_mode="private-egress",
@@ -12997,6 +13028,8 @@ def validate_device_plane_control_core_release_predecessor(payload_dir):
"descriptor": descriptor,
"identityState": identity_state,
"predecessorArtifact": artifact,
"selectedRuntime": selected_runtime,
"preservedRuntimeHealth": preserved_runtime_health,
}
@@ -13287,6 +13320,7 @@ def device_plane_inventory_service_names(inventory):
or item.get("service") not in (
*DEVICE_PLANE_RUNTIME_SERVICES,
"device-manager",
DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,
)
or not isinstance(item.get("containerId"), str)
or not re.fullmatch(r"[a-f0-9]{64}", item["containerId"])
@@ -13311,6 +13345,80 @@ def device_plane_inventory_service_names(inventory):
return tuple(names)
def validate_device_plane_control_core_selected_predecessor_runtime():
inventory = device_plane_runtime_inventory(("device-control-core",))
names = device_plane_inventory_service_names(inventory)
if names != ("device-control-core",):
die("Device Control Core selected predecessor runtime is missing")
selected = inventory["services"][0]
if (
selected["status"] != "running"
or selected["running"] is not True
or selected["health"] not in ("healthy", "unhealthy")
):
die("Device Control Core selected predecessor state is invalid")
return selected
def accept_device_plane_control_core_rollback_runtime(runtime_before):
expected_services = (
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
)
names = device_plane_inventory_service_names(runtime_before)
if set(names) != set(expected_services):
die("Device Control Core rollback predecessor inventory mismatch")
before = {
item["service"]: item
for item in runtime_before["services"]
}
selected_before = before["device-control-core"]
if (
selected_before["status"] != "running"
or selected_before["running"] is not True
or selected_before["health"] not in ("healthy", "unhealthy")
):
die("Device Control Core rollback predecessor state is invalid")
for service in expected_services[1:]:
healthcheck_compose_service_with_grace("device-plane", service)
container_id = compose_service_container_id(
"device-plane",
"device-control-core",
)
if container_id == selected_before["containerId"]:
die("Device Control Core rollback generation was not recreated")
accepted_health = {"healthy"}
if selected_before["health"] == "unhealthy":
accepted_health.add("unhealthy")
last_status = "unknown"
for attempt in range(1, 61):
container = inspect_device_plane_container(container_id)
state = container.get("State") or {}
health = (state.get("Health") or {}).get("Status")
last_status = health or state.get("Status") or "unknown"
if (
state.get("Status") == "running"
and state.get("Running") is True
and health in accepted_health
):
return {
"containerId": container_id,
"health": health,
"predecessorHealth": selected_before["health"],
}
if attempt < 60:
time.sleep(5)
die(
"Device Control Core rollback state did not converge: "
f"{last_status}"
)
def validate_device_plane_foundation_recovery_evidence(payload_dir):
backup_dir = BACKUPS_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID
try:
@@ -19354,7 +19462,8 @@ def plan_artifact(artifact):
):
device_plane_control_core_release_preflight = (
validate_device_plane_control_core_release_predecessor(
payload_dir
payload_dir,
preflight_phase="plan",
)
)
if is_device_plane_manager_reconciliation_slice(
@@ -21821,16 +21930,9 @@ def rollback_device_plane_apply(
"device-plane",
entries,
):
for service in (
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
):
healthcheck_compose_service_with_grace(
"device-plane",
service,
)
accept_device_plane_control_core_rollback_runtime(
runtime_inventory
)
validate_device_manager_control_plane_runtime(
require_edge_channel=True,
core_network_mode="private-egress",
@@ -23595,6 +23697,7 @@ def run_healthchecks(component, entries=None, services=None):
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
):
healthcheck_compose_service_with_grace(
"device-plane",
@@ -24659,7 +24762,8 @@ def apply_artifact(artifact):
entries,
):
validate_device_plane_control_core_release_predecessor(
payload_dir
payload_dir,
preflight_phase="apply",
)
if is_device_plane_manager_reconciliation_slice(
component,
@@ -24986,11 +25090,16 @@ def apply_artifact(artifact):
component,
entries,
)
or is_device_plane_edge_core_channel_bootstrap_slice(
):
inventory_services = (
*inventory_services,
"device-manager",
"device-backhaul-target",
)
elif is_device_plane_edge_core_channel_bootstrap_slice(
component,
entries,
)
):
):
inventory_services = (
*inventory_services,
"device-manager",
infra/deploy-runner/test_device_plane_registry.py
+191
View File
@@ -28,6 +28,197 @@ RUNNER = load_runner()
class DevicePlaneRegistryTest(unittest.TestCase):
def test_control_core_predecessor_health_is_phase_scoped(self):
with self.assertRaisesRegex(
RUNNER.DeployError,
"preflight phase is invalid",
):
RUNNER.validate_device_plane_control_core_release_predecessor(
Path("/not-used"),
preflight_phase="unknown",
)
plan_source = inspect.getsource(RUNNER.plan_artifact)
self.assertIn(
'validate_device_plane_control_core_release_predecessor(\n'
' payload_dir,\n'
' preflight_phase="plan",',
plan_source,
)
apply_source = inspect.getsource(RUNNER.apply_artifact)
self.assertIn(
'validate_device_plane_control_core_release_predecessor(\n'
' payload_dir,\n'
' preflight_phase="apply",',
apply_source,
)
def test_control_core_preflight_health_excludes_selected_target(self):
descriptor = {
"preservedServices": [
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
}
with mock.patch.object(
RUNNER,
"healthcheck_compose_service",
) as healthcheck:
services = (
RUNNER.validate_device_plane_control_core_preserved_runtime_health(
descriptor
)
)
self.assertEqual(
services,
(
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
),
)
self.assertEqual(
healthcheck.call_args_list,
[
mock.call("device-plane", "device-manager"),
mock.call("device-plane", "device-gateway"),
mock.call("device-plane", "device-postgres"),
mock.call("device-plane", "device-backhaul-target"),
],
)
self.assertNotIn(
mock.call("device-plane", "device-control-core"),
healthcheck.call_args_list,
)
def test_control_core_selected_predecessor_may_be_unhealthy(self):
inventory = {
"schemaVersion": "nodedc.device-plane.runtime-inventory.v1",
"composeProject": "nodedc-device-plane",
"services": [{
"service": "device-control-core",
"containerId": "a" * 64,
"imageId": "sha256:" + "b" * 64,
"status": "running",
"running": True,
"health": "unhealthy",
"restartCount": 4,
}],
}
with mock.patch.object(
RUNNER,
"device_plane_runtime_inventory",
return_value=inventory,
):
selected = (
RUNNER.validate_device_plane_control_core_selected_predecessor_runtime()
)
self.assertEqual(selected["health"], "unhealthy")
def test_control_core_rollback_accepts_restored_unhealthy_boundary(self):
service_names = (
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
)
inventory = {
"schemaVersion": "nodedc.device-plane.runtime-inventory.v1",
"composeProject": "nodedc-device-plane",
"services": [
{
"service": service,
"containerId": chr(97 + index) * 64,
"imageId": "sha256:" + str(index + 1) * 64,
"status": "running",
"running": True,
"health": "unhealthy" if index == 0 else "healthy",
"restartCount": index,
}
for index, service in enumerate(service_names)
],
}
restored_id = "f" * 64
restored = {
"Id": restored_id,
"State": {
"Status": "running",
"Running": True,
"Health": {"Status": "unhealthy"},
},
}
with (
mock.patch.object(
RUNNER,
"healthcheck_compose_service_with_grace",
) as preserved_health,
mock.patch.object(
RUNNER,
"compose_service_container_id",
return_value=restored_id,
),
mock.patch.object(
RUNNER,
"inspect_device_plane_container",
return_value=restored,
),
):
accepted = (
RUNNER.accept_device_plane_control_core_rollback_runtime(
inventory
)
)
self.assertEqual(accepted["health"], "unhealthy")
self.assertEqual(accepted["predecessorHealth"], "unhealthy")
self.assertEqual(
preserved_health.call_args_list,
[
mock.call("device-plane", "device-manager"),
mock.call("device-plane", "device-gateway"),
mock.call("device-plane", "device-postgres"),
mock.call("device-plane", "device-backhaul-target"),
],
)
def test_control_core_post_apply_health_includes_backhaul(self):
with (
mock.patch.object(
RUNNER,
"healthcheck_compose_service_with_grace",
) as healthcheck,
mock.patch.object(
RUNNER,
"component_healthchecks",
return_value=(),
),
mock.patch.object(
RUNNER,
"validate_device_manager_control_plane_runtime",
),
):
RUNNER.run_healthchecks(
"device-plane",
RUNNER.DEVICE_PLANE_CONTROL_CORE_RELEASE_V2_ENTRIES,
("device-control-core",),
)
self.assertEqual(
healthcheck.call_args_list,
[
mock.call("device-plane", "device-control-core"),
mock.call("device-plane", "device-manager"),
mock.call("device-plane", "device-gateway"),
mock.call("device-plane", "device-postgres"),
mock.call("device-plane", "device-backhaul-target"),
],
)
def test_manager_predecessor_health_is_phase_scoped(self):
with self.assertRaisesRegex(
RUNNER.DeployError,