From e642113124bb26af246b470c1a61dcdfc0910ae2 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 22 Aug 2026 21:12:27 +0300 Subject: [PATCH] fix(deploy): recover control core migration replay --- infra/deploy-runner/nodedc-deploy | 643 +++++++++++++++++++++++++++++- 1 file changed, 639 insertions(+), 4 deletions(-) diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 0bbf6f4..2e59440 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -450,6 +450,67 @@ DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES = ( DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID = ( "device-control-core-incident-audit-20260822-043" ) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL = ( + "deployment/device-control-core-migration-replay-recovery-v1.json" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL = ( + "services/device-control-core/migrations/" + "014_device_registry_profile_commands.sql" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES = ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL, + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL, +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID = ( + "device-control-core-migration-replay-recovery-20260822-044" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = ( + "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256 = ( + "38bd86b42828d44c7101d5433ddc36018e92eedeee37b9de296432ad676edd46" +) +DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS = ( + "owner_scope.ensure", + "project.ensure", + "collection.ensure", + "project_grant.upsert", + "adapter_package.ensure", + "adapter_version.register", + "model_profile.register", + "edge.ensure", + "route.ensure", + "enrollment_intent.ensure", + "device.claim", + "device.update", + "device.transfer", + "discovery.reject", + "discovery.expire", + "device_credential_binding.upsert", + "device_credential_binding.revoke", + "device_binding.ensure", + "device_binding.revoke", + "device_configuration_revision.create", + "device_configuration_desired.set", + "asset.ensure", + "asset_binding.ensure", + "asset_binding.close", + "infrastructure_host.ensure", + "infrastructure_endpoint.ensure", + "infrastructure_deployment.ensure", + "infrastructure_service_instance.ensure", + "health_observation.record", +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS = ( + "asset.ensure", + "asset_binding.ensure", + "asset_binding.close", + "infrastructure_host.ensure", + "infrastructure_endpoint.ensure", + "infrastructure_deployment.ensure", + "infrastructure_service_instance.ensure", + "health_observation.record", +) DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT = ( "nodedc-device-plane-device-control-core-release-v3-reconciliation-" "20260822-042.tgz.20260822-195448" @@ -4813,6 +4874,7 @@ def allowed_payload_path(component, rel): DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL, DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL, + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL, DEVICE_PLANE_MANAGER_RECONCILIATION_REL, DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, "packages/device-protocol-contract", @@ -11274,6 +11336,23 @@ def load_artifact(artifact, work_dir): validate_device_plane_control_core_v3_reconciliation_payload( payload_dir ) + if is_device_plane_control_core_incident_audit_slice( + manifest["component"], + entries, + ): + validate_device_plane_control_core_incident_audit_payload(payload_dir) + if is_device_plane_control_core_migration_replay_recovery_slice( + manifest["component"], + entries, + ): + if ( + manifest["id"] + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + ): + die("Device Control Core migration recovery patch id mismatch") + validate_device_plane_control_core_migration_replay_recovery_payload( + payload_dir + ) if is_device_plane_postgres_bootstrap_slice( manifest["component"], entries, @@ -11931,6 +12010,18 @@ def is_device_plane_control_core_incident_audit_slice(component, entries): ) +def is_device_plane_control_core_migration_replay_recovery_slice( + component, + entries, +): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) + == DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES + ) + + def is_device_plane_manager_v2_control_plane_slice(component, entries): return ( component == "device-plane" @@ -12670,6 +12761,54 @@ def expected_device_plane_control_core_incident_audit_descriptor(): } +def expected_device_plane_control_core_migration_replay_recovery_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane." + "device-control-core-migration-replay-recovery.v1" + ), + "mode": "double-rollback-failed-migration-014-forward-repair", + "failedIncidentAudit": ( + DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID + ), + "sourcePredecessor": { + "path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL, + "sha256": ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 + ), + }, + "sourceTarget": { + "path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL, + "sha256": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256, + }, + "rootCause": ( + "intermediate-command-kind-check-revalidated-historical-receipts" + ), + "repair": "migration-014-add-constraint-not-valid", + "databasePreflight": ( + "all-live-command-kinds-covered-by-final-migration-016" + ), + "databaseRowMutation": "none", + "databaseSchemaOutcome": ( + "final-migration-016-validated-command-kind-check" + ), + "runtimeAction": "build+recreate-device-control-core-only", + "runtimePredecessor": "proven-degraded-double-rollback-state", + "preservedServices": [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "publicIngress": "disabled", + "edgeChannel": ( + "core-initiated-pinned-mtls-registered-edges-only" + ), + "rollback": "source+exact-degraded-predecessor-image-runtime", + } + + def validate_platform_device_core_hub_trust_payload(payload_dir): descriptor = read_strict_json( payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL, @@ -13572,6 +13711,41 @@ def validate_device_plane_control_core_incident_audit_payload(payload_dir): return descriptor +def validate_device_plane_control_core_migration_replay_recovery_payload( + payload_dir, +): + descriptor = read_strict_json( + payload_dir + / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL, + "Device Control Core migration replay recovery descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_control_core_migration_replay_recovery_descriptor() + ): + die("Device Control Core migration replay recovery descriptor mismatch") + migration = payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + try: + migration_stat = migration.lstat() + migration_text = migration.read_text(encoding="utf-8") + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("Device Control Core migration 014 repair is unreadable") + if ( + stat.S_ISLNK(migration_stat.st_mode) + or not stat.S_ISREG(migration_stat.st_mode) + or sha256_file(migration) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256 + or ")) not valid;" not in migration_text.lower() + or re.search( + r"(?im)^\s*(?:delete|update|insert|truncate)\b", + migration_text, + ) + ): + die("Device Control Core migration 014 repair mismatch") + return descriptor + + def is_device_plane_postgres_bootstrap_slice(component, entries): return ( component == "device-plane" @@ -16417,6 +16591,12 @@ def validate_device_plane_control_core_incident_audit_evidence(payload_dir): descriptor = validate_device_plane_control_core_incident_audit_payload( payload_dir ) + evidence = validate_device_plane_control_core_double_failure_evidence() + evidence["mode"] = descriptor["mode"] + return evidence + + +def validate_device_plane_control_core_double_failure_evidence(): first_backup = validate_device_plane_control_core_v3_reconciliation_backup() first_failed_artifact = ( FAILED_DIR / DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT @@ -16570,13 +16750,282 @@ def validate_device_plane_control_core_incident_audit_evidence(payload_dir): "device-backhaul-target", )) return { - "mode": descriptor["mode"], + "mode": "double-rollback-failed-evidence", "firstBackup": first_backup, "secondBackup": backup_dir, "runtime": current, } +def collect_device_plane_control_core_migration_replay_database_evidence(): + postgres_ids = device_plane_service_container_ids("device-postgres") + if len(postgres_ids) != 1: + die("Device Control Core migration recovery PostgreSQL is missing") + allowed = ",".join( + "'" + value.replace("'", "''") + "'" + for value in DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS + ) + triggering = ",".join( + "'" + value.replace("'", "''") + "'" + for value in DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS + ) + query = f""" +with constraint_state as ( + select convalidated, + (select bool_and(pg_get_constraintdef(c.oid) like '%' || kind || '%') + from unnest(array[{allowed}]::text[]) as kind) as covers_final + from pg_constraint c + where c.conrelid = 'public.device_management_command_receipts'::regclass + and c.conname = 'device_management_command_receipts_command_kind_check' +) +select ( + select count(*) + from device_management_command_receipts + where not (command_kind = any(array[{allowed}]::text[])) + )::text, + ( + select count(*) + from device_management_command_receipts + where command_kind = any(array[{triggering}]::text[]) + )::text, + coalesce((select convalidated::text from constraint_state), 'missing'), + coalesce((select covers_final::text from constraint_state), 'false'), + (to_regclass('public.device_infrastructure_host_telemetry_samples') + is null)::text +""".strip() + result = subprocess.run( + [ + str(DOCKER), + "exec", + postgres_ids[0], + "psql", + "-X", + "-qAt", + "-v", + "ON_ERROR_STOP=1", + "-U", + "device_plane", + "-d", + "device_plane", + "-F", + "\t", + "-c", + query, + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + values = result.stdout.strip().split("\t") + if ( + result.returncode != 0 + or result.stderr.strip() + or len(values) != 5 + or not re.fullmatch(r"[0-9]+", values[0]) + or not re.fullmatch(r"[0-9]+", values[1]) + or values[2] not in ("true", "false", "missing") + or values[3] not in ("true", "false") + or values[4] not in ("true", "false") + ): + die("Device Control Core migration recovery database read failed") + evidence = { + "invalidCommandKindCount": int(values[0]), + "triggeringReceiptCount": int(values[1]), + "constraintValidated": values[2] == "true", + "constraintCoversFinalKinds": values[3] == "true", + "hostTelemetryTableAbsent": values[4] == "true", + "query": query, + } + if ( + evidence["invalidCommandKindCount"] != 0 + or evidence["triggeringReceiptCount"] < 1 + or not evidence["constraintValidated"] + or not evidence["constraintCoversFinalKinds"] + or not evidence["hostTelemetryTableAbsent"] + ): + die("Device Control Core migration recovery database invariant mismatch") + return evidence + + +def validate_device_plane_control_core_migration_replay_preserved_runtime( + evidence, +): + expected_services = { + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + } + current = device_plane_runtime_inventory(tuple(sorted(expected_services))) + if set(device_plane_inventory_service_names(current)) != expected_services: + die("Device Control Core migration recovery runtime is incomplete") + second_before = read_strict_json( + evidence["secondBackup"] / "runtime-before.json", + "Device Control Core second failed-apply runtime inventory", + max_bytes=64 * 1024, + ) + if set(device_plane_inventory_service_names(second_before)) != expected_services: + die("Device Control Core second failed-apply runtime evidence mismatch") + before = {item["service"]: item for item in second_before["services"]} + observed = {item["service"]: item for item in current["services"]} + for service in expected_services - {"device-control-core"}: + if ( + observed[service]["containerId"] != before[service]["containerId"] + or observed[service]["imageId"] != before[service]["imageId"] + or observed[service]["status"] != "running" + or observed[service]["running"] is not True + or observed[service]["health"] != "healthy" + ): + die( + "Device Control Core migration recovery changed preserved " + f"service: {service}" + ) + return {"current": current, "core": observed["device-control-core"]} + + +def validate_device_plane_control_core_migration_replay_recovery_evidence( + payload_dir, +): + descriptor = ( + validate_device_plane_control_core_migration_replay_recovery_payload( + payload_dir + ) + ) + evidence = validate_device_plane_control_core_double_failure_evidence() + root = component_root("device-plane") + live_migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + if ( + live_migration.is_symlink() + or not live_migration.is_file() + or sha256_file(live_migration) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 + ): + die("Device Control Core migration 014 predecessor mismatch") + installed_descriptor = ( + root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL + ) + if installed_descriptor.exists() or installed_descriptor.is_symlink(): + die("Device Control Core migration replay recovery already installed") + runtime = ( + validate_device_plane_control_core_migration_replay_preserved_runtime( + evidence + ) + ) + core = runtime["core"] + if ( + core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID + or core["status"] not in ("running", "restarting", "exited") + or core["health"] not in ("starting", "unhealthy", None) + ): + die("Device Control Core migration recovery predecessor mismatch") + image_id = inspect_optional_local_image( + DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "Device Control Core migration recovery exact predecessor image", + ) + if image_id != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID: + die("Device Control Core migration recovery predecessor image is missing") + database = ( + collect_device_plane_control_core_migration_replay_database_evidence() + ) + return { + "mode": descriptor["mode"], + "descriptor": descriptor, + "firstBackup": evidence["firstBackup"], + "secondBackup": evidence["secondBackup"], + "runtime": runtime["current"], + "core": core, + "database": database, + } + + +def accept_device_plane_control_core_migration_replay_recovery(): + root = component_root("device-plane") + migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + if ( + migration.is_symlink() + or not migration.is_file() + or sha256_file(migration) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256 + ): + die("installed Device Control Core migration 014 repair mismatch") + descriptor = read_strict_json( + root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL, + "installed Device Control Core migration replay recovery descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_control_core_migration_replay_recovery_descriptor() + ): + die("installed Device Control Core migration recovery mismatch") + evidence = { + "secondBackup": ( + BACKUPS_DIR + / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID + ) + } + runtime = ( + validate_device_plane_control_core_migration_replay_preserved_runtime( + evidence + ) + ) + core = runtime["core"] + if ( + core["imageId"] == DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID + or core["status"] != "running" + or core["running"] is not True + or core["health"] != "healthy" + ): + die("Device Control Core migration recovery did not converge") + database = ( + collect_device_plane_control_core_migration_replay_database_evidence() + ) + validate_device_manager_control_plane_runtime( + require_edge_channel=True, + core_network_mode="private-egress", + ) + return {"runtime": runtime["current"], "database": database} + + +def emit_bounded_device_control_core_failure_logs(): + core_ids = device_plane_service_container_ids("device-control-core") + if len(core_ids) != 1: + return + result = subprocess.run( + [str(DOCKER), "logs", "--tail", "160", core_ids[0]], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != 0: + return + selected = [] + for raw_line in f"{result.stdout}\n{result.stderr}".splitlines(): + line = re.sub(r"\s+", " ", raw_line.strip()) + if not line or not re.search( + r"(?:error|exception|failed|constraint|migration|postgres|relation)", + line, + re.IGNORECASE, + ): + continue + if re.search( + r"(?:authorization|bearer|password|private[_ -]?key|token=|secret=)", + line, + re.IGNORECASE, + ): + line = "[sensitive log line redacted]" + if line not in selected: + selected.append(line[:480]) + for index, line in enumerate(selected[-16:], start=1): + print( + f"device_control_core_failure_log_{index:02d}={line}", + file=sys.stderr, + ) + + def collect_device_plane_control_core_incident_audit(): core_ids = device_plane_service_container_ids("device-control-core") postgres_ids = device_plane_service_container_ids("device-postgres") @@ -16849,6 +17298,25 @@ def accept_device_plane_control_core_rollback_runtime(runtime_before): for service in expected_services[1:]: healthcheck_compose_service_with_grace("device-plane", service) + preserved = device_plane_runtime_inventory(expected_services[1:]) + if set(device_plane_inventory_service_names(preserved)) != set( + expected_services[1:] + ): + die("Device Control Core rollback preserved runtime is incomplete") + for item in preserved["services"]: + expected = before[item["service"]] + if ( + item["containerId"] != expected["containerId"] + or item["imageId"] != expected["imageId"] + or item["status"] != "running" + or item["running"] is not True + or item["health"] != "healthy" + ): + die( + "Device Control Core rollback changed preserved service: " + f"{item['service']}" + ) + container_id = compose_service_container_id( "device-plane", "device-control-core", @@ -20079,6 +20547,12 @@ def component_services(component, entries=None): if is_device_plane_control_core_incident_audit_slice(component, entries): return () + if is_device_plane_control_core_migration_replay_recovery_slice( + component, + entries, + ): + return ("device-control-core",) + if is_device_plane_manager_only_release_slice(component, entries): return ("device-manager",) @@ -20688,6 +21162,24 @@ def component_builds(component, entries=None): if is_device_plane_control_core_incident_audit_slice(component, entries): return () + if is_device_plane_control_core_migration_replay_recovery_slice( + component, + entries, + ): + return (( + DEVICE_PLANE_ROOT, + ( + "build", + "--no-cache", + "--network=host", + "-f", + "services/device-control-core/Dockerfile", + "-t", + DEVICE_PLANE_CONTROL_CORE_IMAGE, + ".", + ), + ),) + if is_device_plane_manager_only_release_slice(component, entries): return (( DEVICE_PLANE_ROOT / "services/device-manager", @@ -28882,6 +29374,7 @@ def plan_artifact(artifact): device_plane_manager_v2_reconciliation_preflight = None device_plane_control_core_v3_reconciliation_preflight = None device_plane_control_core_incident_audit_preflight = None + device_plane_control_core_migration_replay_recovery_preflight = None device_plane_backhaul_preflight = None device_plane_backhaul_vps_enrollment_preflight = None device_plane_runtime_before = None @@ -29165,6 +29658,20 @@ def plan_artifact(artifact): collect_device_plane_control_core_incident_audit() ) device_plane_control_core_incident_audit_preflight = incident + if is_device_plane_control_core_migration_replay_recovery_slice( + manifest["component"], + entries, + ): + if ( + manifest["id"] + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + ): + die("Device Control Core migration recovery patch id mismatch") + device_plane_control_core_migration_replay_recovery_preflight = ( + validate_device_plane_control_core_migration_replay_recovery_evidence( + payload_dir + ) + ) if is_device_plane_backhaul_target_slice( manifest["component"], entries, @@ -31383,6 +31890,65 @@ def plan_artifact(artifact): "read-only:device-control-core,device-manager,device-gateway," "device-postgres,device-backhaul-target" ) + if ( + device_plane_control_core_migration_replay_recovery_preflight + is not None + ): + recovery = ( + device_plane_control_core_migration_replay_recovery_preflight + ) + database = recovery["database"] + core = recovery["core"] + print(f"device_plane_transition={recovery['mode']}") + print( + "device_control_core_root_cause=" + "migration-014-intermediate-constraint-revalidated-" + "newer-valid-receipts" + ) + print( + "device_control_core_migration_predecessor_sha256=" + f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256}" + ) + print( + "device_control_core_migration_target_sha256=" + f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256}" + ) + print( + "device_control_core_current_runtime=" + f"{core['containerId']}:{core['imageId']}:" + f"{core['status']}:{core['health']}:" + f"restarts={core['restartCount']}" + ) + print( + "device_control_core_live_command_kind_incompatible_count=" + f"{database['invalidCommandKindCount']}" + ) + print( + "device_control_core_triggering_newer_receipt_count=" + f"{database['triggeringReceiptCount']}" + ) + print( + "device_control_core_final_constraint=" + "validated:covers-migration-016-command-kinds" + ) + print("device_postgres_row_mutation=none") + print("device_postgres_telemetry_table=absent") + print( + "device_plane_runtime_mutation=" + "build+recreate:device-control-core" + ) + print( + "device_plane_runtime_services=" + "preserved:device-manager,device-gateway,device-postgres," + "device-backhaul-target" + ) + print("device_gateway_public_ingress=disabled") + print("device_gateway_tcp_9921=preserved:loopback-only") + print("gelios=untouched-legacy-only") + print( + "device_plane_rollback=" + "source+exact-degraded-predecessor-image-runtime" + ) if device_plane_backhaul_vps_enrollment_preflight is not None: print( "device_plane_transition=" @@ -31990,9 +32556,15 @@ def rollback_device_plane_apply( if not runtime_started or not baseline_services: return f"source-restored-runtime-unchanged:{restored_count}" - control_core_release_rollback = is_device_plane_control_core_release_slice( - "device-plane", - entries, + control_core_release_rollback = ( + is_device_plane_control_core_release_slice( + "device-plane", + entries, + ) + or is_device_plane_control_core_migration_replay_recovery_slice( + "device-plane", + entries, + ) ) if control_core_release_rollback: before = { @@ -33598,6 +34170,21 @@ def component_healthchecks(component, entries=None, services=None): "deviceCoreInternalAccessConfigured": True, }, },) + if is_device_plane_control_core_migration_replay_recovery_slice( + component, + entries, + ): + return ({ + "url": "http://127.0.0.1:18120/healthz", + "expected_json": { + "ok": True, + "service": "nodedc-device-control-core", + "database": "ready", + "discoveryIngest": "enabled", + "managementApi": "enabled", + "commandTransport": "typed-service-ping-v1", + }, + },) if is_device_plane_manager_control_plane_slice(component, entries): command_transport = ( "typed-service-ping-v1" @@ -34633,6 +35220,35 @@ def run_healthchecks(component, entries=None, services=None): ) return + if is_device_plane_control_core_migration_replay_recovery_slice( + component, + entries, + ): + if tuple(services or ()) != ("device-control-core",): + die("Device Control Core migration recovery service set mismatch") + try: + healthcheck_compose_service_with_grace( + "device-plane", + "device-control-core", + ) + except DeployError: + emit_bounded_device_control_core_failure_logs() + raise + for service in ( + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ): + healthcheck_compose_service_with_grace( + "device-plane", + service, + ) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + accept_device_plane_control_core_migration_replay_recovery() + return + if is_device_plane_control_core_release_slice(component, entries): if tuple(services or ()) != ("device-control-core",): die("Device Control Core release service set mismatch") @@ -35782,6 +36398,21 @@ def apply_artifact(artifact): validate_device_plane_control_core_v3_reconciliation_evidence( payload_dir ) + if is_device_plane_control_core_migration_replay_recovery_slice( + component, + entries, + ): + if ( + patch_id + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + ): + die( + "Device Control Core migration recovery patch id " + "mismatch" + ) + validate_device_plane_control_core_migration_replay_recovery_evidence( + payload_dir + ) if is_device_plane_backhaul_target_slice( component, entries, @@ -36104,6 +36735,10 @@ def apply_artifact(artifact): component, entries, ) + or is_device_plane_control_core_migration_replay_recovery_slice( + component, + entries, + ) ): inventory_services = ( *inventory_services,