diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 60e009f..99bc70d 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -480,6 +480,31 @@ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT = ( DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 = ( "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7" ) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-control-core-migration-replay-recovery-" + "20260822-044.tgz.20260822-232936" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT = ( + "2026-08-22T20:29:59+00:00" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL = ( + "deployment/" + "device-control-core-migration-replay-checkpoint-recovery-v2.json" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES = ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL, + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL, +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID = ( + "device-control-core-migration-replay-checkpoint-recovery-20260822-046" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT = ( + "nodedc-device-plane-device-control-core-migration-replay-checkpoint-" + "recovery-20260822-046.tgz" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT_SHA256 = ( + "46000c76977fb583fc7c9cf74ecf624efd8b404f7b8d0322e0270e7b8ac6e450" +) DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = ( "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508" ) @@ -517,6 +542,30 @@ DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS = ( "infrastructure_service_instance.ensure", "health_observation.record", ) +DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS = ( + "owner_scope.ensure", + "project.ensure", + "collection.ensure", + "project_grant.upsert", + "adapter_package.ensure", + "adapter_version.register", + "model_profile.register", + "edge.ensure", + "route.ensure", + "enrollment_intent.ensure", +) +DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS = ( + *DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS, + "device.claim", + "device.transfer", + "discovery.reject", + "discovery.expire", +) +DEVICE_PLANE_CONTROL_CORE_REPLAY_009_COMMAND_KINDS = ( + *DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS, + "device_credential_binding.upsert", + "device_credential_binding.revoke", +) DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS = ( "owner_scope.ensure", "project.ensure", @@ -539,6 +588,15 @@ DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS = ( "device_configuration_revision.create", "device_configuration_desired.set", ) +DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS = ( + ("replay-005", DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS), + ("replay-007", DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS), + ("replay-009", DEVICE_PLANE_CONTROL_CORE_REPLAY_009_COMMAND_KINDS), + ("replay-011", DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS), +) +DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES = tuple( + phase for phase, _kinds in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS +) DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS = ( "asset.ensure", "asset_binding.ensure", @@ -4914,6 +4972,7 @@ def allowed_payload_path(component, rel): DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL, DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL, DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL, + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL, DEVICE_PLANE_MANAGER_RECONCILIATION_REL, DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, "packages/device-protocol-contract", @@ -11404,6 +11463,21 @@ def load_artifact(artifact, work_dir): validate_device_plane_control_core_migration_replay_recovery_payload( payload_dir ) + if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + manifest["component"], + entries, + ): + if ( + manifest["id"] + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID + ): + die( + "Device Control Core migration replay checkpoint recovery " + "patch id mismatch" + ) + validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload( + payload_dir + ) if is_device_plane_postgres_bootstrap_slice( manifest["component"], entries, @@ -12085,6 +12159,18 @@ def is_device_plane_control_core_migration_replay_recovery_slice( ) +def is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + component, + entries, +): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) + == DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES + ) + + def is_device_plane_manager_v2_control_plane_slice(component, entries): return ( component == "device-plane" @@ -12911,6 +12997,66 @@ def expected_device_plane_control_core_migration_replay_recovery_descriptor(): } +def expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane." + "device-control-core-migration-replay-checkpoint-recovery.v2" + ), + "mode": "terminal-044-replay-checkpoint-forward-repair", + "failedIncidentAudit": ( + DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID + ), + "failedRecovery": { + "patchId": ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + ), + "artifactSha256": ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 + ), + "failure": "preflight-replay-checkpoint-race", + "startedApply": False, + }, + "sourcePredecessor": { + "path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL, + "sha256": ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 + ), + }, + "sourceTarget": { + "path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL, + "sha256": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256, + }, + "rootCause": ( + "restarting-core-cycles-exact-committed-migration-checkpoints" + ), + "repair": "migration-014-add-constraint-not-valid", + "databasePreflight": ( + "exact-replay-checkpoint-005-007-009-011-and-final-compatible-rows" + ), + "databaseRowMutation": "none", + "databaseSchemaOutcome": ( + "exact-final-migration-016-validated-command-kind-check" + ), + "runtimeAction": "build+recreate-device-control-core-only", + "runtimePredecessor": ( + "proven-degraded-restarting-exact-preapply-image" + ), + "preservedServices": [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "publicIngress": "disabled", + "edgeChannel": ( + "core-initiated-pinned-mtls-registered-edges-only" + ), + "rollback": "source+exact-degraded-predecessor-image-runtime", + } + + def validate_platform_device_core_hub_trust_payload(payload_dir): descriptor = read_strict_json( payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL, @@ -13843,6 +13989,32 @@ def validate_device_plane_control_core_migration_replay_recovery_payload( != expected_device_plane_control_core_migration_replay_recovery_descriptor() ): die("Device Control Core migration replay recovery descriptor mismatch") + validate_device_plane_control_core_migration_014_repair(payload_dir) + return descriptor + + +def validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload( + payload_dir, +): + descriptor = read_strict_json( + payload_dir + / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL, + "Device Control Core migration replay checkpoint recovery descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor() + ): + die( + "Device Control Core migration replay checkpoint recovery " + "descriptor mismatch" + ) + validate_device_plane_control_core_migration_014_repair(payload_dir) + return descriptor + + +def validate_device_plane_control_core_migration_014_repair(payload_dir): migration = payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL try: migration_stat = migration.lstat() @@ -13861,7 +14033,6 @@ def validate_device_plane_control_core_migration_replay_recovery_payload( ) ): die("Device Control Core migration 014 repair mismatch") - return descriptor def is_device_plane_postgres_bootstrap_slice(component, entries): @@ -16875,6 +17046,79 @@ def validate_device_plane_control_core_double_failure_evidence(): } +def validate_device_plane_control_core_migration_replay_recovery_failure(): + failed_artifact = ( + FAILED_DIR + / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT + ) + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Control Core migration recovery 044 failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 + ): + die("Device Control Core migration recovery 044 failed artifact mismatch") + if any( + value.get("id") + == DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + for value in load_state(STATE_FILE) + ): + die("Device Control Core migration recovery 044 applied journal conflict") + records = [ + value + for value in load_state(FAILED_STATE_FILE) + if value.get("id") + == DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + ] + if len(records) != 1: + die("Device Control Core migration recovery 044 failed journal count mismatch") + record = records[0] + if ( + record.get("artifact") + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT + or record.get("backup_id") is not None + or record.get("component") != "device-plane" + or record.get("failed_at") + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT + or record.get("sha256") + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 + or record.get("started_apply") is not False + or record.get("rollback_status") != "not-required" + or record.get("status") != "failed" + or record.get("message") + != "Device Control Core migration recovery database invariant mismatch" + ): + die("Device Control Core migration recovery 044 failed journal mismatch") + with tempfile.TemporaryDirectory( + prefix="device-control-core-migration-recovery-044-failed-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES + ): + die( + "Device Control Core migration recovery 044 failed artifact " + "contract mismatch" + ) + validate_device_plane_control_core_migration_replay_recovery_payload( + failed_payload + ) + return {"artifact": failed_artifact, "record": record} + + def device_plane_control_core_migration_replay_database_invariants_match( evidence, ): @@ -16883,6 +17127,7 @@ def device_plane_control_core_migration_replay_database_invariants_match( and evidence["triggeringReceiptCount"] >= 1 and evidence["constraintValidated"] and evidence["constraintCoversFinalKinds"] + and evidence["constraintPhase"] == "final-016" and evidence["hostTelemetryTableAbsent"] ) @@ -16900,6 +17145,20 @@ def device_plane_control_core_migration_replay_predecessor_invariants_match( ) +def device_plane_control_core_migration_replay_checkpoint_invariants_match( + evidence, +): + return ( + evidence["invalidCommandKindCount"] == 0 + and evidence["triggeringReceiptCount"] >= 1 + and evidence["constraintValidationState"] == "false" + and not evidence["constraintCoversFinalKinds"] + and evidence["constraintPhase"] + in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES + and evidence["hostTelemetryTableAbsent"] + ) + + def emit_device_plane_control_core_migration_replay_database_evidence( evidence, ): @@ -16930,10 +17189,26 @@ def emit_device_plane_control_core_migration_replay_database_evidence( "device_control_core_constraint_matches_replay_011_kinds=" f"{boolean(evidence['constraintMatchesReplay011Kinds'])}" ) + print( + "device_control_core_constraint_phase=" + f"{evidence['constraintPhase']}" + ) + print( + "device_control_core_constraint_matches_known_replay_checkpoint=" + f"{boolean(evidence['constraintMatchesKnownReplayCheckpoint'])}" + ) + print( + "device_control_core_constraint_matches_final_016_kinds=" + f"{boolean(evidence['constraintMatchesFinalKinds'])}" + ) print( "device_control_core_recovery_044_ready=" f"{boolean(evidence['recovery044Ready'])}" ) + print( + "device_control_core_checkpoint_recovery_ready=" + f"{boolean(evidence['checkpointRecoveryReady'])}" + ) print( "device_control_core_recovery_final_state_ready=" f"{boolean(evidence['finalStateReady'])}" @@ -16944,7 +17219,11 @@ def collect_device_plane_control_core_migration_replay_database_evidence( enforce_recovery_invariants=True, expected_state="final", ): - if expected_state not in ("final", "replay-011-predecessor"): + if expected_state not in ( + "final", + "replay-011-predecessor", + "replay-checkpoint-predecessor", + ): die("Device Control Core migration recovery database state is invalid") postgres_ids = device_plane_service_container_ids("device-postgres") if len(postgres_ids) != 1: @@ -16957,10 +17236,6 @@ def collect_device_plane_control_core_migration_replay_database_evidence( "'" + value.replace("'", "''") + "'" for value in DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS ) - replay_011 = ",".join( - "'" + value.replace("'", "''") + "'" - for value in DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS - ) query = f""" with constraint_state as ( select convalidated, @@ -16979,10 +17254,6 @@ constraint_literals as ( '''([^'']+)''', 'g' ) as matched(value) -), -expected_replay_011 as ( - select array_agg(kind order by kind) as values - from unnest(array[{replay_011}]::text[]) as expected(kind) ) select ( select count(*) @@ -16998,11 +17269,8 @@ select ( coalesce((select covers_final::text from constraint_state), 'false'), (to_regclass('public.device_infrastructure_host_telemetry_samples') is null)::text, - coalesce(( - select constraint_literals.values = expected_replay_011.values - from constraint_literals - cross join expected_replay_011 - ), false)::text + coalesce((select array_to_json(values)::text from constraint_literals), + '[]') """.strip() result = subprocess.run( [ @@ -17038,16 +17306,55 @@ select ( or values[2] not in ("true", "false", "missing") or values[3] not in ("true", "false") or values[4] not in ("true", "false") - or values[5] not in ("true", "false") ): die("Device Control Core migration recovery database read failed") + try: + constraint_literal_kinds = json.loads(values[5]) + except (TypeError, ValueError): + die("Device Control Core migration recovery constraint read failed") + if ( + not isinstance(constraint_literal_kinds, list) + or len(constraint_literal_kinds) > 128 + or any( + not isinstance(value, str) + or not re.fullmatch(r"[a-z][a-z0-9._-]{1,127}", value) + for value in constraint_literal_kinds + ) + or len(set(constraint_literal_kinds)) != len(constraint_literal_kinds) + ): + die("Device Control Core migration recovery constraint read failed") + constraint_literal_kinds = tuple(sorted(constraint_literal_kinds)) + constraint_phase_by_kinds = { + tuple(sorted(kinds)): phase + for phase, kinds in ( + *DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS, + ("final-016", DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS), + ) + } + if values[2] == "missing": + constraint_phase = "missing" + else: + constraint_phase = constraint_phase_by_kinds.get( + constraint_literal_kinds, + "unknown", + ) evidence = { "invalidCommandKindCount": int(values[0]), "triggeringReceiptCount": int(values[1]), + "constraintValidationState": values[2], "constraintValidated": values[2] == "true", "constraintCoversFinalKinds": values[3] == "true", "hostTelemetryTableAbsent": values[4] == "true", - "constraintMatchesReplay011Kinds": values[5] == "true", + "constraintLiteralKinds": constraint_literal_kinds, + "constraintPhase": constraint_phase, + "constraintMatchesReplay011Kinds": ( + constraint_phase == "replay-011" + ), + "constraintMatchesKnownReplayCheckpoint": ( + constraint_phase + in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES + ), + "constraintMatchesFinalKinds": constraint_phase == "final-016", "query": query, } evidence["finalStateReady"] = ( @@ -17060,11 +17367,18 @@ select ( evidence ) ) - expected_ready = ( - evidence["recovery044Ready"] - if expected_state == "replay-011-predecessor" - else evidence["finalStateReady"] + evidence["checkpointRecoveryReady"] = ( + device_plane_control_core_migration_replay_checkpoint_invariants_match( + evidence + ) ) + expected_ready = { + "final": evidence["finalStateReady"], + "replay-011-predecessor": evidence["recovery044Ready"], + "replay-checkpoint-predecessor": ( + evidence["checkpointRecoveryReady"] + ), + }[expected_state] if enforce_recovery_invariants and not expected_ready: emit_device_plane_control_core_migration_replay_database_evidence( evidence @@ -17265,6 +17579,78 @@ def validate_device_plane_control_core_migration_replay_recovery_evidence( } +def validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence( + payload_dir, +): + descriptor = ( + validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload( + payload_dir + ) + ) + evidence = validate_device_plane_control_core_double_failure_evidence() + failed_recovery = ( + validate_device_plane_control_core_migration_replay_recovery_failure() + ) + root = component_root("device-plane") + live_migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + if ( + live_migration.is_symlink() + or not live_migration.is_file() + or sha256_file(live_migration) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 + ): + die("Device Control Core migration 014 predecessor mismatch") + for descriptor_rel in ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL, + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL, + ): + installed_descriptor = root / descriptor_rel + if installed_descriptor.exists() or installed_descriptor.is_symlink(): + die( + "Device Control Core migration replay checkpoint recovery " + "already installed" + ) + runtime = ( + validate_device_plane_control_core_migration_replay_preserved_runtime( + evidence + ) + ) + core = runtime["core"] + if ( + core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID + or core["status"] not in ("running", "restarting", "exited") + or core["health"] not in ("starting", "unhealthy", None) + ): + die( + "Device Control Core migration replay checkpoint predecessor " + "mismatch" + ) + image_id = inspect_optional_local_image( + DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "Device Control Core migration replay checkpoint exact predecessor image", + ) + if image_id != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID: + die( + "Device Control Core migration replay checkpoint predecessor " + "image is missing" + ) + database = ( + collect_device_plane_control_core_migration_replay_database_evidence( + expected_state="replay-checkpoint-predecessor", + ) + ) + return { + "mode": descriptor["mode"], + "descriptor": descriptor, + "firstBackup": evidence["firstBackup"], + "secondBackup": evidence["secondBackup"], + "failedRecovery": failed_recovery, + "runtime": runtime["current"], + "core": core, + "database": database, + } + + def accept_device_plane_control_core_migration_replay_recovery(): root = component_root("device-plane") migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL @@ -17316,6 +17702,68 @@ def accept_device_plane_control_core_migration_replay_recovery(): return {"runtime": runtime["current"], "database": database} +def accept_device_plane_control_core_migration_replay_checkpoint_recovery(): + root = component_root("device-plane") + migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + if ( + migration.is_symlink() + or not migration.is_file() + or sha256_file(migration) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256 + ): + die("installed Device Control Core migration 014 repair mismatch") + descriptor = read_strict_json( + root + / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL, + "installed Device Control Core replay checkpoint recovery descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor() + ): + die( + "installed Device Control Core replay checkpoint recovery mismatch" + ) + legacy_descriptor = ( + root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL + ) + if legacy_descriptor.exists() or legacy_descriptor.is_symlink(): + die("legacy Device Control Core migration recovery descriptor appeared") + evidence = { + "secondBackup": ( + BACKUPS_DIR + / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID + ) + } + runtime = ( + validate_device_plane_control_core_migration_replay_preserved_runtime( + evidence + ) + ) + core = runtime["core"] + if ( + core["imageId"] == DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID + or core["status"] != "running" + or core["running"] is not True + or core["health"] != "healthy" + ): + die( + "Device Control Core migration replay checkpoint recovery did " + "not converge" + ) + database = ( + collect_device_plane_control_core_migration_replay_database_evidence( + expected_state="final", + ) + ) + validate_device_manager_control_plane_runtime( + require_edge_channel=True, + core_network_mode="private-egress", + ) + return {"runtime": runtime["current"], "database": database} + + def emit_bounded_device_control_core_failure_logs(): core_ids = device_plane_service_container_ids("device-control-core") if len(core_ids) != 1: @@ -20886,6 +21334,12 @@ def component_services(component, entries=None): ): return ("device-control-core",) + if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + component, + entries, + ): + return ("device-control-core",) + if is_device_plane_manager_only_release_slice(component, entries): return ("device-manager",) @@ -21519,6 +21973,24 @@ def component_builds(component, entries=None): ), ),) + if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + component, + entries, + ): + return (( + DEVICE_PLANE_ROOT, + ( + "build", + "--no-cache", + "--network=host", + "-f", + "services/device-control-core/Dockerfile", + "-t", + DEVICE_PLANE_CONTROL_CORE_IMAGE, + ".", + ), + ),) + if is_device_plane_manager_only_release_slice(component, entries): return (( DEVICE_PLANE_ROOT / "services/device-manager", @@ -29715,6 +30187,7 @@ def plan_artifact(artifact): device_plane_control_core_incident_audit_preflight = None device_plane_control_core_migration_replay_audit_preflight = None device_plane_control_core_migration_replay_recovery_preflight = None + device_plane_control_core_migration_replay_checkpoint_recovery_preflight = None device_plane_backhaul_preflight = None device_plane_backhaul_vps_enrollment_preflight = None device_plane_runtime_before = None @@ -30026,6 +30499,23 @@ def plan_artifact(artifact): payload_dir ) ) + if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + manifest["component"], + entries, + ): + if ( + manifest["id"] + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID + ): + die( + "Device Control Core migration replay checkpoint recovery " + "patch id mismatch" + ) + device_plane_control_core_migration_replay_checkpoint_recovery_preflight = ( + validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence( + payload_dir + ) + ) if is_device_plane_backhaul_target_slice( manifest["component"], entries, @@ -32344,6 +32834,82 @@ def plan_artifact(artifact): "device_plane_rollback=" "source+exact-degraded-predecessor-image-runtime" ) + if ( + device_plane_control_core_migration_replay_checkpoint_recovery_preflight + is not None + ): + recovery = ( + device_plane_control_core_migration_replay_checkpoint_recovery_preflight + ) + database = recovery["database"] + core = recovery["core"] + print(f"device_plane_transition={recovery['mode']}") + print( + "device_control_core_root_cause=" + "restart-replay-cycles-exact-committed-migration-checkpoints" + ) + print( + "device_control_core_failed_recovery_044_artifact_sha256=" + f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256}" + ) + print( + "device_control_core_failed_recovery_044_started_apply=false" + ) + print( + "device_control_core_migration_predecessor_sha256=" + f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256}" + ) + print( + "device_control_core_migration_target_sha256=" + f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256}" + ) + print( + "device_control_core_current_runtime=" + f"{core['containerId']}:{core['imageId']}:" + f"{core['status']}:{core['health']}:" + f"restarts={core['restartCount']}" + ) + print( + "device_control_core_live_command_kind_incompatible_count=" + f"{database['invalidCommandKindCount']}" + ) + print( + "device_control_core_triggering_newer_receipt_count=" + f"{database['triggeringReceiptCount']}" + ) + print( + "device_control_core_current_constraint=" + f"not-validated:exact-{database['constraintPhase']}-command-kinds" + ) + print( + "device_control_core_allowed_predecessor_constraints=" + "exact-replay-005|replay-007|replay-009|replay-011" + ) + print( + "device_control_core_target_constraint=" + "validated:exact-migration-016-command-kinds" + ) + emit_device_plane_control_core_migration_replay_database_evidence( + database + ) + print("device_postgres_row_mutation=none") + print("device_postgres_telemetry_table=absent") + print( + "device_plane_runtime_mutation=" + "build+recreate:device-control-core" + ) + print( + "device_plane_runtime_services=" + "preserved:device-manager,device-gateway,device-postgres," + "device-backhaul-target" + ) + print("device_gateway_public_ingress=disabled") + print("device_gateway_tcp_9921=preserved:loopback-only") + print("gelios=untouched-legacy-only") + print( + "device_plane_rollback=" + "source+exact-degraded-predecessor-image-runtime" + ) if device_plane_backhaul_vps_enrollment_preflight is not None: print( "device_plane_transition=" @@ -32960,6 +33526,10 @@ def rollback_device_plane_apply( "device-plane", entries, ) + or is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + "device-plane", + entries, + ) ) if control_core_release_rollback: before = { @@ -34580,6 +35150,21 @@ def component_healthchecks(component, entries=None, services=None): "commandTransport": "typed-service-ping-v1", }, },) + if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + component, + entries, + ): + return ({ + "url": "http://127.0.0.1:18120/healthz", + "expected_json": { + "ok": True, + "service": "nodedc-device-control-core", + "database": "ready", + "discoveryIngest": "enabled", + "managementApi": "enabled", + "commandTransport": "typed-service-ping-v1", + }, + },) if is_device_plane_manager_control_plane_slice(component, entries): command_transport = ( "typed-service-ping-v1" @@ -35644,6 +36229,38 @@ def run_healthchecks(component, entries=None, services=None): accept_device_plane_control_core_migration_replay_recovery() return + if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + component, + entries, + ): + if tuple(services or ()) != ("device-control-core",): + die( + "Device Control Core migration replay checkpoint recovery " + "service set mismatch" + ) + try: + healthcheck_compose_service_with_grace( + "device-plane", + "device-control-core", + ) + except DeployError: + emit_bounded_device_control_core_failure_logs() + raise + for service in ( + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ): + healthcheck_compose_service_with_grace( + "device-plane", + service, + ) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + accept_device_plane_control_core_migration_replay_checkpoint_recovery() + return + if is_device_plane_control_core_release_slice(component, entries): if tuple(services or ()) != ("device-control-core",): die("Device Control Core release service set mismatch") @@ -36820,6 +37437,21 @@ def apply_artifact(artifact): validate_device_plane_control_core_migration_replay_recovery_evidence( payload_dir ) + if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + component, + entries, + ): + if ( + patch_id + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID + ): + die( + "Device Control Core migration replay checkpoint " + "recovery patch id mismatch" + ) + validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence( + payload_dir + ) if is_device_plane_backhaul_target_slice( component, entries, @@ -37146,6 +37778,10 @@ def apply_artifact(artifact): component, entries, ) + or is_device_plane_control_core_migration_replay_checkpoint_recovery_slice( + component, + entries, + ) ): inventory_services = ( *inventory_services,