From 7b0e6d0bdf48f463f74f2bedd0a98bec1b8e8b98 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 22 Aug 2026 22:11:35 +0300 Subject: [PATCH] chore(deploy): register migration invariant audit --- infra/deploy-runner/nodedc-deploy | 331 +++++++++++++++++- .../test_device_plane_registry.py | 9 + 2 files changed, 332 insertions(+), 8 deletions(-) diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 2e59440..1cf1d94 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -450,6 +450,15 @@ DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES = ( DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID = ( "device-control-core-incident-audit-20260822-043" ) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL = ( + "deployment/device-control-core-migration-replay-audit-v1.json" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_ENTRIES = ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL, +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_PATCH_ID = ( + "device-control-core-migration-replay-audit-20260822-045" +) DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL = ( "deployment/device-control-core-migration-replay-recovery-v1.json" ) @@ -464,6 +473,13 @@ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES = ( DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID = ( "device-control-core-migration-replay-recovery-20260822-044" ) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT = ( + "nodedc-device-plane-device-control-core-migration-replay-recovery-" + "20260822-044.tgz" +) +DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 = ( + "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7" +) DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = ( "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508" ) @@ -4874,6 +4890,7 @@ def allowed_payload_path(component, rel): DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL, DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL, + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL, DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL, DEVICE_PLANE_MANAGER_RECONCILIATION_REL, DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, @@ -11341,6 +11358,18 @@ def load_artifact(artifact, work_dir): entries, ): validate_device_plane_control_core_incident_audit_payload(payload_dir) + if is_device_plane_control_core_migration_replay_audit_slice( + manifest["component"], + entries, + ): + if ( + manifest["id"] + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_PATCH_ID + ): + die("Device Control Core migration replay audit patch id mismatch") + validate_device_plane_control_core_migration_replay_audit_payload( + payload_dir + ) if is_device_plane_control_core_migration_replay_recovery_slice( manifest["component"], entries, @@ -12010,6 +12039,18 @@ def is_device_plane_control_core_incident_audit_slice(component, entries): ) +def is_device_plane_control_core_migration_replay_audit_slice( + component, + entries, +): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) + == DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_ENTRIES + ) + + def is_device_plane_control_core_migration_replay_recovery_slice( component, entries, @@ -12761,6 +12802,45 @@ def expected_device_plane_control_core_incident_audit_descriptor(): } +def expected_device_plane_control_core_migration_replay_audit_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane." + "device-control-core-migration-replay-audit.v1" + ), + "mode": "rejected-recovery-044-live-invariants-read-only-audit", + "allowedOperation": "canonical-plan-only", + "applyAllowed": False, + "rejectedRecovery": { + "patchId": ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + ), + "artifactSha256": ( + DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 + ), + }, + "readOnlyEvidence": [ + "invalid-command-kind-count", + "triggering-receipt-count", + "constraint-validated", + "constraint-covers-final-command-kinds", + "host-telemetry-table-absent", + ], + "runtimeMutation": "none", + "sourceMutation": "none", + "databaseMutation": "none", + "networkMutation": "none", + "secretRead": "none", + "preservedServices": [ + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + } + + def expected_device_plane_control_core_migration_replay_recovery_descriptor(): return { "schemaVersion": ( @@ -13711,6 +13791,22 @@ def validate_device_plane_control_core_incident_audit_payload(payload_dir): return descriptor +def validate_device_plane_control_core_migration_replay_audit_payload( + payload_dir, +): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL, + "Device Control Core migration replay audit descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_control_core_migration_replay_audit_descriptor() + ): + die("Device Control Core migration replay audit descriptor mismatch") + return descriptor + + def validate_device_plane_control_core_migration_replay_recovery_payload( payload_dir, ): @@ -16757,7 +16853,53 @@ def validate_device_plane_control_core_double_failure_evidence(): } -def collect_device_plane_control_core_migration_replay_database_evidence(): +def device_plane_control_core_migration_replay_database_invariants_match( + evidence, +): + return ( + evidence["invalidCommandKindCount"] == 0 + and evidence["triggeringReceiptCount"] >= 1 + and evidence["constraintValidated"] + and evidence["constraintCoversFinalKinds"] + and evidence["hostTelemetryTableAbsent"] + ) + + +def emit_device_plane_control_core_migration_replay_database_evidence( + evidence, +): + def boolean(value): + return "true" if value else "false" + + print( + "device_control_core_invalid_command_kind_count=" + f"{evidence['invalidCommandKindCount']}" + ) + print( + "device_control_core_triggering_receipt_count=" + f"{evidence['triggeringReceiptCount']}" + ) + print( + "device_control_core_constraint_validated=" + f"{boolean(evidence['constraintValidated'])}" + ) + print( + "device_control_core_constraint_covers_final_kinds=" + f"{boolean(evidence['constraintCoversFinalKinds'])}" + ) + print( + "device_control_core_host_telemetry_table_absent=" + f"{boolean(evidence['hostTelemetryTableAbsent'])}" + ) + print( + "device_control_core_recovery_044_ready=" + f"{boolean(evidence['recovery044Ready'])}" + ) + + +def collect_device_plane_control_core_migration_replay_database_evidence( + enforce_recovery_invariants=True, +): postgres_ids = device_plane_service_container_ids("device-postgres") if len(postgres_ids) != 1: die("Device Control Core migration recovery PostgreSQL is missing") @@ -16837,17 +16979,117 @@ select ( "hostTelemetryTableAbsent": values[4] == "true", "query": query, } - if ( - evidence["invalidCommandKindCount"] != 0 - or evidence["triggeringReceiptCount"] < 1 - or not evidence["constraintValidated"] - or not evidence["constraintCoversFinalKinds"] - or not evidence["hostTelemetryTableAbsent"] - ): + evidence["recovery044Ready"] = ( + device_plane_control_core_migration_replay_database_invariants_match( + evidence + ) + ) + if enforce_recovery_invariants and not evidence["recovery044Ready"]: + emit_device_plane_control_core_migration_replay_database_evidence( + evidence + ) die("Device Control Core migration recovery database invariant mismatch") return evidence +def validate_device_plane_control_core_migration_replay_audit_evidence( + payload_dir, +): + descriptor = ( + validate_device_plane_control_core_migration_replay_audit_payload( + payload_dir + ) + ) + failure_evidence = validate_device_plane_control_core_double_failure_evidence() + recovery_artifact = ( + INBOX_DIR + / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT + ) + try: + recovery_stat = recovery_artifact.lstat() + except FileNotFoundError: + die("Device Control Core migration recovery staged artifact is missing") + if ( + stat.S_ISLNK(recovery_stat.st_mode) + or not stat.S_ISREG(recovery_stat.st_mode) + or sha256_file(recovery_artifact) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 + ): + die("Device Control Core migration recovery staged artifact mismatch") + with tempfile.TemporaryDirectory( + prefix="device-control-core-migration-replay-audit-", + dir=TMP_DIR, + ) as directory: + recovery_manifest, recovery_entries, recovery_payload = load_artifact( + recovery_artifact, + Path(directory), + ) + if ( + recovery_manifest.get("id") + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + or recovery_manifest.get("component") != "device-plane" + or recovery_manifest.get("type") != "app-overlay" + or tuple(recovery_entries) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES + ): + die("Device Control Core migration recovery artifact contract mismatch") + validate_device_plane_control_core_migration_replay_recovery_payload( + recovery_payload + ) + if any( + value.get("id") + == DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID + for value in ( + *load_state(APPLIED_STATE_FILE), + *load_state(FAILED_STATE_FILE), + ) + ): + die("Device Control Core migration recovery has terminal journal state") + + root = component_root("device-plane") + live_migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + if ( + live_migration.is_symlink() + or not live_migration.is_file() + or sha256_file(live_migration) + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 + ): + die("Device Control Core migration replay audit source mismatch") + installed_descriptor = ( + root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL + ) + if installed_descriptor.exists() or installed_descriptor.is_symlink(): + die("Device Control Core migration recovery is already installed") + + runtime = ( + validate_device_plane_control_core_migration_replay_preserved_runtime( + failure_evidence + ) + ) + core = runtime["core"] + if ( + core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID + or core["status"] not in ("running", "restarting", "exited") + or core["health"] not in ("starting", "unhealthy", None) + ): + die("Device Control Core migration replay audit runtime mismatch") + database = ( + collect_device_plane_control_core_migration_replay_database_evidence( + enforce_recovery_invariants=False, + ) + ) + return { + "mode": descriptor["mode"], + "descriptor": descriptor, + "firstBackup": failure_evidence["firstBackup"], + "secondBackup": failure_evidence["secondBackup"], + "runtime": runtime["current"], + "core": core, + "database": database, + "recoveryArtifact": recovery_artifact, + } + + def validate_device_plane_control_core_migration_replay_preserved_runtime( evidence, ): @@ -20547,6 +20789,12 @@ def component_services(component, entries=None): if is_device_plane_control_core_incident_audit_slice(component, entries): return () + if is_device_plane_control_core_migration_replay_audit_slice( + component, + entries, + ): + return () + if is_device_plane_control_core_migration_replay_recovery_slice( component, entries, @@ -21162,6 +21410,12 @@ def component_builds(component, entries=None): if is_device_plane_control_core_incident_audit_slice(component, entries): return () + if is_device_plane_control_core_migration_replay_audit_slice( + component, + entries, + ): + return () + if is_device_plane_control_core_migration_replay_recovery_slice( component, entries, @@ -29374,6 +29628,7 @@ def plan_artifact(artifact): device_plane_manager_v2_reconciliation_preflight = None device_plane_control_core_v3_reconciliation_preflight = None device_plane_control_core_incident_audit_preflight = None + device_plane_control_core_migration_replay_audit_preflight = None device_plane_control_core_migration_replay_recovery_preflight = None device_plane_backhaul_preflight = None device_plane_backhaul_vps_enrollment_preflight = None @@ -29658,6 +29913,20 @@ def plan_artifact(artifact): collect_device_plane_control_core_incident_audit() ) device_plane_control_core_incident_audit_preflight = incident + if is_device_plane_control_core_migration_replay_audit_slice( + manifest["component"], + entries, + ): + if ( + manifest["id"] + != DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_PATCH_ID + ): + die("Device Control Core migration replay audit patch id mismatch") + device_plane_control_core_migration_replay_audit_preflight = ( + validate_device_plane_control_core_migration_replay_audit_evidence( + payload_dir + ) + ) if is_device_plane_control_core_migration_replay_recovery_slice( manifest["component"], entries, @@ -31890,6 +32159,40 @@ def plan_artifact(artifact): "read-only:device-control-core,device-manager,device-gateway," "device-postgres,device-backhaul-target" ) + if ( + device_plane_control_core_migration_replay_audit_preflight + is not None + ): + audit = device_plane_control_core_migration_replay_audit_preflight + database = audit["database"] + core = audit["core"] + print(f"device_plane_transition={audit['mode']}") + print("allowed_operation=canonical-plan-only") + print("apply=forbidden") + print("device_plane_build=none") + print("device_plane_runtime_mutation=none") + print("device_plane_source_mutation=none") + print("device_postgres_mutation=none") + print("device_plane_network_mutation=none") + print("device_plane_secret_read=none") + print( + "device_control_core_current_runtime=" + f"{core['containerId']}:{core['imageId']}:" + f"{core['status']}:{core['health']}:" + f"restarts={core['restartCount']}" + ) + print( + "device_control_core_rejected_recovery_artifact_sha256=" + f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256}" + ) + emit_device_plane_control_core_migration_replay_database_evidence( + database + ) + print( + "device_plane_runtime_services=" + "read-only:device-control-core,device-manager,device-gateway," + "device-postgres,device-backhaul-target" + ) if ( device_plane_control_core_migration_replay_recovery_preflight is not None @@ -36308,6 +36611,18 @@ def apply_artifact(artifact): "apply is forbidden" ) + if is_device_plane_control_core_migration_replay_audit_slice( + component, + entries, + ): + validate_device_plane_control_core_migration_replay_audit_payload( + payload_dir + ) + die( + "Device Control Core migration replay audit is " + "canonical-plan-only; apply is forbidden" + ) + if state_has_sha(sha): die(f"artifact sha already applied: {sha}") if state_has_patch_id(patch_id): diff --git a/infra/deploy-runner/test_device_plane_registry.py b/infra/deploy-runner/test_device_plane_registry.py index c048689..2854a6c 100644 --- a/infra/deploy-runner/test_device_plane_registry.py +++ b/infra/deploy-runner/test_device_plane_registry.py @@ -417,11 +417,20 @@ class DevicePlaneRegistryTest(unittest.TestCase): "Health": {"Status": "unhealthy"}, }, } + preserved_inventory = { + **inventory, + "services": inventory["services"][1:], + } with ( mock.patch.object( RUNNER, "healthcheck_compose_service_with_grace", ) as preserved_health, + mock.patch.object( + RUNNER, + "device_plane_runtime_inventory", + return_value=preserved_inventory, + ), mock.patch.object( RUNNER, "compose_service_container_id",