fix(device-plane): reconcile failed manager activation

2026-08-11 09:35:58 +03:00
parent 8defd55715
commit 37bdbebb4e
9 changed files with 735 additions and 18 deletions
@@ -315,6 +315,64 @@ DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING = (
    "services/device-manager",
    DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_REL,
 )
+DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL = (
+    "deployment/device-manager-control-plane-v2-reconciliation-v1.json"
+)
+DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES = (
+    DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
+)
+DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID = (
+    "device-manager-control-plane-20260811-003"
+)
+DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT = (
+    "nodedc-device-plane-device-manager-control-plane-20260811-003.tgz."
+    "20260811-012505"
+)
+DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 = (
+    "ba29618ffbfed55448768794f28b18dda439ddb39a1d2a4f1dece19de7f29990"
+)
+DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID = (
+    "device-manager-control-plane-v2-reconciliation-20260811-004"
+)
+DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID = (
+    "device-plane-device-manager-control-plane-20260811-003-"
+    "20260811-012505"
+)
+DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_SHA256 = {
+    "manifest.env": (
+        "4f7535e5eeee0bc3339173c2425787c153528d26f2c60f6cdd386ad3359e05cf"
+    ),
+    "files.txt": (
+        "191e81991258673881435083e4a6733062ec0e30afb9b80a875c1a6581dd18a2"
+    ),
+    "existing-files.txt": (
+        "b605a86ce5ed6c0adb7b4cbeb5758795fe4219367df2c18db9411825f232e308"
+    ),
+    "missing-files.txt": (
+        "c1ab9da97b257b467fd222eda0e34b5a7b29b628c71f10a5f4069b2991b3d39d"
+    ),
+    "source-before.tgz": (
+        "32d6d6341e410806bcff7007c5eacec34090bc337483222f396f19ca2edad9be"
+    ),
+    "runtime-before.json": (
+        "af8f2c71caeedcd61ebf02de98ff6c61d35c832fbd446a6b38a7e7c16150c2bd"
+    ),
+}
+DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING = (
+    ".dockerignore",
+    "package.json",
+    "package-lock.json",
+    "packages/device-protocol-contract",
+    "packages/arusnavi-b2-adapter",
+    "services/device-control-core",
+    "services/device-gateway/package.json",
+    "services/device-edge-relay/package.json",
+)
+DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING = (
+    DEVICE_PLANE_MANAGER_COMPOSE_REL,
+    "services/device-manager",
+    DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
+)
 DEVICE_PLANE_FOUNDATION_RECOVERY_REL = (
    "deployment/device-plane-foundation-recovery-v1.json"
 )
@@ -3202,6 +3260,7 @@ def allowed_payload_path(component, rel):
            DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_REL,
            DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
            DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
+            DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
            "packages/device-protocol-contract",
            "packages/arusnavi-b2-adapter",
            "services/device-control-core",
@@ -8259,6 +8318,11 @@ def load_artifact(artifact, work_dir):
        entries,
    ):
        validate_device_plane_manager_reconciliation_payload(payload_dir)
+    if is_device_plane_manager_v2_reconciliation_slice(
+        manifest["component"],
+        entries,
+    ):
+        validate_device_plane_manager_v2_reconciliation_payload(payload_dir)
    if is_device_plane_postgres_bootstrap_slice(
        manifest["component"],
        entries,
@@ -8610,6 +8674,14 @@ def reject_terminal_device_plane_manager_artifact(
            "Device Manager control-plane 001 is terminal failed; "
            "use reconciliation followed by the exact v2 activation successor"
        )
+    if (
+        manifest.get("id") == DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID
+        or sha256 == DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
+    ):
+        die(
+            "Device Manager control-plane 003 is terminal failed; "
+            "use the exact v2 reconciliation successor"
+        )


 def reject_terminal_device_plane_backhaul_artifact(manifest, sha256):
@@ -8690,6 +8762,15 @@ def is_device_plane_manager_reconciliation_slice(component, entries):
    )


+def is_device_plane_manager_v2_reconciliation_slice(component, entries):
+    return (
+        component == "device-plane"
+        and entries is not None
+        and tuple(entries)
+        == DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES
+    )
+
+
 def expected_platform_device_core_hub_trust_descriptor():
    return {
        "schemaVersion": "nodedc.platform.device-core-hub-trust.v1",
@@ -8780,6 +8861,43 @@ def expected_device_plane_manager_reconciliation_descriptor():
    }


+def expected_device_plane_manager_v2_reconciliation_descriptor():
+    return {
+        "schemaVersion": (
+            "nodedc.device-plane.device-manager-control-plane-"
+            "v2-reconciliation.v1"
+        ),
+        "mode": "failed-v2-control-plane-baseline-adoption",
+        "failedPatchId": DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID,
+        "failedArtifactSha256": (
+            DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
+        ),
+        "backupId": DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID,
+        "failureClass": "deterministic-runtime-module-resolution",
+        "missingModule": (
+            "/packages/external-provider-contract/src/"
+            "credential-reference.mjs"
+        ),
+        "correctiveAction": (
+            "runtime-local-contract-adapter+staged-module-import-gate"
+        ),
+        "sourceAction": "publish-reconciliation-marker-only",
+        "runtimeAction": "read-only-acceptance",
+        "preservedServices": [
+            "device-control-core",
+            "device-gateway",
+            "device-postgres",
+            "device-backhaul-target",
+        ],
+        "absentService": "device-manager",
+        "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
+        "publicIngress": "disabled",
+        "commandTransport": "disabled",
+        "gelios": "untouched",
+        "rollback": "marker-only-runtime-unchanged",
+    }
+
+
 def validate_platform_device_core_hub_trust_payload(payload_dir):
    descriptor = read_strict_json(
        payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
@@ -8889,6 +9007,23 @@ def validate_device_plane_manager_reconciliation_payload(payload_dir):
    return descriptor


+def validate_device_plane_manager_v2_reconciliation_payload(payload_dir):
+    descriptor = read_strict_json(
+        payload_dir / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
+        "Device Manager v2 control-plane reconciliation descriptor",
+        max_bytes=16 * 1024,
+    )
+    if (
+        descriptor
+        != expected_device_plane_manager_v2_reconciliation_descriptor()
+    ):
+        die(
+            "Device Manager v2 control-plane reconciliation "
+            "descriptor mismatch"
+        )
+    return descriptor
+
+
 def is_device_plane_postgres_bootstrap_slice(component, entries):
    return (
        component == "device-plane"
@@ -10369,6 +10504,185 @@ def validate_device_plane_manager_activation_predecessor(payload_dir):
    }


+def validate_device_plane_manager_v2_reconciliation_backup():
+    backup_dir = (
+        BACKUPS_DIR / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID
+    )
+    try:
+        backup_stat = backup_dir.lstat()
+    except FileNotFoundError:
+        die("Device Manager v2 failed-apply backup is missing")
+    if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
+        backup_stat.st_mode
+    ):
+        die("Device Manager v2 failed-apply backup is unsafe")
+    expected = DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_SHA256
+    if {child.name for child in backup_dir.iterdir()} != set(expected):
+        die("Device Manager v2 failed-apply backup file set mismatch")
+    for name, expected_sha256 in expected.items():
+        path = backup_dir / name
+        path_stat = path.lstat()
+        if (
+            stat.S_ISLNK(path_stat.st_mode)
+            or not stat.S_ISREG(path_stat.st_mode)
+            or sha256_file(path) != expected_sha256
+        ):
+            die(
+                "Device Manager v2 failed-apply backup drift detected: "
+                f"{name}"
+            )
+    existing = tuple(read_backup_path_list(
+        backup_dir / "existing-files.txt"
+    ))
+    missing = tuple(read_backup_path_list(
+        backup_dir / "missing-files.txt"
+    ))
+    validate_backup_partition(
+        DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES,
+        existing,
+        missing,
+        "Device Manager v2 reconciliation",
+    )
+    if (
+        existing != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING
+        or missing != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING
+    ):
+        die("Device Manager v2 failed-apply backup partition mismatch")
+    return backup_dir
+
+
+def validate_device_plane_manager_v2_reconciled_baseline(
+    backup_dir,
+    *,
+    marker_installed,
+):
+    prior_backup = validate_device_plane_manager_reconciliation_backup()
+    runtime = validate_device_plane_manager_reconciled_baseline(
+        prior_backup,
+        marker_installed=True,
+    )
+    root = component_root("device-plane")
+    with tempfile.TemporaryDirectory(
+        prefix="device-manager-v2-reconciled-baseline-",
+        dir=TMP_DIR,
+    ) as directory:
+        backup_root = Path(directory)
+        materialize_backup_tree(
+            backup_dir / "source-before.tgz",
+            backup_root,
+            set(DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING),
+        )
+        backup_source = collect_exact_files(
+            backup_root,
+            DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING,
+            "Device Manager v2 pre-apply source",
+        )
+        live_source = collect_exact_files(
+            root,
+            DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING,
+            "Device Manager v2 reconciled live source",
+        )
+    if live_source != backup_source:
+        die("Device Manager v2 rollback source does not match backup")
+    for rel in DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING:
+        path = root / rel
+        if path.exists() or path.is_symlink():
+            die(
+                "Device Manager v2 candidate-only source remains installed: "
+                f"{rel}"
+            )
+
+    marker = root / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL
+    if marker_installed:
+        descriptor = read_strict_json(
+            marker,
+            "installed Device Manager v2 reconciliation descriptor",
+            max_bytes=16 * 1024,
+        )
+        if (
+            descriptor
+            != expected_device_plane_manager_v2_reconciliation_descriptor()
+        ):
+            die("installed Device Manager v2 reconciliation descriptor mismatch")
+    elif marker.exists() or marker.is_symlink():
+        die("Device Manager v2 reconciliation descriptor already installed")
+    return runtime
+
+
+def validate_device_plane_manager_v2_reconciliation_evidence(payload_dir):
+    descriptor = validate_device_plane_manager_v2_reconciliation_payload(
+        payload_dir
+    )
+    backup_dir = validate_device_plane_manager_v2_reconciliation_backup()
+    failed_artifact = FAILED_DIR / DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT
+    try:
+        failed_stat = failed_artifact.lstat()
+    except FileNotFoundError:
+        die("Device Manager v2 failed artifact is missing")
+    if (
+        stat.S_ISLNK(failed_stat.st_mode)
+        or not stat.S_ISREG(failed_stat.st_mode)
+        or sha256_file(failed_artifact)
+        != DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
+    ):
+        die("Device Manager v2 failed artifact evidence mismatch")
+
+    records = [
+        value
+        for value in load_state(FAILED_STATE_FILE)
+        if value.get("id") == DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID
+    ]
+    if len(records) != 1:
+        die("Device Manager v2 failed journal evidence count mismatch")
+    record = records[0]
+    if (
+        record.get("artifact") != DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT
+        or record.get("backup_id")
+        != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID
+        or record.get("component") != "device-plane"
+        or record.get("sha256")
+        != DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
+        or record.get("started_apply") is not True
+        or record.get("rollback_status") != "failed:DeployError"
+        or record.get("status") != "failed"
+        or record.get("message")
+        != (
+            "container healthcheck grace exhausted for "
+            "f8593303db6b71468716ed2428e54aab18d7303b34adbe1a93cd6df86effa1cf: "
+            "unhealthy"
+        )
+    ):
+        die("Device Manager v2 failed journal evidence mismatch")
+
+    with tempfile.TemporaryDirectory(
+        prefix="device-manager-v2-failed-artifact-",
+        dir=TMP_DIR,
+    ) as directory:
+        failed_manifest, failed_entries, _failed_payload = load_artifact(
+            failed_artifact,
+            Path(directory),
+        )
+    if (
+        failed_manifest.get("id") != DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID
+        or failed_manifest.get("component") != "device-plane"
+        or failed_manifest.get("type") != "app-overlay"
+        or tuple(failed_entries)
+        != DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES
+    ):
+        die("Device Manager v2 failed artifact contract mismatch")
+
+    runtime = validate_device_plane_manager_v2_reconciled_baseline(
+        backup_dir,
+        marker_installed=False,
+    )
+    return {
+        "mode": descriptor["mode"],
+        "backup": backup_dir,
+        "failedArtifact": failed_artifact,
+        "runtime": runtime,
+    }
+
+
 def device_plane_service_container_ids(service):
    if service not in (
        *DEVICE_PLANE_RUNTIME_SERVICES,
@@ -13558,6 +13872,11 @@ def component_services(component, entries=None):
        # evidence and the already-restored baseline. Runtime is read-only.
        return ()

+    if is_device_plane_manager_v2_reconciliation_slice(component, entries):
+        # The failed v2 apply already restored the prior source and later
+        # converged to the healthy baseline. Publish evidence only.
+        return ()
+
    if is_device_plane_postgres_bootstrap_slice(component, entries):
        # This exact one-time transition is the only Device Plane artifact that
        # may select durable state. Its preflight requires both container and
@@ -13976,6 +14295,9 @@ def component_builds(component, entries=None):
    if is_device_plane_manager_reconciliation_slice(component, entries):
        return ()

+    if is_device_plane_manager_v2_reconciliation_slice(component, entries):
+        return ()
+
    if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
        return ()

@@ -16000,6 +16322,7 @@ def plan_artifact(artifact):
    device_plane_b2_recovery_preflight = None
    device_plane_manager_activation_preflight = None
    device_plane_manager_reconciliation_preflight = None
+    device_plane_manager_v2_reconciliation_preflight = None
    device_plane_backhaul_preflight = None
    device_plane_backhaul_vps_enrollment_preflight = None
    device_plane_runtime_before = None
@@ -16227,6 +16550,15 @@ def plan_artifact(artifact):
                    payload_dir
                )
            )
+        if is_device_plane_manager_v2_reconciliation_slice(
+            manifest["component"],
+            entries,
+        ):
+            device_plane_manager_v2_reconciliation_preflight = (
+                validate_device_plane_manager_v2_reconciliation_evidence(
+                    payload_dir
+                )
+            )
        if is_device_plane_backhaul_target_slice(
            manifest["component"],
            entries,
@@ -17841,6 +18173,42 @@ def plan_artifact(artifact):
            print("device_gateway_tcp_9921=preserved:loopback-only")
            print("device_manager_public_route=unchanged:absent")
            print("device_plane_rollback=marker-only-runtime-unchanged")
+        if device_plane_manager_v2_reconciliation_preflight is not None:
+            print(
+                "device_plane_transition="
+                f"{device_plane_manager_v2_reconciliation_preflight['mode']}"
+            )
+            print(
+                "failed_patch="
+                f"{DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID}"
+            )
+            print(
+                "failed_artifact_sha256="
+                f"{DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256}"
+            )
+            print(
+                "recovery_backup="
+                f"{device_plane_manager_v2_reconciliation_preflight['backup'].name}"
+            )
+            print(
+                "device_plane_failure_class="
+                "deterministic-runtime-module-resolution"
+            )
+            print("device_plane_build=none")
+            print("device_plane_runtime_mutation=none")
+            print(
+                "device_plane_runtime_services="
+                "preserved:device-control-core,device-gateway,"
+                "device-postgres,device-backhaul-target"
+            )
+            print("device_manager=absent")
+            print(
+                "device_plane_source_action="
+                "publish-v2-reconciliation-marker-only"
+            )
+            print("device_gateway_tcp_9921=preserved:loopback-only")
+            print("device_manager_public_route=unchanged:absent")
+            print("device_plane_rollback=marker-only-runtime-unchanged")
        if device_plane_backhaul_vps_enrollment_preflight is not None:
            print(
                "device_plane_transition="
@@ -19356,6 +19724,8 @@ def run_component_runtime(component, entries, services):
        return
    if is_device_plane_manager_reconciliation_slice(component, entries):
        return
+    if is_device_plane_manager_v2_reconciliation_slice(component, entries):
+        return
    if is_device_plane_foundation_recovery_slice(component, entries):
        return
    if is_device_plane_foundation_network_publication_slice(
@@ -19386,6 +19756,8 @@ def run_device_plane_runtime_for_apply(
        "device-plane", entries
    ) or is_device_plane_manager_reconciliation_slice(
        "device-plane", entries
+    ) or is_device_plane_manager_v2_reconciliation_slice(
+        "device-plane", entries
    ):
        return
    # Build and runtime preparation are pre-runtime phases. A failure here
@@ -20132,6 +20504,23 @@ def run_healthchecks(component, entries=None, services=None):
        )
        return

+    if is_device_plane_manager_v2_reconciliation_slice(component, entries):
+        if tuple(services or ()) != ():
+            die("Device Manager v2 reconciliation service set mismatch")
+        for service in DEVICE_PLANE_RUNTIME_SERVICES:
+            healthcheck_compose_service_with_grace(
+                "device-plane",
+                service,
+            )
+        for check in component_healthchecks(component, entries, services):
+            healthcheck_url(check)
+        backup_dir = validate_device_plane_manager_v2_reconciliation_backup()
+        validate_device_plane_manager_v2_reconciled_baseline(
+            backup_dir,
+            marker_installed=True,
+        )
+        return
+
    if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
        if tuple(services or ()) != (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,):
            die("Device Plane VPS enrollment service set mismatch")
@@ -21112,6 +21501,13 @@ def apply_artifact(artifact):
                    validate_device_plane_manager_reconciliation_evidence(
                        payload_dir
                    )
+                if is_device_plane_manager_v2_reconciliation_slice(
+                    component,
+                    entries,
+                ):
+                    validate_device_plane_manager_v2_reconciliation_evidence(
+                        payload_dir
+                    )
                if is_device_plane_backhaul_target_slice(
                    component,
                    entries,