From 72b4846b32295b18470e052f78807f6363be67ab Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 11 Aug 2026 10:37:13 +0300 Subject: [PATCH] refactor(deploy): make device manager releases declarative --- ...v3.json => device-manager-release-v1.json} | 9 +- ...-device-manager-control-plane-artifact.mjs | 32 +- infra/deploy-runner/nodedc-deploy | 291 ++++++++++++---- ..._device_manager_control_plane_artifacts.py | 324 ++++++++++++++++-- 4 files changed, 546 insertions(+), 110 deletions(-) rename device-plane/deployment/{device-manager-control-plane-v3.json => device-manager-release-v1.json} (71%) diff --git a/device-plane/deployment/device-manager-control-plane-v3.json b/device-plane/deployment/device-manager-release-v1.json similarity index 71% rename from device-plane/deployment/device-manager-control-plane-v3.json rename to device-plane/deployment/device-manager-release-v1.json index 69fde1f..4c65141 100644 --- a/device-plane/deployment/device-manager-control-plane-v3.json +++ b/device-plane/deployment/device-manager-release-v1.json @@ -1,10 +1,11 @@ { - "schemaVersion": "nodedc.device-plane.device-manager-control-plane.v3", + "schemaVersion": "nodedc.device-plane.device-manager-release.v1", + "releaseId": "__PATCH_ID__", "action": "activate", "predecessor": { + "kind": "reconciliation", "patchId": "device-manager-control-plane-v2-reconciliation-20260811-004", - "artifactSha256": "09600bfd99717314b43936e17ffaaf6b6fca1f2fd008beddddc68480cdf3d284", - "mode": "failed-v2-control-plane-baseline-adoption" + "artifactSha256": "09600bfd99717314b43936e17ffaaf6b6fca1f2fd008beddddc68480cdf3d284" }, "service": "device-manager", "publicIngress": "reverse-proxy-only", @@ -13,5 +14,5 @@ "healthGate": "bounded-container-grace+core-contract", "commandTransport": "disabled", "gelios": "untouched", - "rollback": "restore-v2-reconciled-baseline" + "rollback": "restore-preapply-snapshot" } diff --git a/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs b/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs index 554a86c..9ae126a 100644 --- a/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs +++ b/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs @@ -12,7 +12,7 @@ const devicePlaneRoot = resolve(platformRoot, "device-plane"); const designRoot = resolve(process.env.NODEDC_DEVICE_MANAGER_SOURCE_ROOT || resolve(platformRoot, "../NODEDC_DESIGN_GUIDELINE")); const managerRoot = resolve(designRoot, "apps/device-manager"); const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts")); -const [patchId = "device-manager-control-plane-v3-20260811-005", ...extra] = process.argv.slice(2); +const [patchId = "device-manager-release-20260811-006", ...extra] = process.argv.slice(2); if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) throw new Error("usage: build-device-manager-control-plane-artifact.mjs [patch-id]"); const entries = [ @@ -26,7 +26,7 @@ const entries = [ "services/device-gateway/package.json", "services/device-edge-relay/package.json", "services/device-manager", - "deployment/device-manager-control-plane-v3.json", + "deployment/device-manager-release-v1.json", ]; const stage = await mkdtemp(join(tmpdir(), "nodedc-device-manager-control-plane-")); const payload = join(stage, "payload"); @@ -40,6 +40,17 @@ try { if (build.status !== 0) throw new Error(`device_manager_build_failed:${build.stderr || build.stdout}`); await mkdir(payload, { recursive: true }); for (const entry of entries) { + if (entry === "deployment/device-manager-release-v1.json") { + const descriptor = JSON.parse(await readFile(resolve(devicePlaneRoot, entry), "utf8")); + if (descriptor.releaseId !== "__PATCH_ID__") { + throw new Error("device_manager_release_template_id_mismatch"); + } + descriptor.releaseId = patchId; + const destination = join(payload, entry); + await mkdir(dirname(destination), { recursive: true }); + await writeFile(destination, `${JSON.stringify(descriptor, null, 2)}\n`, "utf8"); + continue; + } if (entry === "services/device-manager") { const destination = join(payload, entry); await mkdir(destination, { recursive: true }); @@ -89,16 +100,21 @@ try { if (compose.includes(forbidden)) throw new Error(`device_manager_compose_boundary_violation:${forbidden}`); } const descriptor = JSON.parse(await readFile( - join(payload, "deployment/device-manager-control-plane-v3.json"), + join(payload, "deployment/device-manager-release-v1.json"), "utf8", )); + const predecessor = descriptor.predecessor; if ( - descriptor.schemaVersion !== "nodedc.device-plane.device-manager-control-plane.v3" - || descriptor.predecessor?.patchId !== "device-manager-control-plane-v2-reconciliation-20260811-004" - || descriptor.predecessor?.artifactSha256 !== "09600bfd99717314b43936e17ffaaf6b6fca1f2fd008beddddc68480cdf3d284" - || descriptor.predecessor?.mode !== "failed-v2-control-plane-baseline-adoption" + descriptor.schemaVersion !== "nodedc.device-plane.device-manager-release.v1" + || descriptor.releaseId !== patchId + || !["activate", "upgrade"].includes(descriptor.action) + || !predecessor + || !["reconciliation", "release"].includes(predecessor.kind) + || !/^[A-Za-z0-9._-]{1,96}$/.test(predecessor.patchId || "") + || !/^[a-f0-9]{64}$/.test(predecessor.artifactSha256 || "") + || (descriptor.action === "activate") !== (predecessor.kind === "reconciliation") || descriptor.healthGate !== "bounded-container-grace+core-contract" - || descriptor.rollback !== "restore-v2-reconciled-baseline" + || descriptor.rollback !== "restore-preapply-snapshot" ) throw new Error("device_manager_activation_successor_contract_mismatch"); await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, "utf8"); await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8"); diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index b71eb3b..2772d3d 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -217,7 +217,7 @@ DEVICE_PLANE_MANAGER_V2_CONTROL_PLANE_REL = ( "deployment/device-manager-control-plane-v2.json" ) DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL = ( - "deployment/device-manager-control-plane-v3.json" + "deployment/device-manager-release-v1.json" ) DEVICE_PLANE_MANAGER_COMPOSE_REL = "docker-compose.device-manager.yml" DEVICE_PLANE_MANAGER_COMPOSE_SHA256 = ( @@ -350,17 +350,6 @@ DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 = ( DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID = ( "device-manager-control-plane-v2-reconciliation-20260811-004" ) -DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT = ( - "nodedc-device-plane-device-manager-control-plane-v2-reconciliation-" - "20260811-004.tgz" -) -DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT_SHA256 = ( - "09600bfd99717314b43936e17ffaaf6b6fca1f2fd008beddddc68480cdf3d284" -) -DEVICE_PLANE_MANAGER_V2_RECONCILIATION_APPLY_BACKUP_ID = ( - "device-plane-device-manager-control-plane-v2-reconciliation-" - "20260811-004-20260811-100533" -) DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID = ( "device-plane-device-manager-control-plane-20260811-003-" "20260811-012505" @@ -8332,7 +8321,10 @@ def load_artifact(artifact, work_dir): manifest["component"], entries, ): - validate_device_plane_manager_control_plane_payload(payload_dir) + validate_device_plane_manager_control_plane_payload( + payload_dir, + expected_release_id=manifest["id"], + ) if is_device_plane_manager_failed_control_plane_slice( manifest["component"], entries, @@ -8652,6 +8644,34 @@ def state_has_patch_id(patch_id): return any(row.get("id") == patch_id for row in load_state(STATE_FILE)) +def failed_state_has_sha(sha): + return any( + row.get("sha256") == sha + for row in load_state(FAILED_STATE_FILE) + ) + + +def failed_state_has_patch_id(patch_id): + return any( + row.get("id") == patch_id + for row in load_state(FAILED_STATE_FILE) + ) + + +def reject_failed_artifact_replay(manifest, sha256): + patch_id = manifest.get("id") + if failed_state_has_sha(sha256): + die( + "artifact SHA is terminal failed and cannot be replayed; " + "build a new immutable artifact" + ) + if patch_id and failed_state_has_patch_id(patch_id): + die( + "patch id is terminal failed and cannot be reused; " + "build a new patch id" + ) + + def reject_terminal_engine_l2_failed_artifact(manifest, sha256): if ( manifest.get("id") == ENGINE_L2_CLOSED_LOOP_FAILED_PATCH_ID @@ -8712,7 +8732,7 @@ def reject_terminal_device_plane_manager_artifact( die( "Device Manager control-plane 003 is terminal failed; " "use the exact v2 reconciliation successor followed by the " - "exact v3 activation successor" + "declarative Device Manager release successor" ) @@ -8833,19 +8853,8 @@ def expected_platform_device_manager_public_route_descriptor(): } -def expected_device_plane_manager_control_plane_descriptor(): +def expected_device_plane_manager_release_boundaries(): return { - "schemaVersion": ( - "nodedc.device-plane.device-manager-control-plane.v3" - ), - "action": "activate", - "predecessor": { - "patchId": DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID, - "artifactSha256": ( - DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT_SHA256 - ), - "mode": "failed-v2-control-plane-baseline-adoption", - }, "service": "device-manager", "publicIngress": "reverse-proxy-only", "deviceCoreManagementApi": "file-token-authenticated", @@ -8853,7 +8862,7 @@ def expected_device_plane_manager_control_plane_descriptor(): "healthGate": "bounded-container-grace+core-contract", "commandTransport": "disabled", "gelios": "untouched", - "rollback": "restore-v2-reconciled-baseline", + "rollback": "restore-preapply-snapshot", } @@ -8982,14 +8991,62 @@ def validate_platform_device_manager_public_route_payload(payload_dir): return descriptor -def validate_device_plane_manager_control_plane_payload(payload_dir): +def validate_device_plane_manager_control_plane_payload( + payload_dir, + *, + expected_release_id=None, +): descriptor = read_strict_json( payload_dir / DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL, - "Device Manager control-plane descriptor", + "Device Manager release descriptor", max_bytes=16 * 1024, ) - if descriptor != expected_device_plane_manager_control_plane_descriptor(): - die("Device Manager control-plane descriptor mismatch") + required_keys = { + "schemaVersion", + "releaseId", + "action", + "predecessor", + *expected_device_plane_manager_release_boundaries(), + } + if set(descriptor) != required_keys: + die("Device Manager release descriptor key set mismatch") + if ( + descriptor.get("schemaVersion") + != "nodedc.device-plane.device-manager-release.v1" + ): + die("Device Manager release descriptor schema mismatch") + release_id = descriptor.get("releaseId") + if ( + not isinstance(release_id, str) + or not re.fullmatch(r"[A-Za-z0-9._-]{1,96}", release_id) + or (expected_release_id is not None and release_id != expected_release_id) + ): + die("Device Manager release id mismatch") + action = descriptor.get("action") + predecessor = descriptor.get("predecessor") + if ( + action not in ("activate", "upgrade") + or not isinstance(predecessor, dict) + or set(predecessor) != {"kind", "patchId", "artifactSha256"} + or predecessor.get("kind") not in ("reconciliation", "release") + or not isinstance(predecessor.get("patchId"), str) + or not re.fullmatch( + r"[A-Za-z0-9._-]{1,96}", + predecessor["patchId"], + ) + or predecessor["patchId"] == release_id + or not isinstance(predecessor.get("artifactSha256"), str) + or not re.fullmatch( + r"[a-f0-9]{64}", + predecessor["artifactSha256"], + ) + or (action == "activate") + != (predecessor["kind"] == "reconciliation") + ): + die("Device Manager release predecessor mismatch") + boundaries = expected_device_plane_manager_release_boundaries() + if any(descriptor.get(key) != value for key, value in boundaries.items()): + die("Device Manager release security boundary mismatch") compose_path = payload_dir / DEVICE_PLANE_MANAGER_COMPOSE_REL if sha256_file(compose_path) != DEVICE_PLANE_MANAGER_COMPOSE_SHA256: die("Device Manager control-plane Compose mismatch") @@ -10494,52 +10551,136 @@ def validate_device_plane_manager_activation_predecessor(payload_dir): descriptor = validate_device_plane_manager_control_plane_payload( payload_dir ) - reconciliation_artifact = ( - APPLIED_DIR / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT - ) + predecessor = descriptor["predecessor"] + patch_id = predecessor["patchId"] + sha256 = predecessor["artifactSha256"] + artifact_name = f"nodedc-device-plane-{patch_id}.tgz" + artifact = APPLIED_DIR / artifact_name try: - artifact_stat = reconciliation_artifact.lstat() + artifact_stat = artifact.lstat() except FileNotFoundError: - die("Device Manager reconciliation applied artifact is missing") + die("Device Manager predecessor applied artifact is missing") if ( stat.S_ISLNK(artifact_stat.st_mode) or not stat.S_ISREG(artifact_stat.st_mode) - or sha256_file(reconciliation_artifact) - != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT_SHA256 + or sha256_file(artifact) != sha256 ): - die("Device Manager reconciliation applied artifact mismatch") + die("Device Manager predecessor applied artifact mismatch") - records = [ + id_records = [ value for value in load_state(STATE_FILE) - if value.get("id") - == DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID + if value.get("id") == patch_id + ] + sha_records = [ + value + for value in load_state(STATE_FILE) + if value.get("sha256") == sha256 ] - if len(records) != 1: - die("Device Manager reconciliation applied journal count mismatch") - record = records[0] if ( - record.get("artifact") - != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT - or record.get("backup_id") - != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_APPLY_BACKUP_ID - or record.get("component") != "device-plane" - or record.get("sha256") - != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT_SHA256 - or record.get("status") != "ok" + len(id_records) != 1 + or len(sha_records) != 1 + or id_records[0] != sha_records[0] ): - die("Device Manager reconciliation applied journal mismatch") + die("Device Manager predecessor applied journal identity mismatch") + record = id_records[0] + backup_id = record.get("backup_id") + if ( + record.get("artifact") != artifact_name + or record.get("component") != "device-plane" + or record.get("id") != patch_id + or record.get("sha256") != sha256 + or record.get("status") != "ok" + or not isinstance(backup_id, str) + or not backup_id + or safe_name(backup_id) != backup_id + ): + die("Device Manager predecessor applied journal mismatch") + backup_dir = BACKUPS_DIR / backup_id + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Manager predecessor backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Manager predecessor backup is unsafe") - backup_dir = validate_device_plane_manager_v2_reconciliation_backup() - runtime = validate_device_plane_manager_v2_reconciled_baseline( - backup_dir, - marker_installed=True, - ) + with tempfile.TemporaryDirectory( + prefix="device-manager-applied-predecessor-", + dir=TMP_DIR, + ) as directory: + predecessor_manifest, predecessor_entries, predecessor_payload = ( + load_artifact(artifact, Path(directory)) + ) + if ( + predecessor_manifest.get("id") != patch_id + or predecessor_manifest.get("component") != "device-plane" + or predecessor_manifest.get("type") != "app-overlay" + ): + die("Device Manager predecessor artifact manifest mismatch") + root = component_root("device-plane") + if predecessor["kind"] == "reconciliation": + if ( + descriptor["action"] != "activate" + or tuple(predecessor_entries) + != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES + ): + die("Device Manager reconciliation predecessor type mismatch") + predecessor_descriptor = ( + validate_device_plane_manager_v2_reconciliation_payload( + predecessor_payload + ) + ) + installed_descriptor = read_strict_json( + root / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, + "installed Device Manager reconciliation predecessor", + max_bytes=16 * 1024, + ) + if installed_descriptor != predecessor_descriptor: + die("Device Manager reconciliation predecessor is not current") + baseline_backup = ( + validate_device_plane_manager_v2_reconciliation_backup() + ) + runtime = validate_device_plane_manager_v2_reconciled_baseline( + baseline_backup, + marker_installed=True, + ) + mode = "reconciled-manager-forward-activation" + else: + if ( + descriptor["action"] != "upgrade" + or tuple(predecessor_entries) + != DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES + ): + die("Device Manager release predecessor type mismatch") + predecessor_descriptor = ( + validate_device_plane_manager_control_plane_payload( + predecessor_payload, + expected_release_id=patch_id, + ) + ) + installed_descriptor = read_strict_json( + root / DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL, + "installed Device Manager release predecessor", + max_bytes=16 * 1024, + ) + if installed_descriptor != predecessor_descriptor: + die("Device Manager release predecessor is not current") + for service in ( + "device-control-core", + "device-manager", + "device-postgres", + ): + healthcheck_compose_service("device-plane", service) + runtime = {"accepted": True} + mode = "active-manager-forward-upgrade" return { - "mode": "reconciled-v2-manager-forward-activation", + "mode": mode, "descriptor": descriptor, - "reconciliationArtifact": reconciliation_artifact, - "reconciliationRecord": record, + "predecessorArtifact": artifact, + "predecessorRecord": record, + "predecessorBackup": backup_dir, "runtime": runtime, } @@ -10828,7 +10969,10 @@ def device_plane_inventory_service_names(inventory): "health", "restartCount", } - or item.get("service") not in DEVICE_PLANE_RUNTIME_SERVICES + or item.get("service") not in ( + *DEVICE_PLANE_RUNTIME_SERVICES, + "device-manager", + ) or not isinstance(item.get("containerId"), str) or not re.fullmatch(r"[a-f0-9]{64}", item["containerId"]) or not isinstance(item.get("imageId"), str) @@ -16391,6 +16535,7 @@ def plan_artifact(artifact): device_plane_foundation_recovery_preflight = None with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) + reject_failed_artifact_replay(manifest, sha) reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_device_plane_foundation_artifact(manifest, sha) reject_terminal_device_plane_manager_artifact( @@ -18162,11 +18307,15 @@ def plan_artifact(artifact): ) print( "device_plane_predecessor_patch=" - f"{DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID}" + f"{device_plane_manager_activation_preflight['descriptor']['predecessor']['patchId']}" ) print( "device_plane_predecessor_artifact_sha256=" - f"{DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT_SHA256}" + f"{device_plane_manager_activation_preflight['descriptor']['predecessor']['artifactSha256']}" + ) + print( + "device_plane_predecessor_kind=" + f"{device_plane_manager_activation_preflight['descriptor']['predecessor']['kind']}" ) print( "device_plane_runtime_mutation=" @@ -21469,6 +21618,7 @@ def apply_artifact(artifact): with tempfile.TemporaryDirectory(prefix=f"apply-{current_stamp}-", dir=TMP_DIR) as tmp: work = Path(tmp) manifest, entries, payload_dir = load_artifact(artifact, work) + reject_failed_artifact_replay(manifest, sha) reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_device_plane_foundation_artifact( manifest, @@ -21848,9 +21998,18 @@ def apply_artifact(artifact): if not artifact_only and not DOCKER.is_file(): die(f"docker not found: {DOCKER}") if component == "device-plane": + inventory_services = DEVICE_PLANE_RUNTIME_SERVICES + if is_device_plane_manager_control_plane_slice( + component, + entries, + ): + inventory_services = ( + *inventory_services, + "device-manager", + ) device_plane_runtime_before = ( device_plane_runtime_inventory( - DEVICE_PLANE_RUNTIME_SERVICES + inventory_services ) ) diff --git a/infra/deploy-runner/test_device_manager_control_plane_artifacts.py b/infra/deploy-runner/test_device_manager_control_plane_artifacts.py index 0ae766d..6fc348e 100644 --- a/infra/deploy-runner/test_device_manager_control_plane_artifacts.py +++ b/infra/deploy-runner/test_device_manager_control_plane_artifacts.py @@ -36,7 +36,14 @@ LAUNCHER_HUB_SERVICE_TRUST_UI_ENTRIES = ( ) -def healthy_device_plane_inventory(): +def healthy_device_plane_inventory(*, include_manager=False): + services = [ + ("device-control-core", "a"), + ("device-gateway", "b"), + ("device-postgres", "c"), + ] + if include_manager: + services.append(("device-manager", "d")) return { "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", "composeProject": "nodedc-device-plane", @@ -50,15 +57,32 @@ def healthy_device_plane_inventory(): "health": "healthy", "restartCount": 0, } - for service, character in ( - ("device-control-core", "a"), - ("device-gateway", "b"), - ("device-postgres", "c"), - ) + for service, character in services ], } +def device_manager_release_descriptor( + release_id="device-manager-release-unit-001", + *, + action="activate", + predecessor_kind="reconciliation", + predecessor_patch="device-manager-reconciliation-unit-001", + predecessor_sha="a" * 64, +): + return { + "schemaVersion": "nodedc.device-plane.device-manager-release.v1", + "releaseId": release_id, + "action": action, + "predecessor": { + "kind": predecessor_kind, + "patchId": predecessor_patch, + "artifactSha256": predecessor_sha, + }, + **RUNNER.expected_device_plane_manager_release_boundaries(), + } + + class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): def build(self, script, patch_id, artifact_dir): environment = os.environ.copy() @@ -239,55 +263,87 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): ) runtime_acceptance.assert_called_once_with() - def test_v3_activation_requires_exact_applied_v2_reconciliation(self): + def test_activation_resolves_predecessor_from_descriptor_and_journal(self): with tempfile.TemporaryDirectory( - prefix="nodedc-device-manager-v3-predecessor-", + prefix="nodedc-device-manager-release-predecessor-", ) as directory: root = Path(directory) - artifact = ( - root - / RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT - ) + applied = root / "applied" + backups = root / "backups" + temporary = root / "tmp" + device_plane = root / "device-plane" + for path in (applied, backups, temporary, device_plane / "deployment"): + path.mkdir(parents=True) + predecessor_patch = "device-manager-reconciliation-unit-004" + artifact_name = f"nodedc-device-plane-{predecessor_patch}.tgz" + artifact = applied / artifact_name artifact_bytes = b"reviewed-reconciliation-artifact" artifact.write_bytes(artifact_bytes) + artifact_sha = hashlib.sha256(artifact_bytes).hexdigest() + descriptor = device_manager_release_descriptor( + predecessor_patch=predecessor_patch, + predecessor_sha=artifact_sha, + ) state_file = root / "applied.jsonl" state_file.write_text( json.dumps({ - "id": ( - RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID - ), - "artifact": ( - RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT - ), - "backup_id": ( - RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_APPLY_BACKUP_ID - ), + "id": predecessor_patch, + "artifact": artifact_name, + "backup_id": "predecessor-backup", "component": "device-plane", - "sha256": hashlib.sha256(artifact_bytes).hexdigest(), + "sha256": artifact_sha, "status": "ok", }) + "\n", encoding="utf-8", ) - backup = root / "failed-backup" + (backups / "predecessor-backup").mkdir() + baseline_backup = root / "failed-backup" + predecessor_descriptor = {"marker": "exact"} + marker = ( + device_plane + / RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL + ) + marker.write_text( + json.dumps(predecessor_descriptor), + encoding="utf-8", + ) with ( - mock.patch.object(RUNNER, "APPLIED_DIR", root), + mock.patch.object(RUNNER, "APPLIED_DIR", applied), + mock.patch.object(RUNNER, "BACKUPS_DIR", backups), + mock.patch.object(RUNNER, "TMP_DIR", temporary), mock.patch.object(RUNNER, "STATE_FILE", state_file), mock.patch.object( RUNNER, - "DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ARTIFACT_SHA256", - hashlib.sha256(artifact_bytes).hexdigest(), + "component_root", + return_value=device_plane, ), mock.patch.object( RUNNER, "validate_device_plane_manager_control_plane_payload", - return_value=( - RUNNER.expected_device_plane_manager_control_plane_descriptor() - ), + return_value=descriptor, ) as payload, + mock.patch.object( + RUNNER, + "load_artifact", + return_value=( + { + "id": predecessor_patch, + "component": "device-plane", + "type": "app-overlay", + }, + RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES, + root / "predecessor-payload", + ), + ), + mock.patch.object( + RUNNER, + "validate_device_plane_manager_v2_reconciliation_payload", + return_value=predecessor_descriptor, + ), mock.patch.object( RUNNER, "validate_device_plane_manager_v2_reconciliation_backup", - return_value=backup, + return_value=baseline_backup, ), mock.patch.object( RUNNER, @@ -303,10 +359,147 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): self.assertEqual( result["mode"], - "reconciled-v2-manager-forward-activation", + "reconciled-manager-forward-activation", ) payload.assert_called_once_with(root / "payload") - baseline.assert_called_once_with(backup, marker_installed=True) + baseline.assert_called_once_with( + baseline_backup, + marker_installed=True, + ) + + def test_failed_patch_id_and_sha_are_globally_non_replayable(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-manager-failed-replay-", + ) as directory: + failed_state = Path(directory) / "failed.jsonl" + failed_state.write_text( + json.dumps({ + "id": "device-manager-release-failed-001", + "sha256": "b" * 64, + "status": "failed", + }) + "\n", + encoding="utf-8", + ) + with mock.patch.object( + RUNNER, + "FAILED_STATE_FILE", + failed_state, + ): + with self.assertRaisesRegex( + RUNNER.DeployError, + "SHA is terminal failed", + ): + RUNNER.reject_failed_artifact_replay( + {"id": "different"}, + "b" * 64, + ) + with self.assertRaisesRegex( + RUNNER.DeployError, + "patch id is terminal failed", + ): + RUNNER.reject_failed_artifact_replay( + {"id": "device-manager-release-failed-001"}, + "c" * 64, + ) + + def test_upgrade_accepts_any_current_release_without_runner_patch_ids(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-manager-release-upgrade-", + ) as directory: + root = Path(directory) + applied = root / "applied" + backups = root / "backups" + temporary = root / "tmp" + device_plane = root / "device-plane" + for path in (applied, backups, temporary, device_plane / "deployment"): + path.mkdir(parents=True) + predecessor_patch = "device-manager-release-arbitrary-041" + artifact_name = f"nodedc-device-plane-{predecessor_patch}.tgz" + artifact = applied / artifact_name + artifact_bytes = b"arbitrary-reviewed-device-manager-release" + artifact.write_bytes(artifact_bytes) + artifact_sha = hashlib.sha256(artifact_bytes).hexdigest() + predecessor_descriptor = device_manager_release_descriptor( + release_id=predecessor_patch, + ) + candidate_descriptor = device_manager_release_descriptor( + release_id="device-manager-release-arbitrary-042", + action="upgrade", + predecessor_kind="release", + predecessor_patch=predecessor_patch, + predecessor_sha=artifact_sha, + ) + installed = ( + device_plane / RUNNER.DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL + ) + installed.write_text( + json.dumps(predecessor_descriptor), + encoding="utf-8", + ) + state_file = root / "applied.jsonl" + state_file.write_text( + json.dumps({ + "id": predecessor_patch, + "artifact": artifact_name, + "backup_id": "arbitrary-release-backup", + "component": "device-plane", + "sha256": artifact_sha, + "status": "ok", + }) + "\n", + encoding="utf-8", + ) + (backups / "arbitrary-release-backup").mkdir() + with ( + mock.patch.object(RUNNER, "APPLIED_DIR", applied), + mock.patch.object(RUNNER, "BACKUPS_DIR", backups), + mock.patch.object(RUNNER, "TMP_DIR", temporary), + mock.patch.object(RUNNER, "STATE_FILE", state_file), + mock.patch.object( + RUNNER, + "component_root", + return_value=device_plane, + ), + mock.patch.object( + RUNNER, + "validate_device_plane_manager_control_plane_payload", + side_effect=( + candidate_descriptor, + predecessor_descriptor, + ), + ), + mock.patch.object( + RUNNER, + "load_artifact", + return_value=( + { + "id": predecessor_patch, + "component": "device-plane", + "type": "app-overlay", + }, + RUNNER.DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES, + root / "predecessor-payload", + ), + ), + mock.patch.object( + RUNNER, + "healthcheck_compose_service", + ) as health, + ): + result = ( + RUNNER.validate_device_plane_manager_activation_predecessor( + root / "candidate-payload" + ) + ) + + self.assertEqual(result["mode"], "active-manager-forward-upgrade") + self.assertEqual( + [call.args for call in health.call_args_list], + [ + ("device-plane", "device-control-core"), + ("device-plane", "device-manager"), + ("device-plane", "device-postgres"), + ], + ) def test_initial_install_rollback_removes_manager_and_restores_core_only(self): entries = RUNNER.DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES @@ -393,6 +586,73 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): f"source+runtime-restored:{len(entries)}", ) + def test_upgrade_rollback_restores_previous_core_and_manager_release(self): + entries = RUNNER.DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES + with tempfile.TemporaryDirectory( + prefix="nodedc-device-manager-upgrade-rollback-", + ) as directory: + backup = Path(directory) / "backup" + backup.mkdir() + (backup / "existing-files.txt").write_text( + "\n".join(entries) + "\n", + encoding="utf-8", + ) + (backup / "missing-files.txt").write_text("", encoding="utf-8") + (backup / "runtime-before.json").write_text( + json.dumps(healthy_device_plane_inventory(include_manager=True)), + encoding="utf-8", + ) + with ( + mock.patch.object( + RUNNER, + "stop_and_remove_compose_services", + ) as stop, + mock.patch.object( + RUNNER, + "restore_platform_overlay", + return_value=len(entries), + ), + mock.patch.object( + RUNNER, + "run_component_runtime", + ) as restore_runtime, + mock.patch.object( + RUNNER, + "healthcheck_compose_service_with_grace", + ) as restore_health, + mock.patch.object( + RUNNER, + "component_healthchecks", + return_value=(), + ), + ): + result = RUNNER.rollback_device_plane_apply( + Path(directory) / "live", + backup, + entries, + "test-stamp", + True, + ("device-control-core", "device-manager"), + ) + + stop.assert_not_called() + restore_runtime.assert_called_once_with( + "device-plane", + list(entries), + ("device-control-core", "device-manager"), + ) + self.assertEqual( + [call.args for call in restore_health.call_args_list], + [ + ("device-plane", "device-control-core"), + ("device-plane", "device-manager"), + ], + ) + self.assertEqual( + result, + f"source+runtime-restored:{len(entries)}", + ) + if __name__ == "__main__": unittest.main(verbosity=2)