From 6dad30c565eb2cf17354a541725d80aa3be35a5f Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Sat, 22 Aug 2026 19:49:36 +0300 Subject: [PATCH] fix(deploy): add exact-image Core v3 recovery --- ...rol-core-release-v3-reconciliation-v1.json | 25 ++ ...ore-release-v3-reconciliation-artifact.mjs | 114 ++++++++ ...control_core_v3_reconciliation_artifact.py | 275 ++++++++++++++++++ 3 files changed, 414 insertions(+) create mode 100644 deployment/device-control-core-release-v3-reconciliation-v1.json create mode 100644 infra/deploy-runner/build-device-control-core-release-v3-reconciliation-artifact.mjs create mode 100644 infra/deploy-runner/test_device_control_core_v3_reconciliation_artifact.py diff --git a/deployment/device-control-core-release-v3-reconciliation-v1.json b/deployment/device-control-core-release-v3-reconciliation-v1.json new file mode 100644 index 0000000..3368a5c --- /dev/null +++ b/deployment/device-control-core-release-v3-reconciliation-v1.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": "nodedc.device-plane.device-control-core-release-v3-reconciliation.v1", + "mode": "failed-release-v3-exact-preapply-image-restore", + "failedPatchId": "device-control-core-release-v3-20260822-040", + "failedArtifactSha256": "08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92", + "failedArtifact": "nodedc-device-plane-device-control-core-release-v3-20260822-040.tgz.20260822-184245", + "backupId": "device-plane-device-control-core-release-v3-20260822-040-20260822-184245", + "predecessorPatchId": "device-control-core-release-v2-20260822-038", + "predecessorArtifactSha256": "e2d062b82b022dba662522b5d6e192026ac964d78950d903295ca3cbbc95ab28", + "preapplyImageId": "sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a", + "sourceAction": "accept-byte-exact-restored-preapply-source", + "runtimeAction": "retag-exact-preapply-image+recreate-device-control-core-only", + "preservedServices": [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target" + ], + "databaseVolume": "nodedc-device-plane-postgres-data", + "publicIngress": "disabled", + "edgeChannel": "core-initiated-pinned-mtls-registered-edges-only", + "commandTransport": "typed-service-ping-v1", + "gelios": "untouched-legacy-only", + "rollback": "marker+exact-preapply-image-runtime" +} diff --git a/infra/deploy-runner/build-device-control-core-release-v3-reconciliation-artifact.mjs b/infra/deploy-runner/build-device-control-core-release-v3-reconciliation-artifact.mjs new file mode 100644 index 0000000..9e98a3e --- /dev/null +++ b/infra/deploy-runner/build-device-control-core-release-v3-reconciliation-artifact.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const sourceRoot = resolve(scriptDir, "../.."); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-control-core-release-v3-reconciliation-20260822-042", + ...extra +] = process.argv.slice(2); +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-control-core-release-v3-reconciliation-artifact.mjs [patch-id]", + ); +} + +const entry = "deployment/device-control-core-release-v3-reconciliation-v1.json"; +const stage = await mkdtemp(join(tmpdir(), "nodedc-control-core-v3-reconciliation-")); +const payload = join(stage, "payload"); +const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`); + +try { + const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entry), "utf8")); + const expected = { + schemaVersion: "nodedc.device-plane.device-control-core-release-v3-reconciliation.v1", + mode: "failed-release-v3-exact-preapply-image-restore", + failedPatchId: "device-control-core-release-v3-20260822-040", + failedArtifactSha256: + "08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92", + failedArtifact: + "nodedc-device-plane-device-control-core-release-v3-20260822-040.tgz.20260822-184245", + backupId: + "device-plane-device-control-core-release-v3-20260822-040-20260822-184245", + predecessorPatchId: "device-control-core-release-v2-20260822-038", + predecessorArtifactSha256: + "e2d062b82b022dba662522b5d6e192026ac964d78950d903295ca3cbbc95ab28", + preapplyImageId: + "sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a", + sourceAction: "accept-byte-exact-restored-preapply-source", + runtimeAction: "retag-exact-preapply-image+recreate-device-control-core-only", + preservedServices: [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + databaseVolume: "nodedc-device-plane-postgres-data", + publicIngress: "disabled", + edgeChannel: "core-initiated-pinned-mtls-registered-edges-only", + commandTransport: "typed-service-ping-v1", + gelios: "untouched-legacy-only", + rollback: "marker+exact-preapply-image-runtime", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error("device_control_core_v3_reconciliation_descriptor_mismatch"); + } + + await mkdir(dirname(join(payload, entry)), { recursive: true }); + await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true }); + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${entry}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, + ); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + const sha256 = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + component: "device-plane", + artifact: target, + sha256, + entries: [entry], + build: [], + services: ["device-control-core"], + transition: descriptor.mode, + runtimeAction: descriptor.runtimeAction, + preservedServices: descriptor.preservedServices, + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} diff --git a/infra/deploy-runner/test_device_control_core_v3_reconciliation_artifact.py b/infra/deploy-runner/test_device_control_core_v3_reconciliation_artifact.py new file mode 100644 index 0000000..908f35e --- /dev/null +++ b/infra/deploy-runner/test_device_control_core_v3_reconciliation_artifact.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +RUNNER_PATH = Path( + os.environ.get( + "NODEDC_DEPLOY_RUNNER", + SCRIPT_DIR.parents[2] / "platform/infra/deploy-runner/nodedc-deploy", + ) +) +BUILDER = ( + SCRIPT_DIR + / "build-device-control-core-release-v3-reconciliation-artifact.mjs" +) + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_control_core_v3_reconciliation_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class DeviceControlCoreV3ReconciliationArtifactTest(unittest.TestCase): + def build(self, artifact_dir, patch_id): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + ["node", str(BUILDER), patch_id], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_artifact_is_exact_deterministic_runtime_reconciliation(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-control-core-v3-reconciliation-artifact-", + ) as directory: + artifact_dir = Path(directory) + patch_id = "device-control-core-v3-reconciliation-unit-042" + first = self.build(artifact_dir, patch_id) + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build(artifact_dir, patch_id) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + self.assertEqual(first["component"], "device-plane") + self.assertEqual( + first["entries"], + list(RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES), + ) + self.assertEqual(first["build"], []) + self.assertEqual(first["services"], ["device-control-core"]) + + with tarfile.open(first["artifact"], "r:gz") as archive: + names = [member.name for member in archive.getmembers()] + descriptor = json.loads( + archive.extractfile( + "payload/" + + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL + ).read().decode("utf-8") + ) + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_control_core_v3_reconciliation_descriptor(), + ) + self.assertEqual( + names, + [ + "manifest.env", + "files.txt", + "payload", + "payload/deployment", + "payload/" + + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, + ], + ) + self.assertEqual( + RUNNER.component_services( + "device-plane", + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES, + ), + ("device-control-core",), + ) + self.assertEqual( + RUNNER.component_builds( + "device-plane", + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES, + ), + (), + ) + + def test_descriptor_pins_exact_failed_evidence_and_preapply_image(self): + descriptor = ( + RUNNER.expected_device_plane_control_core_v3_reconciliation_descriptor() + ) + self.assertEqual( + descriptor["failedArtifactSha256"], + "08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92", + ) + self.assertEqual( + descriptor["preapplyImageId"], + "sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a", + ) + self.assertEqual( + descriptor["runtimeAction"], + "retag-exact-preapply-image+recreate-device-control-core-only", + ) + + def test_runtime_recovery_retags_exact_image_without_build(self): + completed = mock.Mock(returncode=0, stdout="", stderr="") + with mock.patch.object( + RUNNER, + "inspect_optional_local_image", + side_effect=[ + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + ], + ), mock.patch.object( + RUNNER.subprocess, + "run", + return_value=completed, + ) as run: + RUNNER.restore_device_plane_control_core_v3_preapply_image() + + run.assert_called_once_with( + [ + str(RUNNER.DOCKER), + "image", + "tag", + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + RUNNER.DEVICE_PLANE_CONTROL_CORE_IMAGE, + ], + check=False, + capture_output=True, + text=True, + ) + + def test_reconciliation_runtime_selects_only_core_and_no_build(self): + mark_runtime_started = mock.Mock() + with mock.patch.object( + RUNNER, + "prepare_component_runtime", + ) as prepare, mock.patch.object( + RUNNER, + "restore_device_plane_control_core_v3_preapply_image", + ) as restore, mock.patch.object( + RUNNER, + "run_compose", + ) as compose, mock.patch.object( + RUNNER, + "run_build", + ) as build: + RUNNER.run_device_plane_runtime_for_apply( + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES, + ("device-control-core",), + mark_runtime_started, + ) + + prepare.assert_called_once_with( + "device-plane", + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES, + ) + mark_runtime_started.assert_called_once_with() + restore.assert_called_once_with() + compose.assert_called_once_with( + "device-plane", + ("device-control-core",), + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES, + ) + build.assert_not_called() + + def test_release_rollback_reuses_exact_preapply_image_without_rebuild(self): + runtime_inventory = { + "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [{ + "service": "device-control-core", + "containerId": "1" * 64, + "imageId": RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + }], + } + entries = RUNNER.DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES + with tempfile.TemporaryDirectory( + prefix="nodedc-control-core-v3-rollback-unit-", + ) as directory, mock.patch.object( + RUNNER, + "read_backup_path_list", + side_effect=[list(entries), []], + ), mock.patch.object( + RUNNER, + "validate_backup_partition", + return_value=(set(entries), set()), + ), mock.patch.object( + RUNNER, + "read_strict_json", + return_value=runtime_inventory, + ), mock.patch.object( + RUNNER, + "restore_platform_overlay", + return_value=len(entries), + ), mock.patch.object( + RUNNER, + "retag_device_plane_control_core_image", + ) as retag, mock.patch.object( + RUNNER, + "prepare_component_runtime", + ), mock.patch.object( + RUNNER, + "run_compose", + ) as compose, mock.patch.object( + RUNNER, + "run_component_runtime", + ) as build_runtime, mock.patch.object( + RUNNER, + "accept_device_plane_control_core_rollback_runtime", + ), mock.patch.object( + RUNNER, + "validate_device_manager_control_plane_runtime", + ): + result = RUNNER.rollback_device_plane_apply( + Path(directory), + Path(directory), + entries, + "unit-stamp", + True, + ("device-control-core",), + ) + + self.assertEqual( + result, + f"source+runtime-restored:{len(entries)}", + ) + retag.assert_called_once_with( + RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "Device Control Core exact pre-apply rollback image", + ) + compose.assert_called_once_with( + "device-plane", + ("device-control-core",), + list(entries), + ) + build_runtime.assert_not_called() + + +if __name__ == "__main__": + unittest.main()