fix(deploy): add exact-image Core v3 recovery
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"schemaVersion": "nodedc.device-plane.device-control-core-release-v3-reconciliation.v1",
|
||||||
|
"mode": "failed-release-v3-exact-preapply-image-restore",
|
||||||
|
"failedPatchId": "device-control-core-release-v3-20260822-040",
|
||||||
|
"failedArtifactSha256": "08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92",
|
||||||
|
"failedArtifact": "nodedc-device-plane-device-control-core-release-v3-20260822-040.tgz.20260822-184245",
|
||||||
|
"backupId": "device-plane-device-control-core-release-v3-20260822-040-20260822-184245",
|
||||||
|
"predecessorPatchId": "device-control-core-release-v2-20260822-038",
|
||||||
|
"predecessorArtifactSha256": "e2d062b82b022dba662522b5d6e192026ac964d78950d903295ca3cbbc95ab28",
|
||||||
|
"preapplyImageId": "sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a",
|
||||||
|
"sourceAction": "accept-byte-exact-restored-preapply-source",
|
||||||
|
"runtimeAction": "retag-exact-preapply-image+recreate-device-control-core-only",
|
||||||
|
"preservedServices": [
|
||||||
|
"device-manager",
|
||||||
|
"device-gateway",
|
||||||
|
"device-postgres",
|
||||||
|
"device-backhaul-target"
|
||||||
|
],
|
||||||
|
"databaseVolume": "nodedc-device-plane-postgres-data",
|
||||||
|
"publicIngress": "disabled",
|
||||||
|
"edgeChannel": "core-initiated-pinned-mtls-registered-edges-only",
|
||||||
|
"commandTransport": "typed-service-ping-v1",
|
||||||
|
"gelios": "untouched-legacy-only",
|
||||||
|
"rollback": "marker+exact-preapply-image-runtime"
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { dirname, join, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const sourceRoot = resolve(scriptDir, "../..");
|
||||||
|
const artifactDir = resolve(
|
||||||
|
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|
||||||
|
|| resolve(scriptDir, "../deploy-artifacts"),
|
||||||
|
);
|
||||||
|
const [
|
||||||
|
patchId = "device-control-core-release-v3-reconciliation-20260822-042",
|
||||||
|
...extra
|
||||||
|
] = process.argv.slice(2);
|
||||||
|
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
|
||||||
|
throw new Error(
|
||||||
|
"usage: build-device-control-core-release-v3-reconciliation-artifact.mjs [patch-id]",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry = "deployment/device-control-core-release-v3-reconciliation-v1.json";
|
||||||
|
const stage = await mkdtemp(join(tmpdir(), "nodedc-control-core-v3-reconciliation-"));
|
||||||
|
const payload = join(stage, "payload");
|
||||||
|
const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entry), "utf8"));
|
||||||
|
const expected = {
|
||||||
|
schemaVersion: "nodedc.device-plane.device-control-core-release-v3-reconciliation.v1",
|
||||||
|
mode: "failed-release-v3-exact-preapply-image-restore",
|
||||||
|
failedPatchId: "device-control-core-release-v3-20260822-040",
|
||||||
|
failedArtifactSha256:
|
||||||
|
"08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92",
|
||||||
|
failedArtifact:
|
||||||
|
"nodedc-device-plane-device-control-core-release-v3-20260822-040.tgz.20260822-184245",
|
||||||
|
backupId:
|
||||||
|
"device-plane-device-control-core-release-v3-20260822-040-20260822-184245",
|
||||||
|
predecessorPatchId: "device-control-core-release-v2-20260822-038",
|
||||||
|
predecessorArtifactSha256:
|
||||||
|
"e2d062b82b022dba662522b5d6e192026ac964d78950d903295ca3cbbc95ab28",
|
||||||
|
preapplyImageId:
|
||||||
|
"sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a",
|
||||||
|
sourceAction: "accept-byte-exact-restored-preapply-source",
|
||||||
|
runtimeAction: "retag-exact-preapply-image+recreate-device-control-core-only",
|
||||||
|
preservedServices: [
|
||||||
|
"device-manager",
|
||||||
|
"device-gateway",
|
||||||
|
"device-postgres",
|
||||||
|
"device-backhaul-target",
|
||||||
|
],
|
||||||
|
databaseVolume: "nodedc-device-plane-postgres-data",
|
||||||
|
publicIngress: "disabled",
|
||||||
|
edgeChannel: "core-initiated-pinned-mtls-registered-edges-only",
|
||||||
|
commandTransport: "typed-service-ping-v1",
|
||||||
|
gelios: "untouched-legacy-only",
|
||||||
|
rollback: "marker+exact-preapply-image-runtime",
|
||||||
|
};
|
||||||
|
if (JSON.stringify(descriptor) !== JSON.stringify(expected)) {
|
||||||
|
throw new Error("device_control_core_v3_reconciliation_descriptor_mismatch");
|
||||||
|
}
|
||||||
|
|
||||||
|
await mkdir(dirname(join(payload, entry)), { recursive: true });
|
||||||
|
await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true });
|
||||||
|
await writeFile(
|
||||||
|
join(stage, "manifest.env"),
|
||||||
|
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
await writeFile(join(stage, "files.txt"), `${entry}\n`, "utf8");
|
||||||
|
await mkdir(artifactDir, { recursive: true });
|
||||||
|
const tar = spawnSync(
|
||||||
|
"python3",
|
||||||
|
["-c", canonicalTarScript(), target, stage],
|
||||||
|
{ encoding: "utf8", maxBuffer: 16 * 1024 * 1024 },
|
||||||
|
);
|
||||||
|
if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
|
||||||
|
const sha256 = createHash("sha256")
|
||||||
|
.update(await readFile(target))
|
||||||
|
.digest("hex");
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
ok: true,
|
||||||
|
patchId,
|
||||||
|
component: "device-plane",
|
||||||
|
artifact: target,
|
||||||
|
sha256,
|
||||||
|
entries: [entry],
|
||||||
|
build: [],
|
||||||
|
services: ["device-control-core"],
|
||||||
|
transition: descriptor.mode,
|
||||||
|
runtimeAction: descriptor.runtimeAction,
|
||||||
|
preservedServices: descriptor.preservedServices,
|
||||||
|
}, null, 2));
|
||||||
|
} finally {
|
||||||
|
await rm(stage, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
function canonicalTarScript() {
|
||||||
|
return [
|
||||||
|
"import gzip,io,pathlib,sys,tarfile",
|
||||||
|
"root=pathlib.Path(sys.argv[2])",
|
||||||
|
"with open(sys.argv[1],'wb') as out:",
|
||||||
|
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
|
||||||
|
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
|
||||||
|
" for top in ('manifest.env','files.txt','payload'):",
|
||||||
|
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
|
||||||
|
" for x in paths:",
|
||||||
|
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
|
||||||
|
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import hashlib
|
||||||
|
import importlib.machinery
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
RUNNER_PATH = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"NODEDC_DEPLOY_RUNNER",
|
||||||
|
SCRIPT_DIR.parents[2] / "platform/infra/deploy-runner/nodedc-deploy",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
BUILDER = (
|
||||||
|
SCRIPT_DIR
|
||||||
|
/ "build-device-control-core-release-v3-reconciliation-artifact.mjs"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_runner():
|
||||||
|
loader = importlib.machinery.SourceFileLoader(
|
||||||
|
"nodedc_control_core_v3_reconciliation_runner_under_test",
|
||||||
|
str(RUNNER_PATH),
|
||||||
|
)
|
||||||
|
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
RUNNER = load_runner()
|
||||||
|
|
||||||
|
|
||||||
|
class DeviceControlCoreV3ReconciliationArtifactTest(unittest.TestCase):
|
||||||
|
def build(self, artifact_dir, patch_id):
|
||||||
|
environment = os.environ.copy()
|
||||||
|
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
|
||||||
|
result = subprocess.run(
|
||||||
|
["node", str(BUILDER), patch_id],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
env=environment,
|
||||||
|
)
|
||||||
|
return json.loads(result.stdout)
|
||||||
|
|
||||||
|
def test_artifact_is_exact_deterministic_runtime_reconciliation(self):
|
||||||
|
with tempfile.TemporaryDirectory(
|
||||||
|
prefix="nodedc-control-core-v3-reconciliation-artifact-",
|
||||||
|
) as directory:
|
||||||
|
artifact_dir = Path(directory)
|
||||||
|
patch_id = "device-control-core-v3-reconciliation-unit-042"
|
||||||
|
first = self.build(artifact_dir, patch_id)
|
||||||
|
first_bytes = Path(first["artifact"]).read_bytes()
|
||||||
|
second = self.build(artifact_dir, patch_id)
|
||||||
|
second_bytes = Path(second["artifact"]).read_bytes()
|
||||||
|
|
||||||
|
self.assertEqual(first_bytes, second_bytes)
|
||||||
|
self.assertEqual(
|
||||||
|
first["sha256"],
|
||||||
|
hashlib.sha256(first_bytes).hexdigest(),
|
||||||
|
)
|
||||||
|
self.assertEqual(first["component"], "device-plane")
|
||||||
|
self.assertEqual(
|
||||||
|
first["entries"],
|
||||||
|
list(RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES),
|
||||||
|
)
|
||||||
|
self.assertEqual(first["build"], [])
|
||||||
|
self.assertEqual(first["services"], ["device-control-core"])
|
||||||
|
|
||||||
|
with tarfile.open(first["artifact"], "r:gz") as archive:
|
||||||
|
names = [member.name for member in archive.getmembers()]
|
||||||
|
descriptor = json.loads(
|
||||||
|
archive.extractfile(
|
||||||
|
"payload/"
|
||||||
|
+ RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL
|
||||||
|
).read().decode("utf-8")
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
descriptor,
|
||||||
|
RUNNER.expected_device_plane_control_core_v3_reconciliation_descriptor(),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
names,
|
||||||
|
[
|
||||||
|
"manifest.env",
|
||||||
|
"files.txt",
|
||||||
|
"payload",
|
||||||
|
"payload/deployment",
|
||||||
|
"payload/"
|
||||||
|
+ RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
RUNNER.component_services(
|
||||||
|
"device-plane",
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES,
|
||||||
|
),
|
||||||
|
("device-control-core",),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
RUNNER.component_builds(
|
||||||
|
"device-plane",
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES,
|
||||||
|
),
|
||||||
|
(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_descriptor_pins_exact_failed_evidence_and_preapply_image(self):
|
||||||
|
descriptor = (
|
||||||
|
RUNNER.expected_device_plane_control_core_v3_reconciliation_descriptor()
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
descriptor["failedArtifactSha256"],
|
||||||
|
"08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
descriptor["preapplyImageId"],
|
||||||
|
"sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
descriptor["runtimeAction"],
|
||||||
|
"retag-exact-preapply-image+recreate-device-control-core-only",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_runtime_recovery_retags_exact_image_without_build(self):
|
||||||
|
completed = mock.Mock(returncode=0, stdout="", stderr="")
|
||||||
|
with mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"inspect_optional_local_image",
|
||||||
|
side_effect=[
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
|
||||||
|
],
|
||||||
|
), mock.patch.object(
|
||||||
|
RUNNER.subprocess,
|
||||||
|
"run",
|
||||||
|
return_value=completed,
|
||||||
|
) as run:
|
||||||
|
RUNNER.restore_device_plane_control_core_v3_preapply_image()
|
||||||
|
|
||||||
|
run.assert_called_once_with(
|
||||||
|
[
|
||||||
|
str(RUNNER.DOCKER),
|
||||||
|
"image",
|
||||||
|
"tag",
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_IMAGE,
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_reconciliation_runtime_selects_only_core_and_no_build(self):
|
||||||
|
mark_runtime_started = mock.Mock()
|
||||||
|
with mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"prepare_component_runtime",
|
||||||
|
) as prepare, mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"restore_device_plane_control_core_v3_preapply_image",
|
||||||
|
) as restore, mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"run_compose",
|
||||||
|
) as compose, mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"run_build",
|
||||||
|
) as build:
|
||||||
|
RUNNER.run_device_plane_runtime_for_apply(
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES,
|
||||||
|
("device-control-core",),
|
||||||
|
mark_runtime_started,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepare.assert_called_once_with(
|
||||||
|
"device-plane",
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES,
|
||||||
|
)
|
||||||
|
mark_runtime_started.assert_called_once_with()
|
||||||
|
restore.assert_called_once_with()
|
||||||
|
compose.assert_called_once_with(
|
||||||
|
"device-plane",
|
||||||
|
("device-control-core",),
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES,
|
||||||
|
)
|
||||||
|
build.assert_not_called()
|
||||||
|
|
||||||
|
def test_release_rollback_reuses_exact_preapply_image_without_rebuild(self):
|
||||||
|
runtime_inventory = {
|
||||||
|
"schemaVersion": "nodedc.device-plane.runtime-inventory.v1",
|
||||||
|
"composeProject": "nodedc-device-plane",
|
||||||
|
"services": [{
|
||||||
|
"service": "device-control-core",
|
||||||
|
"containerId": "1" * 64,
|
||||||
|
"imageId": RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
|
||||||
|
"status": "running",
|
||||||
|
"running": True,
|
||||||
|
"health": "healthy",
|
||||||
|
"restartCount": 0,
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
entries = RUNNER.DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES
|
||||||
|
with tempfile.TemporaryDirectory(
|
||||||
|
prefix="nodedc-control-core-v3-rollback-unit-",
|
||||||
|
) as directory, mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"read_backup_path_list",
|
||||||
|
side_effect=[list(entries), []],
|
||||||
|
), mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"validate_backup_partition",
|
||||||
|
return_value=(set(entries), set()),
|
||||||
|
), mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"read_strict_json",
|
||||||
|
return_value=runtime_inventory,
|
||||||
|
), mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"restore_platform_overlay",
|
||||||
|
return_value=len(entries),
|
||||||
|
), mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"retag_device_plane_control_core_image",
|
||||||
|
) as retag, mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"prepare_component_runtime",
|
||||||
|
), mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"run_compose",
|
||||||
|
) as compose, mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"run_component_runtime",
|
||||||
|
) as build_runtime, mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"accept_device_plane_control_core_rollback_runtime",
|
||||||
|
), mock.patch.object(
|
||||||
|
RUNNER,
|
||||||
|
"validate_device_manager_control_plane_runtime",
|
||||||
|
):
|
||||||
|
result = RUNNER.rollback_device_plane_apply(
|
||||||
|
Path(directory),
|
||||||
|
Path(directory),
|
||||||
|
entries,
|
||||||
|
"unit-stamp",
|
||||||
|
True,
|
||||||
|
("device-control-core",),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
result,
|
||||||
|
f"source+runtime-restored:{len(entries)}",
|
||||||
|
)
|
||||||
|
retag.assert_called_once_with(
|
||||||
|
RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
|
||||||
|
"Device Control Core exact pre-apply rollback image",
|
||||||
|
)
|
||||||
|
compose.assert_called_once_with(
|
||||||
|
"device-plane",
|
||||||
|
("device-control-core",),
|
||||||
|
list(entries),
|
||||||
|
)
|
||||||
|
build_runtime.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user