fix(device-plane): reconcile failed manager rollout
This commit is contained in:
Codex
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"schemaVersion": "nodedc.device-plane.device-manager-control-plane-reconciliation.v1",
|
||||
"mode": "failed-control-plane-baseline-adoption",
|
||||
"failedPatchId": "device-manager-control-plane-20260810-001",
|
||||
"failedArtifactSha256": "50e275c1085286bcb3bb2b273aefc8bbba70f446ca2c7bd464dc745710a291a6",
|
||||
"backupId": "device-plane-device-manager-control-plane-20260810-001-20260811-000321",
|
||||
"sourceAction": "publish-reconciliation-marker-only",
|
||||
"runtimeAction": "read-only-acceptance",
|
||||
"preservedServices": [
|
||||
"device-control-core",
|
||||
"device-gateway",
|
||||
"device-postgres",
|
||||
"device-backhaul-target"
|
||||
],
|
||||
"absentService": "device-manager",
|
||||
"databaseVolume": "nodedc-device-plane-postgres-data",
|
||||
"publicIngress": "disabled",
|
||||
"commandTransport": "disabled",
|
||||
"gelios": "untouched",
|
||||
"rollback": "marker-only-runtime-unchanged"
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env node
|
||||
import { createHash } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const platformRoot = resolve(scriptDir, "../..");
|
||||
const sourceRoot = resolve(platformRoot, "device-plane");
|
||||
const artifactDir = resolve(
|
||||
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|
||||
|| resolve(scriptDir, "../deploy-artifacts"),
|
||||
);
|
||||
const [
|
||||
patchId = "device-manager-control-plane-reconciliation-20260811-002",
|
||||
...extra
|
||||
] = process.argv.slice(2);
|
||||
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
|
||||
throw new Error(
|
||||
"usage: build-device-manager-control-plane-reconciliation-artifact.mjs [patch-id]",
|
||||
);
|
||||
}
|
||||
|
||||
const entry = "deployment/device-manager-control-plane-reconciliation-v1.json";
|
||||
const stage = await mkdtemp(join(tmpdir(), "nodedc-device-manager-reconciliation-"));
|
||||
const payload = join(stage, "payload");
|
||||
const target = resolve(
|
||||
artifactDir,
|
||||
`nodedc-device-plane-${patchId}.tgz`,
|
||||
);
|
||||
|
||||
try {
|
||||
const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entry), "utf8"));
|
||||
const expected = {
|
||||
schemaVersion: "nodedc.device-plane.device-manager-control-plane-reconciliation.v1",
|
||||
mode: "failed-control-plane-baseline-adoption",
|
||||
failedPatchId: "device-manager-control-plane-20260810-001",
|
||||
failedArtifactSha256:
|
||||
"50e275c1085286bcb3bb2b273aefc8bbba70f446ca2c7bd464dc745710a291a6",
|
||||
backupId:
|
||||
"device-plane-device-manager-control-plane-20260810-001-20260811-000321",
|
||||
sourceAction: "publish-reconciliation-marker-only",
|
||||
runtimeAction: "read-only-acceptance",
|
||||
preservedServices: [
|
||||
"device-control-core",
|
||||
"device-gateway",
|
||||
"device-postgres",
|
||||
"device-backhaul-target",
|
||||
],
|
||||
absentService: "device-manager",
|
||||
databaseVolume: "nodedc-device-plane-postgres-data",
|
||||
publicIngress: "disabled",
|
||||
commandTransport: "disabled",
|
||||
gelios: "untouched",
|
||||
rollback: "marker-only-runtime-unchanged",
|
||||
};
|
||||
if (JSON.stringify(descriptor) !== JSON.stringify(expected)) {
|
||||
throw new Error("device_manager_reconciliation_descriptor_mismatch");
|
||||
}
|
||||
|
||||
await mkdir(dirname(join(payload, entry)), { recursive: true });
|
||||
await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true });
|
||||
await writeFile(
|
||||
join(stage, "manifest.env"),
|
||||
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(join(stage, "files.txt"), `${entry}\n`, "utf8");
|
||||
await mkdir(artifactDir, { recursive: true });
|
||||
const tar = spawnSync(
|
||||
"python3",
|
||||
["-c", canonicalTarScript(), target, stage],
|
||||
{ encoding: "utf8", maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
|
||||
const sha256 = createHash("sha256")
|
||||
.update(await readFile(target))
|
||||
.digest("hex");
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
patchId,
|
||||
component: "device-plane",
|
||||
artifact: target,
|
||||
sha256,
|
||||
entries: [entry],
|
||||
build: [],
|
||||
services: [],
|
||||
transition: "failed-control-plane-baseline-adoption",
|
||||
runtimeAction: "read-only-acceptance",
|
||||
sourceAction: "publish-reconciliation-marker-only",
|
||||
}, null, 2));
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function canonicalTarScript() {
|
||||
return [
|
||||
"import gzip,io,pathlib,sys,tarfile",
|
||||
"root=pathlib.Path(sys.argv[2])",
|
||||
"with open(sys.argv[1],'wb') as out:",
|
||||
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
|
||||
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
|
||||
" for top in ('manifest.env','files.txt','payload'):",
|
||||
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
|
||||
" for x in paths:",
|
||||
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
|
||||
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
|
||||
].join("\n");
|
||||
}
|
||||
@@ -230,6 +230,61 @@ DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES = (
|
||||
"services/device-manager",
|
||||
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_REL = (
|
||||
"deployment/device-manager-control-plane-reconciliation-v1.json"
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES = (
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_FAILED_PATCH_ID = (
|
||||
"device-manager-control-plane-20260810-001"
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_FAILED_ARTIFACT = (
|
||||
"nodedc-device-plane-device-manager-control-plane-20260810-001.tgz."
|
||||
"20260811-000321"
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256 = (
|
||||
"50e275c1085286bcb3bb2b273aefc8bbba70f446ca2c7bd464dc745710a291a6"
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID = (
|
||||
"device-plane-device-manager-control-plane-20260810-001-"
|
||||
"20260811-000321"
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_SHA256 = {
|
||||
"manifest.env": (
|
||||
"70c8e6514f0a41cdd8892e29ca74c311c68e31bb74a7653d9b37e790db968526"
|
||||
),
|
||||
"files.txt": (
|
||||
"2101d48cfb832f50e6112e443b4b61813e9e6d94c8171ca66fb23ed7f51bc575"
|
||||
),
|
||||
"existing-files.txt": (
|
||||
"b605a86ce5ed6c0adb7b4cbeb5758795fe4219367df2c18db9411825f232e308"
|
||||
),
|
||||
"missing-files.txt": (
|
||||
"84c546b7ddb0cd8c80b3de29f583ec48678bc9bb47ad0460860dc0022a6b87e9"
|
||||
),
|
||||
"source-before.tgz": (
|
||||
"df6f29d173a56b0a067358d4b763c2de280450cf597957d1e79b8ae168e57735"
|
||||
),
|
||||
"runtime-before.json": (
|
||||
"590310c9ad923516a714c7d47ba26eee2d82113cb800a5a7bff29a7de94f165d"
|
||||
),
|
||||
}
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING = (
|
||||
".dockerignore",
|
||||
"package.json",
|
||||
"package-lock.json",
|
||||
"packages/device-protocol-contract",
|
||||
"packages/arusnavi-b2-adapter",
|
||||
"services/device-control-core",
|
||||
"services/device-gateway/package.json",
|
||||
"services/device-edge-relay/package.json",
|
||||
)
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING = (
|
||||
DEVICE_PLANE_MANAGER_COMPOSE_REL,
|
||||
"services/device-manager",
|
||||
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
|
||||
)
|
||||
DEVICE_PLANE_FOUNDATION_RECOVERY_REL = (
|
||||
"deployment/device-plane-foundation-recovery-v1.json"
|
||||
)
|
||||
@@ -3115,6 +3170,7 @@ def allowed_payload_path(component, rel):
|
||||
DEVICE_PLANE_B2_DISCOVERY_ROLLBACK_RECOVERY_REL,
|
||||
DEVICE_PLANE_BACKHAUL_TARGET_REL,
|
||||
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
||||
"packages/device-protocol-contract",
|
||||
"packages/arusnavi-b2-adapter",
|
||||
"services/device-control-core",
|
||||
@@ -8160,6 +8216,11 @@ def load_artifact(artifact, work_dir):
|
||||
entries,
|
||||
):
|
||||
validate_device_plane_manager_control_plane_payload(payload_dir)
|
||||
if is_device_plane_manager_reconciliation_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
):
|
||||
validate_device_plane_manager_reconciliation_payload(payload_dir)
|
||||
if is_device_plane_postgres_bootstrap_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
@@ -8494,6 +8555,17 @@ def reject_terminal_device_plane_foundation_artifact(manifest, sha256):
|
||||
)
|
||||
|
||||
|
||||
def reject_terminal_device_plane_manager_artifact(manifest, sha256):
|
||||
if (
|
||||
manifest.get("id") == DEVICE_PLANE_MANAGER_FAILED_PATCH_ID
|
||||
or sha256 == DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
|
||||
):
|
||||
die(
|
||||
"Device Manager control-plane 001 is terminal failed; "
|
||||
"use the exact registered reconciliation successor"
|
||||
)
|
||||
|
||||
|
||||
def reject_terminal_device_plane_backhaul_artifact(manifest, sha256):
|
||||
if (
|
||||
manifest.get("id") == DEVICE_PLANE_BACKHAUL_FAILED_PATCH_ID
|
||||
@@ -8555,6 +8627,14 @@ def is_device_plane_manager_control_plane_slice(component, entries):
|
||||
)
|
||||
|
||||
|
||||
def is_device_plane_manager_reconciliation_slice(component, entries):
|
||||
return (
|
||||
component == "device-plane"
|
||||
and entries is not None
|
||||
and tuple(entries) == DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES
|
||||
)
|
||||
|
||||
|
||||
def expected_platform_device_core_hub_trust_descriptor():
|
||||
return {
|
||||
"schemaVersion": "nodedc.platform.device-core-hub-trust.v1",
|
||||
@@ -8592,6 +8672,35 @@ def expected_device_plane_manager_control_plane_descriptor():
|
||||
}
|
||||
|
||||
|
||||
def expected_device_plane_manager_reconciliation_descriptor():
|
||||
return {
|
||||
"schemaVersion": (
|
||||
"nodedc.device-plane.device-manager-control-plane-"
|
||||
"reconciliation.v1"
|
||||
),
|
||||
"mode": "failed-control-plane-baseline-adoption",
|
||||
"failedPatchId": DEVICE_PLANE_MANAGER_FAILED_PATCH_ID,
|
||||
"failedArtifactSha256": (
|
||||
DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
|
||||
),
|
||||
"backupId": DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID,
|
||||
"sourceAction": "publish-reconciliation-marker-only",
|
||||
"runtimeAction": "read-only-acceptance",
|
||||
"preservedServices": [
|
||||
"device-control-core",
|
||||
"device-gateway",
|
||||
"device-postgres",
|
||||
"device-backhaul-target",
|
||||
],
|
||||
"absentService": "device-manager",
|
||||
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
|
||||
"publicIngress": "disabled",
|
||||
"commandTransport": "disabled",
|
||||
"gelios": "untouched",
|
||||
"rollback": "marker-only-runtime-unchanged",
|
||||
}
|
||||
|
||||
|
||||
def validate_platform_device_core_hub_trust_payload(payload_dir):
|
||||
descriptor = read_strict_json(
|
||||
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
|
||||
@@ -8673,6 +8782,17 @@ def validate_device_plane_manager_control_plane_payload(payload_dir):
|
||||
return descriptor
|
||||
|
||||
|
||||
def validate_device_plane_manager_reconciliation_payload(payload_dir):
|
||||
descriptor = read_strict_json(
|
||||
payload_dir / DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
||||
"Device Manager control-plane reconciliation descriptor",
|
||||
max_bytes=16 * 1024,
|
||||
)
|
||||
if descriptor != expected_device_plane_manager_reconciliation_descriptor():
|
||||
die("Device Manager control-plane reconciliation descriptor mismatch")
|
||||
return descriptor
|
||||
|
||||
|
||||
def is_device_plane_postgres_bootstrap_slice(component, entries):
|
||||
return (
|
||||
component == "device-plane"
|
||||
@@ -9895,6 +10015,207 @@ def validate_device_plane_b2_discovery_rollback_recovery_evidence(
|
||||
}
|
||||
|
||||
|
||||
def validate_device_plane_manager_reconciliation_backup():
|
||||
backup_dir = (
|
||||
BACKUPS_DIR / DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID
|
||||
)
|
||||
try:
|
||||
backup_stat = backup_dir.lstat()
|
||||
except FileNotFoundError:
|
||||
die("Device Manager failed-apply backup is missing")
|
||||
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
|
||||
backup_stat.st_mode
|
||||
):
|
||||
die("Device Manager failed-apply backup is unsafe")
|
||||
expected = DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_SHA256
|
||||
if {child.name for child in backup_dir.iterdir()} != set(expected):
|
||||
die("Device Manager failed-apply backup file set mismatch")
|
||||
for name, expected_sha256 in expected.items():
|
||||
path = backup_dir / name
|
||||
path_stat = path.lstat()
|
||||
if (
|
||||
stat.S_ISLNK(path_stat.st_mode)
|
||||
or not stat.S_ISREG(path_stat.st_mode)
|
||||
or sha256_file(path) != expected_sha256
|
||||
):
|
||||
die(
|
||||
"Device Manager failed-apply backup drift detected: "
|
||||
f"{name}"
|
||||
)
|
||||
existing = tuple(read_backup_path_list(
|
||||
backup_dir / "existing-files.txt"
|
||||
))
|
||||
missing = tuple(read_backup_path_list(
|
||||
backup_dir / "missing-files.txt"
|
||||
))
|
||||
validate_backup_partition(
|
||||
DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES,
|
||||
existing,
|
||||
missing,
|
||||
"Device Manager reconciliation",
|
||||
)
|
||||
if (
|
||||
existing != DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING
|
||||
or missing != DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING
|
||||
):
|
||||
die("Device Manager failed-apply backup partition mismatch")
|
||||
return backup_dir
|
||||
|
||||
|
||||
def validate_device_plane_manager_reconciled_baseline(
|
||||
backup_dir,
|
||||
*,
|
||||
marker_installed,
|
||||
):
|
||||
root = component_root("device-plane")
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="device-manager-reconciled-baseline-",
|
||||
dir=TMP_DIR,
|
||||
) as directory:
|
||||
backup_root = Path(directory)
|
||||
materialize_backup_tree(
|
||||
backup_dir / "source-before.tgz",
|
||||
backup_root,
|
||||
set(DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING),
|
||||
)
|
||||
backup_source = collect_exact_files(
|
||||
backup_root,
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING,
|
||||
"Device Manager pre-apply source",
|
||||
)
|
||||
live_source = collect_exact_files(
|
||||
root,
|
||||
DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING,
|
||||
"Device Manager reconciled live source",
|
||||
)
|
||||
if live_source != backup_source:
|
||||
die("Device Manager rollback source does not match backup")
|
||||
for rel in DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING:
|
||||
path = root / rel
|
||||
if path.exists() or path.is_symlink():
|
||||
die(
|
||||
"Device Manager candidate-only source remains installed: "
|
||||
f"{rel}"
|
||||
)
|
||||
|
||||
marker = root / DEVICE_PLANE_MANAGER_RECONCILIATION_REL
|
||||
if marker_installed:
|
||||
descriptor = read_strict_json(
|
||||
marker,
|
||||
"installed Device Manager reconciliation descriptor",
|
||||
max_bytes=16 * 1024,
|
||||
)
|
||||
if descriptor != expected_device_plane_manager_reconciliation_descriptor():
|
||||
die("installed Device Manager reconciliation descriptor mismatch")
|
||||
elif marker.exists() or marker.is_symlink():
|
||||
die("Device Manager reconciliation descriptor already installed")
|
||||
|
||||
if device_plane_service_container_ids("device-manager"):
|
||||
die("Device Manager candidate container remains installed")
|
||||
runtime_before = read_strict_json(
|
||||
backup_dir / "runtime-before.json",
|
||||
"Device Manager pre-apply runtime inventory",
|
||||
max_bytes=64 * 1024,
|
||||
)
|
||||
runtime = validate_device_plane_b2_discovery_ingress_runtime(
|
||||
runtime_before,
|
||||
preserved_stateless_services=("device-gateway",),
|
||||
)
|
||||
core_ids = device_plane_service_container_ids("device-control-core")
|
||||
if len(core_ids) != 1:
|
||||
die("Device Manager reconciled Core service count mismatch")
|
||||
core = inspect_device_plane_container(core_ids[0])
|
||||
core_environment = container_environment(
|
||||
core,
|
||||
"Device Manager reconciled Control Core",
|
||||
)
|
||||
if (
|
||||
"DEVICE_MANAGEMENT_API_ENABLED" in core_environment
|
||||
or "DEVICE_MANAGEMENT_CORE_TOKEN_FILE" in core_environment
|
||||
or any(
|
||||
mount.get("Destination")
|
||||
== "/run/nodedc-secrets/management-core-token"
|
||||
for mount in core.get("Mounts") or []
|
||||
)
|
||||
):
|
||||
die("Device Manager management boundary remains active after rollback")
|
||||
return runtime
|
||||
|
||||
|
||||
def validate_device_plane_manager_reconciliation_evidence(payload_dir):
|
||||
descriptor = validate_device_plane_manager_reconciliation_payload(
|
||||
payload_dir
|
||||
)
|
||||
backup_dir = validate_device_plane_manager_reconciliation_backup()
|
||||
failed_artifact = FAILED_DIR / DEVICE_PLANE_MANAGER_FAILED_ARTIFACT
|
||||
try:
|
||||
failed_stat = failed_artifact.lstat()
|
||||
except FileNotFoundError:
|
||||
die("Device Manager failed artifact is missing")
|
||||
if (
|
||||
stat.S_ISLNK(failed_stat.st_mode)
|
||||
or not stat.S_ISREG(failed_stat.st_mode)
|
||||
or sha256_file(failed_artifact)
|
||||
!= DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
|
||||
):
|
||||
die("Device Manager failed artifact evidence mismatch")
|
||||
|
||||
records = [
|
||||
value
|
||||
for value in load_state(FAILED_STATE_FILE)
|
||||
if value.get("id") == DEVICE_PLANE_MANAGER_FAILED_PATCH_ID
|
||||
]
|
||||
if len(records) != 1:
|
||||
die("Device Manager failed journal evidence count mismatch")
|
||||
record = records[0]
|
||||
if (
|
||||
record.get("artifact") != DEVICE_PLANE_MANAGER_FAILED_ARTIFACT
|
||||
or record.get("backup_id")
|
||||
!= DEVICE_PLANE_MANAGER_RECONCILIATION_BACKUP_ID
|
||||
or record.get("component") != "device-plane"
|
||||
or record.get("sha256")
|
||||
!= DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256
|
||||
or record.get("started_apply") is not True
|
||||
or record.get("rollback_status") != "failed:DeployError"
|
||||
or record.get("status") != "failed"
|
||||
or record.get("message")
|
||||
!= (
|
||||
"container healthcheck failed for "
|
||||
"500bc061b97d9007a33dd51d35ca1168eae7f6b019165b143d3bd908220656c0: "
|
||||
"unhealthy"
|
||||
)
|
||||
):
|
||||
die("Device Manager failed journal evidence mismatch")
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="device-manager-failed-artifact-",
|
||||
dir=TMP_DIR,
|
||||
) as directory:
|
||||
failed_manifest, failed_entries, _failed_payload = load_artifact(
|
||||
failed_artifact,
|
||||
Path(directory),
|
||||
)
|
||||
if (
|
||||
failed_manifest.get("id") != DEVICE_PLANE_MANAGER_FAILED_PATCH_ID
|
||||
or failed_manifest.get("component") != "device-plane"
|
||||
or failed_manifest.get("type") != "app-overlay"
|
||||
or tuple(failed_entries)
|
||||
!= DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES
|
||||
):
|
||||
die("Device Manager failed artifact contract mismatch")
|
||||
|
||||
runtime = validate_device_plane_manager_reconciled_baseline(
|
||||
backup_dir,
|
||||
marker_installed=False,
|
||||
)
|
||||
return {
|
||||
"mode": descriptor["mode"],
|
||||
"backup": backup_dir,
|
||||
"failedArtifact": failed_artifact,
|
||||
"runtime": runtime,
|
||||
}
|
||||
|
||||
|
||||
def device_plane_service_container_ids(service):
|
||||
if service not in (
|
||||
*DEVICE_PLANE_RUNTIME_SERVICES,
|
||||
@@ -10803,8 +11124,17 @@ def validate_device_plane_foundation_runtime(network_publication=False):
|
||||
return accepted
|
||||
|
||||
|
||||
def validate_device_plane_b2_discovery_ingress_runtime(runtime_before):
|
||||
def validate_device_plane_b2_discovery_ingress_runtime(
|
||||
runtime_before,
|
||||
preserved_stateless_services=(),
|
||||
):
|
||||
validate_device_plane_runtime_secret_metadata()
|
||||
preserved_stateless = set(preserved_stateless_services)
|
||||
if not preserved_stateless.issubset({
|
||||
"device-control-core",
|
||||
"device-gateway",
|
||||
}):
|
||||
die("Device Plane B2 preserved stateless service set mismatch")
|
||||
before_names = device_plane_inventory_service_names(runtime_before)
|
||||
if set(before_names) != set(DEVICE_PLANE_RUNTIME_SERVICES):
|
||||
die("Device Plane B2 ingress predecessor inventory mismatch")
|
||||
@@ -10903,11 +11233,20 @@ def validate_device_plane_b2_discovery_ingress_runtime(runtime_before):
|
||||
):
|
||||
die("Device Plane B2 ingress changed PostgreSQL generation")
|
||||
for service in ("device-control-core", "device-gateway"):
|
||||
if (
|
||||
same_container = (
|
||||
accepted[service]["containerId"]
|
||||
== before[service]["containerId"]
|
||||
or accepted[service]["imageId"] == before[service]["imageId"]
|
||||
):
|
||||
)
|
||||
same_image = (
|
||||
accepted[service]["imageId"] == before[service]["imageId"]
|
||||
)
|
||||
if service in preserved_stateless:
|
||||
if not (same_container and same_image):
|
||||
die(
|
||||
"Device Plane B2 stateless generation was not preserved: "
|
||||
f"{service}"
|
||||
)
|
||||
elif same_container or same_image:
|
||||
die(
|
||||
"Device Plane B2 ingress stateless generation not replaced: "
|
||||
f"{service}"
|
||||
@@ -13061,6 +13400,11 @@ def component_services(component, entries=None):
|
||||
# failed archive, journal, backup, restored source and live runtime.
|
||||
return ()
|
||||
|
||||
if is_device_plane_manager_reconciliation_slice(component, entries):
|
||||
# Reconciliation publishes one marker after proving the failed
|
||||
# evidence and the already-restored baseline. Runtime is read-only.
|
||||
return ()
|
||||
|
||||
if is_device_plane_postgres_bootstrap_slice(component, entries):
|
||||
# This exact one-time transition is the only Device Plane artifact that
|
||||
# may select durable state. Its preflight requires both container and
|
||||
@@ -13476,6 +13820,9 @@ def component_builds(component, entries=None):
|
||||
),
|
||||
)
|
||||
|
||||
if is_device_plane_manager_reconciliation_slice(component, entries):
|
||||
return ()
|
||||
|
||||
if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
|
||||
return ()
|
||||
|
||||
@@ -15498,6 +15845,7 @@ def plan_artifact(artifact):
|
||||
device_plane_network_publication_preflight = None
|
||||
device_plane_b2_ingress_preflight = None
|
||||
device_plane_b2_recovery_preflight = None
|
||||
device_plane_manager_reconciliation_preflight = None
|
||||
device_plane_backhaul_preflight = None
|
||||
device_plane_backhaul_vps_enrollment_preflight = None
|
||||
device_plane_runtime_before = None
|
||||
@@ -15525,6 +15873,7 @@ def plan_artifact(artifact):
|
||||
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
|
||||
reject_terminal_engine_l2_failed_artifact(manifest, sha)
|
||||
reject_terminal_device_plane_foundation_artifact(manifest, sha)
|
||||
reject_terminal_device_plane_manager_artifact(manifest, sha)
|
||||
reject_terminal_device_plane_backhaul_artifact(manifest, sha)
|
||||
transition_descriptor = None
|
||||
transition_preflight = None
|
||||
@@ -15702,6 +16051,15 @@ def plan_artifact(artifact):
|
||||
payload_dir
|
||||
)
|
||||
)
|
||||
if is_device_plane_manager_reconciliation_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
):
|
||||
device_plane_manager_reconciliation_preflight = (
|
||||
validate_device_plane_manager_reconciliation_evidence(
|
||||
payload_dir
|
||||
)
|
||||
)
|
||||
if is_device_plane_backhaul_target_slice(
|
||||
manifest["component"],
|
||||
entries,
|
||||
@@ -17255,6 +17613,38 @@ def plan_artifact(artifact):
|
||||
)
|
||||
print("device_gateway_tcp_9921=disabled:unpublished")
|
||||
print("device_plane_rollback=marker-only-runtime-unchanged")
|
||||
if device_plane_manager_reconciliation_preflight is not None:
|
||||
print(
|
||||
"device_plane_transition="
|
||||
f"{device_plane_manager_reconciliation_preflight['mode']}"
|
||||
)
|
||||
print(
|
||||
"failed_patch="
|
||||
f"{DEVICE_PLANE_MANAGER_FAILED_PATCH_ID}"
|
||||
)
|
||||
print(
|
||||
"failed_artifact_sha256="
|
||||
f"{DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256}"
|
||||
)
|
||||
print(
|
||||
"recovery_backup="
|
||||
f"{device_plane_manager_reconciliation_preflight['backup'].name}"
|
||||
)
|
||||
print("device_plane_build=none")
|
||||
print("device_plane_runtime_mutation=none")
|
||||
print(
|
||||
"device_plane_runtime_services="
|
||||
"preserved:device-control-core,device-gateway,"
|
||||
"device-postgres,device-backhaul-target"
|
||||
)
|
||||
print("device_manager=absent")
|
||||
print(
|
||||
"device_plane_source_action="
|
||||
"publish-reconciliation-marker-only"
|
||||
)
|
||||
print("device_gateway_tcp_9921=preserved:loopback-only")
|
||||
print("device_manager_public_route=unchanged:absent")
|
||||
print("device_plane_rollback=marker-only-runtime-unchanged")
|
||||
if device_plane_backhaul_vps_enrollment_preflight is not None:
|
||||
print(
|
||||
"device_plane_transition="
|
||||
@@ -17867,11 +18257,24 @@ def rollback_device_plane_apply(
|
||||
baseline_entries,
|
||||
baseline_services,
|
||||
)
|
||||
run_healthchecks(
|
||||
"device-plane",
|
||||
baseline_entries,
|
||||
baseline_services,
|
||||
)
|
||||
if is_device_plane_manager_control_plane_slice("device-plane", entries):
|
||||
for service in baseline_services:
|
||||
healthcheck_compose_service_with_grace(
|
||||
"device-plane",
|
||||
service,
|
||||
)
|
||||
for check in component_healthchecks(
|
||||
"device-plane",
|
||||
baseline_entries,
|
||||
baseline_services,
|
||||
):
|
||||
healthcheck_url(check)
|
||||
else:
|
||||
run_healthchecks(
|
||||
"device-plane",
|
||||
baseline_entries,
|
||||
baseline_services,
|
||||
)
|
||||
return f"source+runtime-restored:{restored_count}"
|
||||
|
||||
|
||||
@@ -18755,6 +19158,8 @@ def run_component_runtime(component, entries, services):
|
||||
entries,
|
||||
):
|
||||
return
|
||||
if is_device_plane_manager_reconciliation_slice(component, entries):
|
||||
return
|
||||
if is_device_plane_foundation_recovery_slice(component, entries):
|
||||
return
|
||||
if is_device_plane_foundation_network_publication_slice(
|
||||
@@ -18782,8 +19187,9 @@ def run_device_plane_runtime_for_apply(
|
||||
"device-plane",
|
||||
entries,
|
||||
) or is_device_plane_foundation_recovery_slice(
|
||||
"device-plane",
|
||||
entries,
|
||||
"device-plane", entries
|
||||
) or is_device_plane_manager_reconciliation_slice(
|
||||
"device-plane", entries
|
||||
):
|
||||
return
|
||||
# Build and runtime preparation are pre-runtime phases. A failure here
|
||||
@@ -19187,6 +19593,46 @@ def healthcheck_container(container_name):
|
||||
die(f"container healthcheck failed for {container_name}: {last_status}")
|
||||
|
||||
|
||||
def healthcheck_container_with_grace(container_name):
|
||||
"""Wait through bounded transient unhealthy/exited states.
|
||||
|
||||
Device Manager activation and its baseline rollback both recreate a Core
|
||||
generation while Docker health probes and restart policy converge. The
|
||||
ordinary helper deliberately fails on the first terminal-looking state;
|
||||
this exact transition instead requires a healthy result within the same
|
||||
bounded five-minute window.
|
||||
"""
|
||||
last_status = "unknown"
|
||||
for attempt in range(1, 61):
|
||||
result = subprocess.run(
|
||||
[
|
||||
str(DOCKER),
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{if .State.Health}}{{.State.Health.Status}}"
|
||||
"{{else}}{{.State.Status}}{{end}}",
|
||||
container_name,
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
status_text = result.stdout.strip().lower()
|
||||
last_status = (
|
||||
status_text
|
||||
or result.stderr.strip()
|
||||
or f"inspect-exit-{result.returncode}"
|
||||
)
|
||||
if status_text in ("healthy", "running"):
|
||||
return
|
||||
if attempt < 60:
|
||||
time.sleep(5)
|
||||
die(
|
||||
"container healthcheck grace exhausted for "
|
||||
f"{container_name}: {last_status}"
|
||||
)
|
||||
|
||||
|
||||
def compose_service_container_id(component, service):
|
||||
result = subprocess.run(
|
||||
[*compose_base_cmd(component), "ps", "-q", service],
|
||||
@@ -19209,6 +19655,12 @@ def healthcheck_compose_service(component, service):
|
||||
healthcheck_container(compose_service_container_id(component, service))
|
||||
|
||||
|
||||
def healthcheck_compose_service_with_grace(component, service):
|
||||
healthcheck_container_with_grace(
|
||||
compose_service_container_id(component, service)
|
||||
)
|
||||
|
||||
|
||||
def container_environment(container, label):
|
||||
environment = {}
|
||||
for raw in (container.get("Config") or {}).get("Env") or []:
|
||||
@@ -19467,6 +19919,23 @@ process.stdout.write('engine-l2-closed-loop:0.7.0:cas+safe-profile+external-plan
|
||||
|
||||
|
||||
def run_healthchecks(component, entries=None, services=None):
|
||||
if is_device_plane_manager_reconciliation_slice(component, entries):
|
||||
if tuple(services or ()) != ():
|
||||
die("Device Manager reconciliation service set mismatch")
|
||||
for service in DEVICE_PLANE_RUNTIME_SERVICES:
|
||||
healthcheck_compose_service_with_grace(
|
||||
"device-plane",
|
||||
service,
|
||||
)
|
||||
for check in component_healthchecks(component, entries, services):
|
||||
healthcheck_url(check)
|
||||
backup_dir = validate_device_plane_manager_reconciliation_backup()
|
||||
validate_device_plane_manager_reconciled_baseline(
|
||||
backup_dir,
|
||||
marker_installed=True,
|
||||
)
|
||||
return
|
||||
|
||||
if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
|
||||
if tuple(services or ()) != (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,):
|
||||
die("Device Plane VPS enrollment service set mismatch")
|
||||
@@ -19583,7 +20052,10 @@ def run_healthchecks(component, entries=None, services=None):
|
||||
):
|
||||
die("Device Manager control-plane service set mismatch")
|
||||
for service in services:
|
||||
healthcheck_compose_service("device-plane", service)
|
||||
healthcheck_compose_service_with_grace(
|
||||
"device-plane",
|
||||
service,
|
||||
)
|
||||
for check in component_healthchecks(component, entries, services):
|
||||
healthcheck_url(check)
|
||||
validate_device_manager_control_plane_runtime()
|
||||
@@ -20374,6 +20846,10 @@ def apply_artifact(artifact):
|
||||
manifest,
|
||||
sha,
|
||||
)
|
||||
reject_terminal_device_plane_manager_artifact(
|
||||
manifest,
|
||||
sha,
|
||||
)
|
||||
reject_terminal_device_plane_backhaul_artifact(
|
||||
manifest,
|
||||
sha,
|
||||
@@ -20425,6 +20901,13 @@ def apply_artifact(artifact):
|
||||
validate_device_plane_b2_discovery_rollback_recovery_evidence(
|
||||
payload_dir
|
||||
)
|
||||
if is_device_plane_manager_reconciliation_slice(
|
||||
component,
|
||||
entries,
|
||||
):
|
||||
validate_device_plane_manager_reconciliation_evidence(
|
||||
payload_dir
|
||||
)
|
||||
if is_device_plane_backhaul_target_slice(
|
||||
component,
|
||||
entries,
|
||||
|
||||
@@ -209,7 +209,7 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase):
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"healthcheck_compose_service",
|
||||
"healthcheck_compose_service_with_grace",
|
||||
) as service_health,
|
||||
mock.patch.object(RUNNER, "healthcheck_url") as url_health,
|
||||
mock.patch.object(
|
||||
@@ -277,8 +277,21 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase):
|
||||
) as restore_runtime,
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"run_healthchecks",
|
||||
"healthcheck_compose_service_with_grace",
|
||||
) as restore_health,
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_healthchecks",
|
||||
return_value=("core-health",),
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"healthcheck_url",
|
||||
) as restore_url_health,
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"run_healthchecks",
|
||||
) as generic_health,
|
||||
):
|
||||
result = RUNNER.rollback_device_plane_apply(
|
||||
Path(directory) / "live",
|
||||
@@ -298,9 +311,10 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase):
|
||||
)
|
||||
restore_health.assert_called_once_with(
|
||||
"device-plane",
|
||||
existing,
|
||||
("device-control-core",),
|
||||
"device-control-core",
|
||||
)
|
||||
restore_url_health.assert_called_once_with("core-health")
|
||||
generic_health.assert_not_called()
|
||||
self.assertEqual(
|
||||
result,
|
||||
f"source+runtime-restored:{len(entries)}",
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
||||
BUILDER = (
|
||||
SCRIPT_DIR
|
||||
/ "build-device-manager-control-plane-reconciliation-artifact.mjs"
|
||||
)
|
||||
|
||||
|
||||
def load_runner():
|
||||
loader = importlib.machinery.SourceFileLoader(
|
||||
"nodedc_device_manager_reconciliation_runner_under_test",
|
||||
str(RUNNER_PATH),
|
||||
)
|
||||
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
RUNNER = load_runner()
|
||||
|
||||
|
||||
class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase):
|
||||
def build(self, artifact_dir, patch_id):
|
||||
environment = os.environ.copy()
|
||||
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
|
||||
result = subprocess.run(
|
||||
["node", str(BUILDER), patch_id],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
def test_artifact_is_marker_only_exact_and_deterministic(self):
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="nodedc-device-manager-reconciliation-artifact-",
|
||||
) as directory:
|
||||
artifact_dir = Path(directory)
|
||||
first = self.build(
|
||||
artifact_dir,
|
||||
"device-manager-control-plane-reconciliation-unit-002",
|
||||
)
|
||||
first_bytes = Path(first["artifact"]).read_bytes()
|
||||
second = self.build(
|
||||
artifact_dir,
|
||||
"device-manager-control-plane-reconciliation-unit-002",
|
||||
)
|
||||
second_bytes = Path(second["artifact"]).read_bytes()
|
||||
|
||||
self.assertEqual(first_bytes, second_bytes)
|
||||
self.assertEqual(
|
||||
first["sha256"],
|
||||
hashlib.sha256(first_bytes).hexdigest(),
|
||||
)
|
||||
self.assertEqual(first["component"], "device-plane")
|
||||
self.assertEqual(
|
||||
first["entries"],
|
||||
list(RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES),
|
||||
)
|
||||
self.assertEqual(first["build"], [])
|
||||
self.assertEqual(first["services"], [])
|
||||
self.assertEqual(
|
||||
first["runtimeAction"],
|
||||
"read-only-acceptance",
|
||||
)
|
||||
|
||||
with tarfile.open(first["artifact"], "r:gz") as archive:
|
||||
names = [member.name for member in archive.getmembers()]
|
||||
descriptor = json.loads(
|
||||
archive.extractfile(
|
||||
"payload/"
|
||||
+ RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_REL
|
||||
).read().decode("utf-8")
|
||||
)
|
||||
self.assertEqual(
|
||||
descriptor,
|
||||
RUNNER.expected_device_plane_manager_reconciliation_descriptor(),
|
||||
)
|
||||
self.assertEqual(
|
||||
names,
|
||||
[
|
||||
"manifest.env",
|
||||
"files.txt",
|
||||
"payload",
|
||||
"payload/deployment",
|
||||
"payload/"
|
||||
+ RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
RUNNER.component_services(
|
||||
"device-plane",
|
||||
RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES,
|
||||
),
|
||||
(),
|
||||
)
|
||||
self.assertEqual(
|
||||
RUNNER.component_builds(
|
||||
"device-plane",
|
||||
RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES,
|
||||
),
|
||||
(),
|
||||
)
|
||||
|
||||
def test_failed_control_plane_is_terminal(self):
|
||||
with self.assertRaisesRegex(
|
||||
RUNNER.DeployError,
|
||||
"exact registered reconciliation successor",
|
||||
):
|
||||
RUNNER.reject_terminal_device_plane_manager_artifact(
|
||||
{"id": RUNNER.DEVICE_PLANE_MANAGER_FAILED_PATCH_ID},
|
||||
"0" * 64,
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
RUNNER.DeployError,
|
||||
"exact registered reconciliation successor",
|
||||
):
|
||||
RUNNER.reject_terminal_device_plane_manager_artifact(
|
||||
{"id": "different"},
|
||||
RUNNER.DEVICE_PLANE_MANAGER_FAILED_ARTIFACT_SHA256,
|
||||
)
|
||||
|
||||
def test_health_grace_waits_through_unhealthy_and_exited(self):
|
||||
results = [
|
||||
mock.Mock(stdout="unhealthy\n", stderr="", returncode=0),
|
||||
mock.Mock(stdout="exited\n", stderr="", returncode=0),
|
||||
mock.Mock(stdout="starting\n", stderr="", returncode=0),
|
||||
mock.Mock(stdout="healthy\n", stderr="", returncode=0),
|
||||
]
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER.subprocess,
|
||||
"run",
|
||||
side_effect=results,
|
||||
) as inspect,
|
||||
mock.patch.object(RUNNER.time, "sleep") as sleep,
|
||||
):
|
||||
RUNNER.healthcheck_container_with_grace("container-id")
|
||||
self.assertEqual(inspect.call_count, 4)
|
||||
self.assertEqual(sleep.call_count, 3)
|
||||
|
||||
def test_manager_rollback_uses_bounded_grace_for_restored_core(self):
|
||||
entries = RUNNER.DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES
|
||||
runtime_before = {
|
||||
"schemaVersion": "nodedc.device-plane.runtime-inventory.v1",
|
||||
"composeProject": "nodedc-device-plane",
|
||||
"services": [
|
||||
{
|
||||
"service": "device-control-core",
|
||||
"containerId": "a" * 64,
|
||||
"imageId": "sha256:" + "b" * 64,
|
||||
"status": "running",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"restartCount": 0,
|
||||
},
|
||||
{
|
||||
"service": "device-gateway",
|
||||
"containerId": "c" * 64,
|
||||
"imageId": "sha256:" + "d" * 64,
|
||||
"status": "running",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"restartCount": 0,
|
||||
},
|
||||
{
|
||||
"service": "device-postgres",
|
||||
"containerId": "e" * 64,
|
||||
"imageId": "sha256:" + "f" * 64,
|
||||
"status": "running",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"restartCount": 0,
|
||||
},
|
||||
],
|
||||
}
|
||||
existing = list(RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_EXISTING)
|
||||
missing = list(RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING)
|
||||
with (
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"read_backup_path_list",
|
||||
side_effect=[existing, missing],
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"read_strict_json",
|
||||
return_value=runtime_before,
|
||||
),
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"stop_and_remove_compose_services",
|
||||
) as stop,
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"restore_platform_overlay",
|
||||
return_value=len(entries),
|
||||
),
|
||||
mock.patch.object(RUNNER, "run_component_runtime") as runtime,
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"healthcheck_compose_service_with_grace",
|
||||
) as health,
|
||||
mock.patch.object(
|
||||
RUNNER,
|
||||
"component_healthchecks",
|
||||
return_value=("core",),
|
||||
),
|
||||
mock.patch.object(RUNNER, "healthcheck_url") as url_health,
|
||||
mock.patch.object(RUNNER, "run_healthchecks") as generic_health,
|
||||
):
|
||||
restored = RUNNER.rollback_device_plane_apply(
|
||||
Path("/live"),
|
||||
Path("/backup"),
|
||||
entries,
|
||||
"stamp",
|
||||
True,
|
||||
("device-control-core", "device-manager"),
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
restored,
|
||||
f"source+runtime-restored:{len(entries)}",
|
||||
)
|
||||
stop.assert_called_once_with("device-plane", ("device-manager",))
|
||||
runtime.assert_called_once_with(
|
||||
"device-plane",
|
||||
existing,
|
||||
("device-control-core",),
|
||||
)
|
||||
health.assert_called_once_with(
|
||||
"device-plane",
|
||||
"device-control-core",
|
||||
)
|
||||
url_health.assert_called_once_with("core")
|
||||
generic_health.assert_not_called()
|
||||
|
||||
def test_reconciliation_runtime_phase_is_read_only(self):
|
||||
entries = RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES
|
||||
with (
|
||||
mock.patch.object(RUNNER, "run_build") as build,
|
||||
mock.patch.object(RUNNER, "prepare_component_runtime") as prepare,
|
||||
mock.patch.object(RUNNER, "run_compose") as compose,
|
||||
):
|
||||
RUNNER.run_component_runtime("device-plane", entries, ())
|
||||
RUNNER.run_device_plane_runtime_for_apply(
|
||||
entries,
|
||||
(),
|
||||
mock.Mock(),
|
||||
)
|
||||
build.assert_not_called()
|
||||
prepare.assert_not_called()
|
||||
compose.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user