SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(deploy): restore exact preapply Core image

Browse Source
This commit is contained in:
Codex
2026-08-22 19:49:36 +03:00
parent c5c4e69acb
commit 8d501224b4
1 changed files with 543 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+543 -7
View File
@@ -383,6 +383,64 @@ DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES = (
"services/device-control-core",
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL = (
"deployment/device-control-core-release-v3-reconciliation-v1.json"
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES = (
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID = (
"device-control-core-release-v3-reconciliation-20260822-042"
)
DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID = (
"device-control-core-release-v3-20260822-040"
)
DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT = (
"nodedc-device-plane-device-control-core-release-v3-20260822-040.tgz."
"20260822-184245"
)
DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 = (
"08448a56cdf391076f92c5242e368fd0033b420167874645eebfa1f22184ee92"
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID = (
"device-plane-device-control-core-release-v3-20260822-040-"
"20260822-184245"
)
DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID = (
"sha256:31d35733ee46225b487c0f02a7b52d4ba2d13f5b99f6a717b7f5e6f5460b412a"
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_SHA256 = {
"manifest.env": (
"eeb6ba544cde0d5d45a7bf16e921ca8969f0e5307872a53a5448d6a1279f76db"
),
"files.txt": (
"be1bda7d2afff11cfaeacd607a63261f99c17d6c04c8c762424669585ad9a75f"
),
"existing-files.txt": (
"a491b4a4ad7bc8dd8e8fc5b008f992663ef64e569b016efe7de1f4f14e5629ad"
),
"missing-files.txt": (
"d04dd94a0f5a1fd6a7809219e953ade89a43ec5a2e29016cb666f684a77323c0"
),
"source-before.tgz": (
"1c119875cccb52b761d1e2b42a6d666c9f9287782b424283b20180ea502237c4"
),
"runtime-before.json": (
"b694a183345181e32a3335bb50d73c4b8df9ce45e54072f3a8dc06789684fa89"
),
}
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING = (
".dockerignore",
"package.json",
"package-lock.json",
"packages/device-protocol-contract",
"packages/device-edge-channel-contract",
"services/device-control-core",
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING = (
"packages/infrastructure-telemetry-contract",
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
)
DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_PATCH_ID = (
"device-edge-core-channel-upgrade-v4-20260812-023"
)
@@ -4713,6 +4771,7 @@ def allowed_payload_path(component, rel):
DEVICE_PLANE_CONTROL_CORE_RELEASE_REL,
DEVICE_PLANE_CONTROL_CORE_RELEASE_V2_REL,
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
"packages/device-protocol-contract",
@@ -11167,6 +11226,13 @@ def load_artifact(artifact, work_dir):
entries,
):
validate_device_plane_manager_v2_reconciliation_payload(payload_dir)
if is_device_plane_control_core_v3_reconciliation_slice(
manifest["component"],
entries,
):
validate_device_plane_control_core_v3_reconciliation_payload(
payload_dir
)
if is_device_plane_postgres_bootstrap_slice(
manifest["component"],
entries,
@@ -11807,6 +11873,15 @@ def is_device_plane_control_core_release_v3_slice(component, entries):
)
def is_device_plane_control_core_v3_reconciliation_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries)
== DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES
)
def is_device_plane_manager_v2_control_plane_slice(component, entries):
return (
component == "device-plane"
@@ -12458,6 +12533,43 @@ def expected_device_plane_manager_v2_reconciliation_descriptor():
}
def expected_device_plane_control_core_v3_reconciliation_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane.device-control-core-release-v3-"
"reconciliation.v1"
),
"mode": "failed-release-v3-exact-preapply-image-restore",
"failedPatchId": DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID,
"failedArtifactSha256": (
DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256
),
"failedArtifact": DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT,
"backupId": DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID,
"predecessorPatchId": "device-control-core-release-v2-20260822-038",
"predecessorArtifactSha256": (
"e2d062b82b022dba662522b5d6e192026ac964d78950d903295ca3cbbc95ab28"
),
"preapplyImageId": DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
"sourceAction": "accept-byte-exact-restored-preapply-source",
"runtimeAction": (
"retag-exact-preapply-image+recreate-device-control-core-only"
),
"preservedServices": [
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
"publicIngress": "disabled",
"edgeChannel": "core-initiated-pinned-mtls-registered-edges-only",
"commandTransport": "typed-service-ping-v1",
"gelios": "untouched-legacy-only",
"rollback": "marker+exact-preapply-image-runtime",
}
def validate_platform_device_core_hub_trust_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
@@ -13335,6 +13447,20 @@ def validate_device_plane_manager_v2_reconciliation_payload(payload_dir):
return descriptor
def validate_device_plane_control_core_v3_reconciliation_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
"Device Control Core v3 reconciliation descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_control_core_v3_reconciliation_descriptor()
):
die("Device Control Core v3 reconciliation descriptor mismatch")
return descriptor
def is_device_plane_postgres_bootstrap_slice(component, entries):
return (
component == "device-plane"
@@ -15935,6 +16061,247 @@ def validate_device_plane_manager_v2_reconciliation_evidence(payload_dir):
}
def validate_device_plane_control_core_v3_reconciliation_backup():
backup_dir = (
BACKUPS_DIR / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID
)
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Device Control Core v3 failed-apply backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Device Control Core v3 failed-apply backup is unsafe")
expected = DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_SHA256
if {child.name for child in backup_dir.iterdir()} != set(expected):
die("Device Control Core v3 failed-apply backup file set mismatch")
for name, expected_sha256 in expected.items():
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected_sha256
):
die(
"Device Control Core v3 failed-apply backup drift detected: "
f"{name}"
)
existing = tuple(read_backup_path_list(
backup_dir / "existing-files.txt"
))
missing = tuple(read_backup_path_list(
backup_dir / "missing-files.txt"
))
validate_backup_partition(
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES,
existing,
missing,
"Device Control Core v3 reconciliation",
)
if (
existing != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING
or missing != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING
):
die("Device Control Core v3 failed-apply backup partition mismatch")
return backup_dir
def validate_device_plane_control_core_v3_restored_source(
backup_dir,
*,
marker_installed,
):
root = component_root("device-plane")
with tempfile.TemporaryDirectory(
prefix="device-control-core-v3-restored-source-",
dir=TMP_DIR,
) as directory:
backup_root = Path(directory)
materialize_backup_tree(
backup_dir / "source-before.tgz",
backup_root,
set(DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING),
)
backup_source = collect_exact_files(
backup_root,
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING,
"Device Control Core v3 pre-apply source",
)
live_source = collect_exact_files(
root,
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_EXISTING,
"Device Control Core v3 restored live source",
)
if live_source != backup_source:
die("Device Control Core v3 rollback source does not match backup")
for rel in DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING:
path = root / rel
if path.exists() or path.is_symlink():
die(
"Device Control Core v3 candidate-only source remains installed: "
f"{rel}"
)
marker = root / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL
if marker_installed:
descriptor = read_strict_json(
marker,
"installed Device Control Core v3 reconciliation descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_control_core_v3_reconciliation_descriptor()
):
die("installed Device Control Core v3 reconciliation mismatch")
elif marker.exists() or marker.is_symlink():
die("Device Control Core v3 reconciliation already installed")
def validate_device_plane_control_core_v3_reconciliation_runtime(
backup_dir,
*,
require_recovered,
):
runtime_before = read_strict_json(
backup_dir / "runtime-before.json",
"Device Control Core v3 pre-apply runtime inventory",
max_bytes=64 * 1024,
)
expected_names = {
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
}
if set(device_plane_inventory_service_names(runtime_before)) != expected_names:
die("Device Control Core v3 pre-apply runtime inventory mismatch")
before = {item["service"]: item for item in runtime_before["services"]}
if (
before["device-control-core"]["imageId"]
!= DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
or before["device-control-core"]["health"] != "healthy"
or before["device-control-core"]["running"] is not True
):
die("Device Control Core v3 pre-apply image evidence mismatch")
current = device_plane_runtime_inventory(tuple(sorted(expected_names)))
if set(device_plane_inventory_service_names(current)) != expected_names:
die("Device Control Core v3 current runtime inventory mismatch")
observed = {item["service"]: item for item in current["services"]}
for service in expected_names - {"device-control-core"}:
expected_item = before[service]
actual_item = observed[service]
if (
actual_item["containerId"] != expected_item["containerId"]
or actual_item["imageId"] != expected_item["imageId"]
or actual_item["running"] is not True
or actual_item["health"] != "healthy"
):
die(
"Device Control Core v3 reconciliation changed preserved service: "
f"{service}"
)
image_id = inspect_optional_local_image(
DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
"Device Control Core v3 exact pre-apply image",
)
if image_id != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID:
die("Device Control Core v3 exact pre-apply image is unavailable")
core = observed["device-control-core"]
if require_recovered:
if (
core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
or core["running"] is not True
or core["health"] != "healthy"
):
die("Device Control Core v3 exact-image recovery did not converge")
validate_device_manager_control_plane_runtime(
require_edge_channel=True,
core_network_mode="private-egress",
)
return {
"before": runtime_before,
"current": current,
"core": core,
}
def validate_device_plane_control_core_v3_reconciliation_evidence(payload_dir):
descriptor = validate_device_plane_control_core_v3_reconciliation_payload(
payload_dir
)
backup_dir = validate_device_plane_control_core_v3_reconciliation_backup()
failed_artifact = FAILED_DIR / DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Control Core v3 failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256
):
die("Device Control Core v3 failed artifact evidence mismatch")
records = [
value
for value in load_state(FAILED_STATE_FILE)
if value.get("id") == DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID
]
if len(records) != 1:
die("Device Control Core v3 failed journal evidence count mismatch")
record = records[0]
if (
record.get("artifact") != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT
or record.get("backup_id")
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID
or record.get("component") != "device-plane"
or record.get("sha256")
!= DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status") != "failed:DeployError"
or record.get("status") != "failed"
or record.get("message")
!= (
"container healthcheck grace exhausted for "
"97513c9a1027ae84f5dad6d0bc4792a67a4b0a1f32665d6d498093f43ee7987b: "
"unhealthy"
)
):
die("Device Control Core v3 failed journal evidence mismatch")
with tempfile.TemporaryDirectory(
prefix="device-control-core-v3-failed-artifact-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, _failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id") != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries) != DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_ENTRIES
):
die("Device Control Core v3 failed artifact contract mismatch")
validate_device_plane_control_core_v3_restored_source(
backup_dir,
marker_installed=False,
)
runtime = validate_device_plane_control_core_v3_reconciliation_runtime(
backup_dir,
require_recovered=False,
)
return {
"mode": descriptor["mode"],
"backup": backup_dir,
"failedArtifact": failed_artifact,
"runtime": runtime,
}
def device_plane_service_container_ids(service):
if service not in (
*DEVICE_PLANE_RUNTIME_SERVICES,
@@ -19322,6 +19689,12 @@ def component_services(component, entries=None):
if is_device_plane_control_core_release_slice(component, entries):
return ("device-control-core",)
if is_device_plane_control_core_v3_reconciliation_slice(
component,
entries,
):
return ("device-control-core",)
if is_device_plane_manager_only_release_slice(component, entries):
return ("device-manager",)
@@ -19922,6 +20295,12 @@ def component_builds(component, entries=None):
),
),)
if is_device_plane_control_core_v3_reconciliation_slice(
component,
entries,
):
return ()
if is_device_plane_manager_only_release_slice(component, entries):
return ((
DEVICE_PLANE_ROOT / "services/device-manager",
@@ -28114,6 +28493,7 @@ def plan_artifact(artifact):
device_plane_control_core_release_preflight = None
device_plane_manager_reconciliation_preflight = None
device_plane_manager_v2_reconciliation_preflight = None
device_plane_control_core_v3_reconciliation_preflight = None
device_plane_backhaul_preflight = None
device_plane_backhaul_vps_enrollment_preflight = None
device_plane_runtime_before = None
@@ -28377,6 +28757,15 @@ def plan_artifact(artifact):
payload_dir
)
)
if is_device_plane_control_core_v3_reconciliation_slice(
manifest["component"],
entries,
):
device_plane_control_core_v3_reconciliation_preflight = (
validate_device_plane_control_core_v3_reconciliation_evidence(
payload_dir
)
)
if is_device_plane_backhaul_target_slice(
manifest["component"],
entries,
@@ -30509,6 +30898,46 @@ def plan_artifact(artifact):
print("device_gateway_tcp_9921=preserved:loopback-only")
print("device_manager_public_route=unchanged:absent")
print("device_plane_rollback=marker-only-runtime-unchanged")
if device_plane_control_core_v3_reconciliation_preflight is not None:
recovery = device_plane_control_core_v3_reconciliation_preflight
current_core = recovery["runtime"]["core"]
print(f"device_plane_transition={recovery['mode']}")
print(
"failed_patch="
f"{DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID}"
)
print(
"failed_artifact_sha256="
f"{DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256}"
)
print(f"recovery_backup={recovery['backup'].name}")
print(
"device_control_core_current_runtime="
f"{current_core['status']}:{current_core['health']}"
)
print(
"device_control_core_recovery_image="
f"{DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID}"
)
print("device_plane_build=none")
print(
"device_plane_runtime_mutation="
"retag-exact-preapply-image+recreate:device-control-core"
)
print(
"device_plane_runtime_services="
"preserved:device-manager,device-gateway,device-postgres,"
"device-backhaul-target"
)
print("device_postgres=preserved:not-selected")
print(
"device_postgres_volume="
"preserved:nodedc-device-plane-postgres-data"
)
print("device_gateway_public_ingress=disabled")
print("device_gateway_tcp_9921=preserved:loopback-only")
print("gelios=untouched-legacy-only")
print("device_plane_rollback=marker+exact-preapply-image-runtime")
if device_plane_backhaul_vps_enrollment_preflight is not None:
print(
"device_plane_transition="
@@ -31116,15 +31545,32 @@ def rollback_device_plane_apply(
if not runtime_started or not baseline_services:
return f"source-restored-runtime-unchanged:{restored_count}"
run_component_runtime(
"device-plane",
baseline_entries,
baseline_services,
)
if is_device_plane_control_core_release_slice(
control_core_release_rollback = is_device_plane_control_core_release_slice(
"device-plane",
entries,
):
)
if control_core_release_rollback:
before = {
item["service"]: item
for item in runtime_inventory["services"]
}
selected_before = before.get("device-control-core") or {}
image_id = selected_before.get("imageId")
if not isinstance(image_id, str):
die("Device Control Core rollback image evidence is missing")
retag_device_plane_control_core_image(
image_id,
"Device Control Core exact pre-apply rollback image",
)
prepare_component_runtime("device-plane", baseline_entries)
run_compose("device-plane", baseline_services, baseline_entries)
else:
run_component_runtime(
"device-plane",
baseline_entries,
baseline_services,
)
if control_core_release_rollback:
accept_device_plane_control_core_rollback_runtime(
runtime_inventory
)
@@ -32360,12 +32806,59 @@ def run_component_runtime(component, entries, services):
run_compose(component, services, entries)
def retag_device_plane_control_core_image(image_id, label):
image_id = inspect_optional_local_image(
image_id,
label,
)
if image_id is None:
die(f"{label} is unavailable")
result = subprocess.run(
[
str(DOCKER),
"image",
"tag",
image_id,
DEVICE_PLANE_CONTROL_CORE_IMAGE,
],
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
die(f"{label} retag failed")
tagged = inspect_optional_local_image(
DEVICE_PLANE_CONTROL_CORE_IMAGE,
f"{label} recovered tag",
)
if tagged != image_id:
die(f"{label} recovered tag mismatch")
def restore_device_plane_control_core_v3_preapply_image():
retag_device_plane_control_core_image(
DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
"Device Control Core v3 exact pre-apply image",
)
def run_device_plane_runtime_for_apply(
entries,
services,
mark_runtime_started,
backhaul_serve_before=None,
):
if is_device_plane_control_core_v3_reconciliation_slice(
"device-plane",
entries,
):
if tuple(services or ()) != ("device-control-core",):
die("Device Control Core v3 reconciliation service set mismatch")
prepare_component_runtime("device-plane", entries)
mark_runtime_started()
restore_device_plane_control_core_v3_preapply_image()
run_compose("device-plane", services, entries)
return
if is_device_plane_b2_discovery_rollback_recovery_slice(
"device-plane",
entries,
@@ -33663,6 +34156,38 @@ def run_healthchecks(component, entries=None, services=None):
accept_gitea_fresh_install_runtime()
return
if is_device_plane_control_core_v3_reconciliation_slice(
component,
entries,
):
if tuple(services or ()) != ("device-control-core",):
die("Device Control Core v3 reconciliation service set mismatch")
for service in (
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
):
healthcheck_compose_service_with_grace(
"device-plane",
service,
)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
recovery_backup = (
validate_device_plane_control_core_v3_reconciliation_backup()
)
validate_device_plane_control_core_v3_restored_source(
recovery_backup,
marker_installed=True,
)
validate_device_plane_control_core_v3_reconciliation_runtime(
recovery_backup,
require_recovered=True,
)
return
if is_device_plane_control_core_release_slice(component, entries):
if tuple(services or ()) != ("device-control-core",):
die("Device Control Core release service set mismatch")
@@ -34793,6 +35318,13 @@ def apply_artifact(artifact):
validate_device_plane_manager_v2_reconciliation_evidence(
payload_dir
)
if is_device_plane_control_core_v3_reconciliation_slice(
component,
entries,
):
validate_device_plane_control_core_v3_reconciliation_evidence(
payload_dir
)
if is_device_plane_backhaul_target_slice(
component,
entries,
@@ -35111,6 +35643,10 @@ def apply_artifact(artifact):
component,
entries,
)
or is_device_plane_control_core_v3_reconciliation_slice(
component,
entries,
)
):
inventory_services = (
*inventory_services,