fix(deploy): recover control core migration replay
This commit is contained in:
@@ -450,6 +450,67 @@ DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES = (
|
|||||||
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID = (
|
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID = (
|
||||||
"device-control-core-incident-audit-20260822-043"
|
"device-control-core-incident-audit-20260822-043"
|
||||||
)
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL = (
|
||||||
|
"deployment/device-control-core-migration-replay-recovery-v1.json"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL = (
|
||||||
|
"services/device-control-core/migrations/"
|
||||||
|
"014_device_registry_profile_commands.sql"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES = (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID = (
|
||||||
|
"device-control-core-migration-replay-recovery-20260822-044"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = (
|
||||||
|
"751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256 = (
|
||||||
|
"38bd86b42828d44c7101d5433ddc36018e92eedeee37b9de296432ad676edd46"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS = (
|
||||||
|
"owner_scope.ensure",
|
||||||
|
"project.ensure",
|
||||||
|
"collection.ensure",
|
||||||
|
"project_grant.upsert",
|
||||||
|
"adapter_package.ensure",
|
||||||
|
"adapter_version.register",
|
||||||
|
"model_profile.register",
|
||||||
|
"edge.ensure",
|
||||||
|
"route.ensure",
|
||||||
|
"enrollment_intent.ensure",
|
||||||
|
"device.claim",
|
||||||
|
"device.update",
|
||||||
|
"device.transfer",
|
||||||
|
"discovery.reject",
|
||||||
|
"discovery.expire",
|
||||||
|
"device_credential_binding.upsert",
|
||||||
|
"device_credential_binding.revoke",
|
||||||
|
"device_binding.ensure",
|
||||||
|
"device_binding.revoke",
|
||||||
|
"device_configuration_revision.create",
|
||||||
|
"device_configuration_desired.set",
|
||||||
|
"asset.ensure",
|
||||||
|
"asset_binding.ensure",
|
||||||
|
"asset_binding.close",
|
||||||
|
"infrastructure_host.ensure",
|
||||||
|
"infrastructure_endpoint.ensure",
|
||||||
|
"infrastructure_deployment.ensure",
|
||||||
|
"infrastructure_service_instance.ensure",
|
||||||
|
"health_observation.record",
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS = (
|
||||||
|
"asset.ensure",
|
||||||
|
"asset_binding.ensure",
|
||||||
|
"asset_binding.close",
|
||||||
|
"infrastructure_host.ensure",
|
||||||
|
"infrastructure_endpoint.ensure",
|
||||||
|
"infrastructure_deployment.ensure",
|
||||||
|
"infrastructure_service_instance.ensure",
|
||||||
|
"health_observation.record",
|
||||||
|
)
|
||||||
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT = (
|
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT = (
|
||||||
"nodedc-device-plane-device-control-core-release-v3-reconciliation-"
|
"nodedc-device-plane-device-control-core-release-v3-reconciliation-"
|
||||||
"20260822-042.tgz.20260822-195448"
|
"20260822-042.tgz.20260822-195448"
|
||||||
@@ -4813,6 +4874,7 @@ def allowed_payload_path(component, rel):
|
|||||||
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
|
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
|
||||||
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
|
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
|
||||||
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
|
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
|
||||||
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
||||||
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
|
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
|
||||||
"packages/device-protocol-contract",
|
"packages/device-protocol-contract",
|
||||||
@@ -11274,6 +11336,23 @@ def load_artifact(artifact, work_dir):
|
|||||||
validate_device_plane_control_core_v3_reconciliation_payload(
|
validate_device_plane_control_core_v3_reconciliation_payload(
|
||||||
payload_dir
|
payload_dir
|
||||||
)
|
)
|
||||||
|
if is_device_plane_control_core_incident_audit_slice(
|
||||||
|
manifest["component"],
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
validate_device_plane_control_core_incident_audit_payload(payload_dir)
|
||||||
|
if is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
manifest["component"],
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if (
|
||||||
|
manifest["id"]
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery patch id mismatch")
|
||||||
|
validate_device_plane_control_core_migration_replay_recovery_payload(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
if is_device_plane_postgres_bootstrap_slice(
|
if is_device_plane_postgres_bootstrap_slice(
|
||||||
manifest["component"],
|
manifest["component"],
|
||||||
entries,
|
entries,
|
||||||
@@ -11931,6 +12010,18 @@ def is_device_plane_control_core_incident_audit_slice(component, entries):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return (
|
||||||
|
component == "device-plane"
|
||||||
|
and entries is not None
|
||||||
|
and tuple(entries)
|
||||||
|
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def is_device_plane_manager_v2_control_plane_slice(component, entries):
|
def is_device_plane_manager_v2_control_plane_slice(component, entries):
|
||||||
return (
|
return (
|
||||||
component == "device-plane"
|
component == "device-plane"
|
||||||
@@ -12670,6 +12761,54 @@ def expected_device_plane_control_core_incident_audit_descriptor():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def expected_device_plane_control_core_migration_replay_recovery_descriptor():
|
||||||
|
return {
|
||||||
|
"schemaVersion": (
|
||||||
|
"nodedc.device-plane."
|
||||||
|
"device-control-core-migration-replay-recovery.v1"
|
||||||
|
),
|
||||||
|
"mode": "double-rollback-failed-migration-014-forward-repair",
|
||||||
|
"failedIncidentAudit": (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID
|
||||||
|
),
|
||||||
|
"sourcePredecessor": {
|
||||||
|
"path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
|
||||||
|
"sha256": (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"sourceTarget": {
|
||||||
|
"path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
|
||||||
|
"sha256": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256,
|
||||||
|
},
|
||||||
|
"rootCause": (
|
||||||
|
"intermediate-command-kind-check-revalidated-historical-receipts"
|
||||||
|
),
|
||||||
|
"repair": "migration-014-add-constraint-not-valid",
|
||||||
|
"databasePreflight": (
|
||||||
|
"all-live-command-kinds-covered-by-final-migration-016"
|
||||||
|
),
|
||||||
|
"databaseRowMutation": "none",
|
||||||
|
"databaseSchemaOutcome": (
|
||||||
|
"final-migration-016-validated-command-kind-check"
|
||||||
|
),
|
||||||
|
"runtimeAction": "build+recreate-device-control-core-only",
|
||||||
|
"runtimePredecessor": "proven-degraded-double-rollback-state",
|
||||||
|
"preservedServices": [
|
||||||
|
"device-manager",
|
||||||
|
"device-gateway",
|
||||||
|
"device-postgres",
|
||||||
|
"device-backhaul-target",
|
||||||
|
],
|
||||||
|
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
|
||||||
|
"publicIngress": "disabled",
|
||||||
|
"edgeChannel": (
|
||||||
|
"core-initiated-pinned-mtls-registered-edges-only"
|
||||||
|
),
|
||||||
|
"rollback": "source+exact-degraded-predecessor-image-runtime",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def validate_platform_device_core_hub_trust_payload(payload_dir):
|
def validate_platform_device_core_hub_trust_payload(payload_dir):
|
||||||
descriptor = read_strict_json(
|
descriptor = read_strict_json(
|
||||||
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
|
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
|
||||||
@@ -13572,6 +13711,41 @@ def validate_device_plane_control_core_incident_audit_payload(payload_dir):
|
|||||||
return descriptor
|
return descriptor
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_migration_replay_recovery_payload(
|
||||||
|
payload_dir,
|
||||||
|
):
|
||||||
|
descriptor = read_strict_json(
|
||||||
|
payload_dir
|
||||||
|
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
|
||||||
|
"Device Control Core migration replay recovery descriptor",
|
||||||
|
max_bytes=16 * 1024,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
descriptor
|
||||||
|
!= expected_device_plane_control_core_migration_replay_recovery_descriptor()
|
||||||
|
):
|
||||||
|
die("Device Control Core migration replay recovery descriptor mismatch")
|
||||||
|
migration = payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
||||||
|
try:
|
||||||
|
migration_stat = migration.lstat()
|
||||||
|
migration_text = migration.read_text(encoding="utf-8")
|
||||||
|
except (FileNotFoundError, OSError, UnicodeDecodeError):
|
||||||
|
die("Device Control Core migration 014 repair is unreadable")
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(migration_stat.st_mode)
|
||||||
|
or not stat.S_ISREG(migration_stat.st_mode)
|
||||||
|
or sha256_file(migration)
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256
|
||||||
|
or ")) not valid;" not in migration_text.lower()
|
||||||
|
or re.search(
|
||||||
|
r"(?im)^\s*(?:delete|update|insert|truncate)\b",
|
||||||
|
migration_text,
|
||||||
|
)
|
||||||
|
):
|
||||||
|
die("Device Control Core migration 014 repair mismatch")
|
||||||
|
return descriptor
|
||||||
|
|
||||||
|
|
||||||
def is_device_plane_postgres_bootstrap_slice(component, entries):
|
def is_device_plane_postgres_bootstrap_slice(component, entries):
|
||||||
return (
|
return (
|
||||||
component == "device-plane"
|
component == "device-plane"
|
||||||
@@ -16417,6 +16591,12 @@ def validate_device_plane_control_core_incident_audit_evidence(payload_dir):
|
|||||||
descriptor = validate_device_plane_control_core_incident_audit_payload(
|
descriptor = validate_device_plane_control_core_incident_audit_payload(
|
||||||
payload_dir
|
payload_dir
|
||||||
)
|
)
|
||||||
|
evidence = validate_device_plane_control_core_double_failure_evidence()
|
||||||
|
evidence["mode"] = descriptor["mode"]
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_double_failure_evidence():
|
||||||
first_backup = validate_device_plane_control_core_v3_reconciliation_backup()
|
first_backup = validate_device_plane_control_core_v3_reconciliation_backup()
|
||||||
first_failed_artifact = (
|
first_failed_artifact = (
|
||||||
FAILED_DIR / DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT
|
FAILED_DIR / DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT
|
||||||
@@ -16570,13 +16750,282 @@ def validate_device_plane_control_core_incident_audit_evidence(payload_dir):
|
|||||||
"device-backhaul-target",
|
"device-backhaul-target",
|
||||||
))
|
))
|
||||||
return {
|
return {
|
||||||
"mode": descriptor["mode"],
|
"mode": "double-rollback-failed-evidence",
|
||||||
"firstBackup": first_backup,
|
"firstBackup": first_backup,
|
||||||
"secondBackup": backup_dir,
|
"secondBackup": backup_dir,
|
||||||
"runtime": current,
|
"runtime": current,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def collect_device_plane_control_core_migration_replay_database_evidence():
|
||||||
|
postgres_ids = device_plane_service_container_ids("device-postgres")
|
||||||
|
if len(postgres_ids) != 1:
|
||||||
|
die("Device Control Core migration recovery PostgreSQL is missing")
|
||||||
|
allowed = ",".join(
|
||||||
|
"'" + value.replace("'", "''") + "'"
|
||||||
|
for value in DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS
|
||||||
|
)
|
||||||
|
triggering = ",".join(
|
||||||
|
"'" + value.replace("'", "''") + "'"
|
||||||
|
for value in DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS
|
||||||
|
)
|
||||||
|
query = f"""
|
||||||
|
with constraint_state as (
|
||||||
|
select convalidated,
|
||||||
|
(select bool_and(pg_get_constraintdef(c.oid) like '%' || kind || '%')
|
||||||
|
from unnest(array[{allowed}]::text[]) as kind) as covers_final
|
||||||
|
from pg_constraint c
|
||||||
|
where c.conrelid = 'public.device_management_command_receipts'::regclass
|
||||||
|
and c.conname = 'device_management_command_receipts_command_kind_check'
|
||||||
|
)
|
||||||
|
select (
|
||||||
|
select count(*)
|
||||||
|
from device_management_command_receipts
|
||||||
|
where not (command_kind = any(array[{allowed}]::text[]))
|
||||||
|
)::text,
|
||||||
|
(
|
||||||
|
select count(*)
|
||||||
|
from device_management_command_receipts
|
||||||
|
where command_kind = any(array[{triggering}]::text[])
|
||||||
|
)::text,
|
||||||
|
coalesce((select convalidated::text from constraint_state), 'missing'),
|
||||||
|
coalesce((select covers_final::text from constraint_state), 'false'),
|
||||||
|
(to_regclass('public.device_infrastructure_host_telemetry_samples')
|
||||||
|
is null)::text
|
||||||
|
""".strip()
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
str(DOCKER),
|
||||||
|
"exec",
|
||||||
|
postgres_ids[0],
|
||||||
|
"psql",
|
||||||
|
"-X",
|
||||||
|
"-qAt",
|
||||||
|
"-v",
|
||||||
|
"ON_ERROR_STOP=1",
|
||||||
|
"-U",
|
||||||
|
"device_plane",
|
||||||
|
"-d",
|
||||||
|
"device_plane",
|
||||||
|
"-F",
|
||||||
|
"\t",
|
||||||
|
"-c",
|
||||||
|
query,
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
values = result.stdout.strip().split("\t")
|
||||||
|
if (
|
||||||
|
result.returncode != 0
|
||||||
|
or result.stderr.strip()
|
||||||
|
or len(values) != 5
|
||||||
|
or not re.fullmatch(r"[0-9]+", values[0])
|
||||||
|
or not re.fullmatch(r"[0-9]+", values[1])
|
||||||
|
or values[2] not in ("true", "false", "missing")
|
||||||
|
or values[3] not in ("true", "false")
|
||||||
|
or values[4] not in ("true", "false")
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery database read failed")
|
||||||
|
evidence = {
|
||||||
|
"invalidCommandKindCount": int(values[0]),
|
||||||
|
"triggeringReceiptCount": int(values[1]),
|
||||||
|
"constraintValidated": values[2] == "true",
|
||||||
|
"constraintCoversFinalKinds": values[3] == "true",
|
||||||
|
"hostTelemetryTableAbsent": values[4] == "true",
|
||||||
|
"query": query,
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
evidence["invalidCommandKindCount"] != 0
|
||||||
|
or evidence["triggeringReceiptCount"] < 1
|
||||||
|
or not evidence["constraintValidated"]
|
||||||
|
or not evidence["constraintCoversFinalKinds"]
|
||||||
|
or not evidence["hostTelemetryTableAbsent"]
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery database invariant mismatch")
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_migration_replay_preserved_runtime(
|
||||||
|
evidence,
|
||||||
|
):
|
||||||
|
expected_services = {
|
||||||
|
"device-control-core",
|
||||||
|
"device-manager",
|
||||||
|
"device-gateway",
|
||||||
|
"device-postgres",
|
||||||
|
"device-backhaul-target",
|
||||||
|
}
|
||||||
|
current = device_plane_runtime_inventory(tuple(sorted(expected_services)))
|
||||||
|
if set(device_plane_inventory_service_names(current)) != expected_services:
|
||||||
|
die("Device Control Core migration recovery runtime is incomplete")
|
||||||
|
second_before = read_strict_json(
|
||||||
|
evidence["secondBackup"] / "runtime-before.json",
|
||||||
|
"Device Control Core second failed-apply runtime inventory",
|
||||||
|
max_bytes=64 * 1024,
|
||||||
|
)
|
||||||
|
if set(device_plane_inventory_service_names(second_before)) != expected_services:
|
||||||
|
die("Device Control Core second failed-apply runtime evidence mismatch")
|
||||||
|
before = {item["service"]: item for item in second_before["services"]}
|
||||||
|
observed = {item["service"]: item for item in current["services"]}
|
||||||
|
for service in expected_services - {"device-control-core"}:
|
||||||
|
if (
|
||||||
|
observed[service]["containerId"] != before[service]["containerId"]
|
||||||
|
or observed[service]["imageId"] != before[service]["imageId"]
|
||||||
|
or observed[service]["status"] != "running"
|
||||||
|
or observed[service]["running"] is not True
|
||||||
|
or observed[service]["health"] != "healthy"
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration recovery changed preserved "
|
||||||
|
f"service: {service}"
|
||||||
|
)
|
||||||
|
return {"current": current, "core": observed["device-control-core"]}
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_migration_replay_recovery_evidence(
|
||||||
|
payload_dir,
|
||||||
|
):
|
||||||
|
descriptor = (
|
||||||
|
validate_device_plane_control_core_migration_replay_recovery_payload(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
|
)
|
||||||
|
evidence = validate_device_plane_control_core_double_failure_evidence()
|
||||||
|
root = component_root("device-plane")
|
||||||
|
live_migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
||||||
|
if (
|
||||||
|
live_migration.is_symlink()
|
||||||
|
or not live_migration.is_file()
|
||||||
|
or sha256_file(live_migration)
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256
|
||||||
|
):
|
||||||
|
die("Device Control Core migration 014 predecessor mismatch")
|
||||||
|
installed_descriptor = (
|
||||||
|
root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL
|
||||||
|
)
|
||||||
|
if installed_descriptor.exists() or installed_descriptor.is_symlink():
|
||||||
|
die("Device Control Core migration replay recovery already installed")
|
||||||
|
runtime = (
|
||||||
|
validate_device_plane_control_core_migration_replay_preserved_runtime(
|
||||||
|
evidence
|
||||||
|
)
|
||||||
|
)
|
||||||
|
core = runtime["core"]
|
||||||
|
if (
|
||||||
|
core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
|
||||||
|
or core["status"] not in ("running", "restarting", "exited")
|
||||||
|
or core["health"] not in ("starting", "unhealthy", None)
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery predecessor mismatch")
|
||||||
|
image_id = inspect_optional_local_image(
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
|
||||||
|
"Device Control Core migration recovery exact predecessor image",
|
||||||
|
)
|
||||||
|
if image_id != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID:
|
||||||
|
die("Device Control Core migration recovery predecessor image is missing")
|
||||||
|
database = (
|
||||||
|
collect_device_plane_control_core_migration_replay_database_evidence()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"mode": descriptor["mode"],
|
||||||
|
"descriptor": descriptor,
|
||||||
|
"firstBackup": evidence["firstBackup"],
|
||||||
|
"secondBackup": evidence["secondBackup"],
|
||||||
|
"runtime": runtime["current"],
|
||||||
|
"core": core,
|
||||||
|
"database": database,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def accept_device_plane_control_core_migration_replay_recovery():
|
||||||
|
root = component_root("device-plane")
|
||||||
|
migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
||||||
|
if (
|
||||||
|
migration.is_symlink()
|
||||||
|
or not migration.is_file()
|
||||||
|
or sha256_file(migration)
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256
|
||||||
|
):
|
||||||
|
die("installed Device Control Core migration 014 repair mismatch")
|
||||||
|
descriptor = read_strict_json(
|
||||||
|
root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
|
||||||
|
"installed Device Control Core migration replay recovery descriptor",
|
||||||
|
max_bytes=16 * 1024,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
descriptor
|
||||||
|
!= expected_device_plane_control_core_migration_replay_recovery_descriptor()
|
||||||
|
):
|
||||||
|
die("installed Device Control Core migration recovery mismatch")
|
||||||
|
evidence = {
|
||||||
|
"secondBackup": (
|
||||||
|
BACKUPS_DIR
|
||||||
|
/ DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID
|
||||||
|
)
|
||||||
|
}
|
||||||
|
runtime = (
|
||||||
|
validate_device_plane_control_core_migration_replay_preserved_runtime(
|
||||||
|
evidence
|
||||||
|
)
|
||||||
|
)
|
||||||
|
core = runtime["core"]
|
||||||
|
if (
|
||||||
|
core["imageId"] == DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
|
||||||
|
or core["status"] != "running"
|
||||||
|
or core["running"] is not True
|
||||||
|
or core["health"] != "healthy"
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery did not converge")
|
||||||
|
database = (
|
||||||
|
collect_device_plane_control_core_migration_replay_database_evidence()
|
||||||
|
)
|
||||||
|
validate_device_manager_control_plane_runtime(
|
||||||
|
require_edge_channel=True,
|
||||||
|
core_network_mode="private-egress",
|
||||||
|
)
|
||||||
|
return {"runtime": runtime["current"], "database": database}
|
||||||
|
|
||||||
|
|
||||||
|
def emit_bounded_device_control_core_failure_logs():
|
||||||
|
core_ids = device_plane_service_container_ids("device-control-core")
|
||||||
|
if len(core_ids) != 1:
|
||||||
|
return
|
||||||
|
result = subprocess.run(
|
||||||
|
[str(DOCKER), "logs", "--tail", "160", core_ids[0]],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return
|
||||||
|
selected = []
|
||||||
|
for raw_line in f"{result.stdout}\n{result.stderr}".splitlines():
|
||||||
|
line = re.sub(r"\s+", " ", raw_line.strip())
|
||||||
|
if not line or not re.search(
|
||||||
|
r"(?:error|exception|failed|constraint|migration|postgres|relation)",
|
||||||
|
line,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
if re.search(
|
||||||
|
r"(?:authorization|bearer|password|private[_ -]?key|token=|secret=)",
|
||||||
|
line,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
line = "[sensitive log line redacted]"
|
||||||
|
if line not in selected:
|
||||||
|
selected.append(line[:480])
|
||||||
|
for index, line in enumerate(selected[-16:], start=1):
|
||||||
|
print(
|
||||||
|
f"device_control_core_failure_log_{index:02d}={line}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def collect_device_plane_control_core_incident_audit():
|
def collect_device_plane_control_core_incident_audit():
|
||||||
core_ids = device_plane_service_container_ids("device-control-core")
|
core_ids = device_plane_service_container_ids("device-control-core")
|
||||||
postgres_ids = device_plane_service_container_ids("device-postgres")
|
postgres_ids = device_plane_service_container_ids("device-postgres")
|
||||||
@@ -16849,6 +17298,25 @@ def accept_device_plane_control_core_rollback_runtime(runtime_before):
|
|||||||
for service in expected_services[1:]:
|
for service in expected_services[1:]:
|
||||||
healthcheck_compose_service_with_grace("device-plane", service)
|
healthcheck_compose_service_with_grace("device-plane", service)
|
||||||
|
|
||||||
|
preserved = device_plane_runtime_inventory(expected_services[1:])
|
||||||
|
if set(device_plane_inventory_service_names(preserved)) != set(
|
||||||
|
expected_services[1:]
|
||||||
|
):
|
||||||
|
die("Device Control Core rollback preserved runtime is incomplete")
|
||||||
|
for item in preserved["services"]:
|
||||||
|
expected = before[item["service"]]
|
||||||
|
if (
|
||||||
|
item["containerId"] != expected["containerId"]
|
||||||
|
or item["imageId"] != expected["imageId"]
|
||||||
|
or item["status"] != "running"
|
||||||
|
or item["running"] is not True
|
||||||
|
or item["health"] != "healthy"
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core rollback changed preserved service: "
|
||||||
|
f"{item['service']}"
|
||||||
|
)
|
||||||
|
|
||||||
container_id = compose_service_container_id(
|
container_id = compose_service_container_id(
|
||||||
"device-plane",
|
"device-plane",
|
||||||
"device-control-core",
|
"device-control-core",
|
||||||
@@ -20079,6 +20547,12 @@ def component_services(component, entries=None):
|
|||||||
if is_device_plane_control_core_incident_audit_slice(component, entries):
|
if is_device_plane_control_core_incident_audit_slice(component, entries):
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
|
if is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return ("device-control-core",)
|
||||||
|
|
||||||
if is_device_plane_manager_only_release_slice(component, entries):
|
if is_device_plane_manager_only_release_slice(component, entries):
|
||||||
return ("device-manager",)
|
return ("device-manager",)
|
||||||
|
|
||||||
@@ -20688,6 +21162,24 @@ def component_builds(component, entries=None):
|
|||||||
if is_device_plane_control_core_incident_audit_slice(component, entries):
|
if is_device_plane_control_core_incident_audit_slice(component, entries):
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
|
if is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return ((
|
||||||
|
DEVICE_PLANE_ROOT,
|
||||||
|
(
|
||||||
|
"build",
|
||||||
|
"--no-cache",
|
||||||
|
"--network=host",
|
||||||
|
"-f",
|
||||||
|
"services/device-control-core/Dockerfile",
|
||||||
|
"-t",
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_IMAGE,
|
||||||
|
".",
|
||||||
|
),
|
||||||
|
),)
|
||||||
|
|
||||||
if is_device_plane_manager_only_release_slice(component, entries):
|
if is_device_plane_manager_only_release_slice(component, entries):
|
||||||
return ((
|
return ((
|
||||||
DEVICE_PLANE_ROOT / "services/device-manager",
|
DEVICE_PLANE_ROOT / "services/device-manager",
|
||||||
@@ -28882,6 +29374,7 @@ def plan_artifact(artifact):
|
|||||||
device_plane_manager_v2_reconciliation_preflight = None
|
device_plane_manager_v2_reconciliation_preflight = None
|
||||||
device_plane_control_core_v3_reconciliation_preflight = None
|
device_plane_control_core_v3_reconciliation_preflight = None
|
||||||
device_plane_control_core_incident_audit_preflight = None
|
device_plane_control_core_incident_audit_preflight = None
|
||||||
|
device_plane_control_core_migration_replay_recovery_preflight = None
|
||||||
device_plane_backhaul_preflight = None
|
device_plane_backhaul_preflight = None
|
||||||
device_plane_backhaul_vps_enrollment_preflight = None
|
device_plane_backhaul_vps_enrollment_preflight = None
|
||||||
device_plane_runtime_before = None
|
device_plane_runtime_before = None
|
||||||
@@ -29165,6 +29658,20 @@ def plan_artifact(artifact):
|
|||||||
collect_device_plane_control_core_incident_audit()
|
collect_device_plane_control_core_incident_audit()
|
||||||
)
|
)
|
||||||
device_plane_control_core_incident_audit_preflight = incident
|
device_plane_control_core_incident_audit_preflight = incident
|
||||||
|
if is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
manifest["component"],
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if (
|
||||||
|
manifest["id"]
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery patch id mismatch")
|
||||||
|
device_plane_control_core_migration_replay_recovery_preflight = (
|
||||||
|
validate_device_plane_control_core_migration_replay_recovery_evidence(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
|
)
|
||||||
if is_device_plane_backhaul_target_slice(
|
if is_device_plane_backhaul_target_slice(
|
||||||
manifest["component"],
|
manifest["component"],
|
||||||
entries,
|
entries,
|
||||||
@@ -31383,6 +31890,65 @@ def plan_artifact(artifact):
|
|||||||
"read-only:device-control-core,device-manager,device-gateway,"
|
"read-only:device-control-core,device-manager,device-gateway,"
|
||||||
"device-postgres,device-backhaul-target"
|
"device-postgres,device-backhaul-target"
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
device_plane_control_core_migration_replay_recovery_preflight
|
||||||
|
is not None
|
||||||
|
):
|
||||||
|
recovery = (
|
||||||
|
device_plane_control_core_migration_replay_recovery_preflight
|
||||||
|
)
|
||||||
|
database = recovery["database"]
|
||||||
|
core = recovery["core"]
|
||||||
|
print(f"device_plane_transition={recovery['mode']}")
|
||||||
|
print(
|
||||||
|
"device_control_core_root_cause="
|
||||||
|
"migration-014-intermediate-constraint-revalidated-"
|
||||||
|
"newer-valid-receipts"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_migration_predecessor_sha256="
|
||||||
|
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_migration_target_sha256="
|
||||||
|
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_current_runtime="
|
||||||
|
f"{core['containerId']}:{core['imageId']}:"
|
||||||
|
f"{core['status']}:{core['health']}:"
|
||||||
|
f"restarts={core['restartCount']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_live_command_kind_incompatible_count="
|
||||||
|
f"{database['invalidCommandKindCount']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_triggering_newer_receipt_count="
|
||||||
|
f"{database['triggeringReceiptCount']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_final_constraint="
|
||||||
|
"validated:covers-migration-016-command-kinds"
|
||||||
|
)
|
||||||
|
print("device_postgres_row_mutation=none")
|
||||||
|
print("device_postgres_telemetry_table=absent")
|
||||||
|
print(
|
||||||
|
"device_plane_runtime_mutation="
|
||||||
|
"build+recreate:device-control-core"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_plane_runtime_services="
|
||||||
|
"preserved:device-manager,device-gateway,device-postgres,"
|
||||||
|
"device-backhaul-target"
|
||||||
|
)
|
||||||
|
print("device_gateway_public_ingress=disabled")
|
||||||
|
print("device_gateway_tcp_9921=preserved:loopback-only")
|
||||||
|
print("gelios=untouched-legacy-only")
|
||||||
|
print(
|
||||||
|
"device_plane_rollback="
|
||||||
|
"source+exact-degraded-predecessor-image-runtime"
|
||||||
|
)
|
||||||
if device_plane_backhaul_vps_enrollment_preflight is not None:
|
if device_plane_backhaul_vps_enrollment_preflight is not None:
|
||||||
print(
|
print(
|
||||||
"device_plane_transition="
|
"device_plane_transition="
|
||||||
@@ -31990,9 +32556,15 @@ def rollback_device_plane_apply(
|
|||||||
if not runtime_started or not baseline_services:
|
if not runtime_started or not baseline_services:
|
||||||
return f"source-restored-runtime-unchanged:{restored_count}"
|
return f"source-restored-runtime-unchanged:{restored_count}"
|
||||||
|
|
||||||
control_core_release_rollback = is_device_plane_control_core_release_slice(
|
control_core_release_rollback = (
|
||||||
"device-plane",
|
is_device_plane_control_core_release_slice(
|
||||||
entries,
|
"device-plane",
|
||||||
|
entries,
|
||||||
|
)
|
||||||
|
or is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
"device-plane",
|
||||||
|
entries,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if control_core_release_rollback:
|
if control_core_release_rollback:
|
||||||
before = {
|
before = {
|
||||||
@@ -33598,6 +34170,21 @@ def component_healthchecks(component, entries=None, services=None):
|
|||||||
"deviceCoreInternalAccessConfigured": True,
|
"deviceCoreInternalAccessConfigured": True,
|
||||||
},
|
},
|
||||||
},)
|
},)
|
||||||
|
if is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return ({
|
||||||
|
"url": "http://127.0.0.1:18120/healthz",
|
||||||
|
"expected_json": {
|
||||||
|
"ok": True,
|
||||||
|
"service": "nodedc-device-control-core",
|
||||||
|
"database": "ready",
|
||||||
|
"discoveryIngest": "enabled",
|
||||||
|
"managementApi": "enabled",
|
||||||
|
"commandTransport": "typed-service-ping-v1",
|
||||||
|
},
|
||||||
|
},)
|
||||||
if is_device_plane_manager_control_plane_slice(component, entries):
|
if is_device_plane_manager_control_plane_slice(component, entries):
|
||||||
command_transport = (
|
command_transport = (
|
||||||
"typed-service-ping-v1"
|
"typed-service-ping-v1"
|
||||||
@@ -34633,6 +35220,35 @@ def run_healthchecks(component, entries=None, services=None):
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if tuple(services or ()) != ("device-control-core",):
|
||||||
|
die("Device Control Core migration recovery service set mismatch")
|
||||||
|
try:
|
||||||
|
healthcheck_compose_service_with_grace(
|
||||||
|
"device-plane",
|
||||||
|
"device-control-core",
|
||||||
|
)
|
||||||
|
except DeployError:
|
||||||
|
emit_bounded_device_control_core_failure_logs()
|
||||||
|
raise
|
||||||
|
for service in (
|
||||||
|
"device-manager",
|
||||||
|
"device-gateway",
|
||||||
|
"device-postgres",
|
||||||
|
"device-backhaul-target",
|
||||||
|
):
|
||||||
|
healthcheck_compose_service_with_grace(
|
||||||
|
"device-plane",
|
||||||
|
service,
|
||||||
|
)
|
||||||
|
for check in component_healthchecks(component, entries, services):
|
||||||
|
healthcheck_url(check)
|
||||||
|
accept_device_plane_control_core_migration_replay_recovery()
|
||||||
|
return
|
||||||
|
|
||||||
if is_device_plane_control_core_release_slice(component, entries):
|
if is_device_plane_control_core_release_slice(component, entries):
|
||||||
if tuple(services or ()) != ("device-control-core",):
|
if tuple(services or ()) != ("device-control-core",):
|
||||||
die("Device Control Core release service set mismatch")
|
die("Device Control Core release service set mismatch")
|
||||||
@@ -35782,6 +36398,21 @@ def apply_artifact(artifact):
|
|||||||
validate_device_plane_control_core_v3_reconciliation_evidence(
|
validate_device_plane_control_core_v3_reconciliation_evidence(
|
||||||
payload_dir
|
payload_dir
|
||||||
)
|
)
|
||||||
|
if is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if (
|
||||||
|
patch_id
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration recovery patch id "
|
||||||
|
"mismatch"
|
||||||
|
)
|
||||||
|
validate_device_plane_control_core_migration_replay_recovery_evidence(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
if is_device_plane_backhaul_target_slice(
|
if is_device_plane_backhaul_target_slice(
|
||||||
component,
|
component,
|
||||||
entries,
|
entries,
|
||||||
@@ -36104,6 +36735,10 @@ def apply_artifact(artifact):
|
|||||||
component,
|
component,
|
||||||
entries,
|
entries,
|
||||||
)
|
)
|
||||||
|
or is_device_plane_control_core_migration_replay_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
)
|
||||||
):
|
):
|
||||||
inventory_services = (
|
inventory_services = (
|
||||||
*inventory_services,
|
*inventory_services,
|
||||||
|
|||||||
Reference in New Issue
Block a user