SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(deploy): accept exact replay checkpoints

Browse Source
This commit is contained in:
Codex
2026-08-22 23:58:17 +03:00
parent c3bd8023d6
commit 680ba0285e
1 changed files with 657 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+657 -21
View File
@@ -480,6 +480,31 @@ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT = (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 = (
"b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT = (
"nodedc-device-plane-device-control-core-migration-replay-recovery-"
"20260822-044.tgz.20260822-232936"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT = (
"2026-08-22T20:29:59+00:00"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL = (
"deployment/"
"device-control-core-migration-replay-checkpoint-recovery-v2.json"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES = (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID = (
"device-control-core-migration-replay-checkpoint-recovery-20260822-046"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT = (
"nodedc-device-plane-device-control-core-migration-replay-checkpoint-"
"recovery-20260822-046.tgz"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT_SHA256 = (
"46000c76977fb583fc7c9cf74ecf624efd8b404f7b8d0322e0270e7b8ac6e450"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = (
"751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
)
@@ -517,6 +542,30 @@ DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS = (
"infrastructure_service_instance.ensure",
"health_observation.record",
)
DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS = (
"owner_scope.ensure",
"project.ensure",
"collection.ensure",
"project_grant.upsert",
"adapter_package.ensure",
"adapter_version.register",
"model_profile.register",
"edge.ensure",
"route.ensure",
"enrollment_intent.ensure",
)
DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS = (
*DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS,
"device.claim",
"device.transfer",
"discovery.reject",
"discovery.expire",
)
DEVICE_PLANE_CONTROL_CORE_REPLAY_009_COMMAND_KINDS = (
*DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS,
"device_credential_binding.upsert",
"device_credential_binding.revoke",
)
DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS = (
"owner_scope.ensure",
"project.ensure",
@@ -539,6 +588,15 @@ DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS = (
"device_configuration_revision.create",
"device_configuration_desired.set",
)
DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS = (
("replay-005", DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS),
("replay-007", DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS),
("replay-009", DEVICE_PLANE_CONTROL_CORE_REPLAY_009_COMMAND_KINDS),
("replay-011", DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS),
)
DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES = tuple(
phase for phase, _kinds in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS = (
"asset.ensure",
"asset_binding.ensure",
@@ -4914,6 +4972,7 @@ def allowed_payload_path(component, rel):
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL,
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
"packages/device-protocol-contract",
@@ -11404,6 +11463,21 @@ def load_artifact(artifact, work_dir):
validate_device_plane_control_core_migration_replay_recovery_payload(
payload_dir
)
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
manifest["component"],
entries,
):
if (
manifest["id"]
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID
):
die(
"Device Control Core migration replay checkpoint recovery "
"patch id mismatch"
)
validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload(
payload_dir
)
if is_device_plane_postgres_bootstrap_slice(
manifest["component"],
entries,
@@ -12085,6 +12159,18 @@ def is_device_plane_control_core_migration_replay_recovery_slice(
)
def is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
component,
entries,
):
return (
component == "device-plane"
and entries is not None
and tuple(entries)
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES
)
def is_device_plane_manager_v2_control_plane_slice(component, entries):
return (
component == "device-plane"
@@ -12911,6 +12997,66 @@ def expected_device_plane_control_core_migration_replay_recovery_descriptor():
}
def expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane."
"device-control-core-migration-replay-checkpoint-recovery.v2"
),
"mode": "terminal-044-replay-checkpoint-forward-repair",
"failedIncidentAudit": (
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID
),
"failedRecovery": {
"patchId": (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
),
"artifactSha256": (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
),
"failure": "preflight-replay-checkpoint-race",
"startedApply": False,
},
"sourcePredecessor": {
"path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
"sha256": (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256
),
},
"sourceTarget": {
"path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
"sha256": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256,
},
"rootCause": (
"restarting-core-cycles-exact-committed-migration-checkpoints"
),
"repair": "migration-014-add-constraint-not-valid",
"databasePreflight": (
"exact-replay-checkpoint-005-007-009-011-and-final-compatible-rows"
),
"databaseRowMutation": "none",
"databaseSchemaOutcome": (
"exact-final-migration-016-validated-command-kind-check"
),
"runtimeAction": "build+recreate-device-control-core-only",
"runtimePredecessor": (
"proven-degraded-restarting-exact-preapply-image"
),
"preservedServices": [
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
"publicIngress": "disabled",
"edgeChannel": (
"core-initiated-pinned-mtls-registered-edges-only"
),
"rollback": "source+exact-degraded-predecessor-image-runtime",
}
def validate_platform_device_core_hub_trust_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
@@ -13843,6 +13989,32 @@ def validate_device_plane_control_core_migration_replay_recovery_payload(
!= expected_device_plane_control_core_migration_replay_recovery_descriptor()
):
die("Device Control Core migration replay recovery descriptor mismatch")
validate_device_plane_control_core_migration_014_repair(payload_dir)
return descriptor
def validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload(
payload_dir,
):
descriptor = read_strict_json(
payload_dir
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
"Device Control Core migration replay checkpoint recovery descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor()
):
die(
"Device Control Core migration replay checkpoint recovery "
"descriptor mismatch"
)
validate_device_plane_control_core_migration_014_repair(payload_dir)
return descriptor
def validate_device_plane_control_core_migration_014_repair(payload_dir):
migration = payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
try:
migration_stat = migration.lstat()
@@ -13861,7 +14033,6 @@ def validate_device_plane_control_core_migration_replay_recovery_payload(
)
):
die("Device Control Core migration 014 repair mismatch")
return descriptor
def is_device_plane_postgres_bootstrap_slice(component, entries):
@@ -16875,6 +17046,79 @@ def validate_device_plane_control_core_double_failure_evidence():
}
def validate_device_plane_control_core_migration_replay_recovery_failure():
failed_artifact = (
FAILED_DIR
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT
)
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Control Core migration recovery 044 failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
):
die("Device Control Core migration recovery 044 failed artifact mismatch")
if any(
value.get("id")
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
for value in load_state(STATE_FILE)
):
die("Device Control Core migration recovery 044 applied journal conflict")
records = [
value
for value in load_state(FAILED_STATE_FILE)
if value.get("id")
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
]
if len(records) != 1:
die("Device Control Core migration recovery 044 failed journal count mismatch")
record = records[0]
if (
record.get("artifact")
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT
or record.get("backup_id") is not None
or record.get("component") != "device-plane"
or record.get("failed_at")
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT
or record.get("sha256")
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
or record.get("started_apply") is not False
or record.get("rollback_status") != "not-required"
or record.get("status") != "failed"
or record.get("message")
!= "Device Control Core migration recovery database invariant mismatch"
):
die("Device Control Core migration recovery 044 failed journal mismatch")
with tempfile.TemporaryDirectory(
prefix="device-control-core-migration-recovery-044-failed-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES
):
die(
"Device Control Core migration recovery 044 failed artifact "
"contract mismatch"
)
validate_device_plane_control_core_migration_replay_recovery_payload(
failed_payload
)
return {"artifact": failed_artifact, "record": record}
def device_plane_control_core_migration_replay_database_invariants_match(
evidence,
):
@@ -16883,6 +17127,7 @@ def device_plane_control_core_migration_replay_database_invariants_match(
and evidence["triggeringReceiptCount"] >= 1
and evidence["constraintValidated"]
and evidence["constraintCoversFinalKinds"]
and evidence["constraintPhase"] == "final-016"
and evidence["hostTelemetryTableAbsent"]
)
@@ -16900,6 +17145,20 @@ def device_plane_control_core_migration_replay_predecessor_invariants_match(
)
def device_plane_control_core_migration_replay_checkpoint_invariants_match(
evidence,
):
return (
evidence["invalidCommandKindCount"] == 0
and evidence["triggeringReceiptCount"] >= 1
and evidence["constraintValidationState"] == "false"
and not evidence["constraintCoversFinalKinds"]
and evidence["constraintPhase"]
in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES
and evidence["hostTelemetryTableAbsent"]
)
def emit_device_plane_control_core_migration_replay_database_evidence(
evidence,
):
@@ -16930,10 +17189,26 @@ def emit_device_plane_control_core_migration_replay_database_evidence(
"device_control_core_constraint_matches_replay_011_kinds="
f"{boolean(evidence['constraintMatchesReplay011Kinds'])}"
)
print(
"device_control_core_constraint_phase="
f"{evidence['constraintPhase']}"
)
print(
"device_control_core_constraint_matches_known_replay_checkpoint="
f"{boolean(evidence['constraintMatchesKnownReplayCheckpoint'])}"
)
print(
"device_control_core_constraint_matches_final_016_kinds="
f"{boolean(evidence['constraintMatchesFinalKinds'])}"
)
print(
"device_control_core_recovery_044_ready="
f"{boolean(evidence['recovery044Ready'])}"
)
print(
"device_control_core_checkpoint_recovery_ready="
f"{boolean(evidence['checkpointRecoveryReady'])}"
)
print(
"device_control_core_recovery_final_state_ready="
f"{boolean(evidence['finalStateReady'])}"
@@ -16944,7 +17219,11 @@ def collect_device_plane_control_core_migration_replay_database_evidence(
enforce_recovery_invariants=True,
expected_state="final",
):
if expected_state not in ("final", "replay-011-predecessor"):
if expected_state not in (
"final",
"replay-011-predecessor",
"replay-checkpoint-predecessor",
):
die("Device Control Core migration recovery database state is invalid")
postgres_ids = device_plane_service_container_ids("device-postgres")
if len(postgres_ids) != 1:
@@ -16957,10 +17236,6 @@ def collect_device_plane_control_core_migration_replay_database_evidence(
"'" + value.replace("'", "''") + "'"
for value in DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS
)
replay_011 = ",".join(
"'" + value.replace("'", "''") + "'"
for value in DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS
)
query = f"""
with constraint_state as (
select convalidated,
@@ -16979,10 +17254,6 @@ constraint_literals as (
'''([^'']+)''',
'g'
) as matched(value)
),
expected_replay_011 as (
select array_agg(kind order by kind) as values
from unnest(array[{replay_011}]::text[]) as expected(kind)
)
select (
select count(*)
@@ -16998,11 +17269,8 @@ select (
coalesce((select covers_final::text from constraint_state), 'false'),
(to_regclass('public.device_infrastructure_host_telemetry_samples')
is null)::text,
coalesce((
select constraint_literals.values = expected_replay_011.values
from constraint_literals
cross join expected_replay_011
), false)::text
coalesce((select array_to_json(values)::text from constraint_literals),
'[]')
""".strip()
result = subprocess.run(
[
@@ -17038,16 +17306,55 @@ select (
or values[2] not in ("true", "false", "missing")
or values[3] not in ("true", "false")
or values[4] not in ("true", "false")
or values[5] not in ("true", "false")
):
die("Device Control Core migration recovery database read failed")
try:
constraint_literal_kinds = json.loads(values[5])
except (TypeError, ValueError):
die("Device Control Core migration recovery constraint read failed")
if (
not isinstance(constraint_literal_kinds, list)
or len(constraint_literal_kinds) > 128
or any(
not isinstance(value, str)
or not re.fullmatch(r"[a-z][a-z0-9._-]{1,127}", value)
for value in constraint_literal_kinds
)
or len(set(constraint_literal_kinds)) != len(constraint_literal_kinds)
):
die("Device Control Core migration recovery constraint read failed")
constraint_literal_kinds = tuple(sorted(constraint_literal_kinds))
constraint_phase_by_kinds = {
tuple(sorted(kinds)): phase
for phase, kinds in (
*DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS,
("final-016", DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS),
)
}
if values[2] == "missing":
constraint_phase = "missing"
else:
constraint_phase = constraint_phase_by_kinds.get(
constraint_literal_kinds,
"unknown",
)
evidence = {
"invalidCommandKindCount": int(values[0]),
"triggeringReceiptCount": int(values[1]),
"constraintValidationState": values[2],
"constraintValidated": values[2] == "true",
"constraintCoversFinalKinds": values[3] == "true",
"hostTelemetryTableAbsent": values[4] == "true",
"constraintMatchesReplay011Kinds": values[5] == "true",
"constraintLiteralKinds": constraint_literal_kinds,
"constraintPhase": constraint_phase,
"constraintMatchesReplay011Kinds": (
constraint_phase == "replay-011"
),
"constraintMatchesKnownReplayCheckpoint": (
constraint_phase
in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES
),
"constraintMatchesFinalKinds": constraint_phase == "final-016",
"query": query,
}
evidence["finalStateReady"] = (
@@ -17060,11 +17367,18 @@ select (
evidence
)
)
expected_ready = (
evidence["recovery044Ready"]
if expected_state == "replay-011-predecessor"
else evidence["finalStateReady"]
evidence["checkpointRecoveryReady"] = (
device_plane_control_core_migration_replay_checkpoint_invariants_match(
evidence
)
)
expected_ready = {
"final": evidence["finalStateReady"],
"replay-011-predecessor": evidence["recovery044Ready"],
"replay-checkpoint-predecessor": (
evidence["checkpointRecoveryReady"]
),
}[expected_state]
if enforce_recovery_invariants and not expected_ready:
emit_device_plane_control_core_migration_replay_database_evidence(
evidence
@@ -17265,6 +17579,78 @@ def validate_device_plane_control_core_migration_replay_recovery_evidence(
}
def validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
payload_dir,
):
descriptor = (
validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload(
payload_dir
)
)
evidence = validate_device_plane_control_core_double_failure_evidence()
failed_recovery = (
validate_device_plane_control_core_migration_replay_recovery_failure()
)
root = component_root("device-plane")
live_migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
if (
live_migration.is_symlink()
or not live_migration.is_file()
or sha256_file(live_migration)
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256
):
die("Device Control Core migration 014 predecessor mismatch")
for descriptor_rel in (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
):
installed_descriptor = root / descriptor_rel
if installed_descriptor.exists() or installed_descriptor.is_symlink():
die(
"Device Control Core migration replay checkpoint recovery "
"already installed"
)
runtime = (
validate_device_plane_control_core_migration_replay_preserved_runtime(
evidence
)
)
core = runtime["core"]
if (
core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
or core["status"] not in ("running", "restarting", "exited")
or core["health"] not in ("starting", "unhealthy", None)
):
die(
"Device Control Core migration replay checkpoint predecessor "
"mismatch"
)
image_id = inspect_optional_local_image(
DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
"Device Control Core migration replay checkpoint exact predecessor image",
)
if image_id != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID:
die(
"Device Control Core migration replay checkpoint predecessor "
"image is missing"
)
database = (
collect_device_plane_control_core_migration_replay_database_evidence(
expected_state="replay-checkpoint-predecessor",
)
)
return {
"mode": descriptor["mode"],
"descriptor": descriptor,
"firstBackup": evidence["firstBackup"],
"secondBackup": evidence["secondBackup"],
"failedRecovery": failed_recovery,
"runtime": runtime["current"],
"core": core,
"database": database,
}
def accept_device_plane_control_core_migration_replay_recovery():
root = component_root("device-plane")
migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
@@ -17316,6 +17702,68 @@ def accept_device_plane_control_core_migration_replay_recovery():
return {"runtime": runtime["current"], "database": database}
def accept_device_plane_control_core_migration_replay_checkpoint_recovery():
root = component_root("device-plane")
migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
if (
migration.is_symlink()
or not migration.is_file()
or sha256_file(migration)
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256
):
die("installed Device Control Core migration 014 repair mismatch")
descriptor = read_strict_json(
root
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
"installed Device Control Core replay checkpoint recovery descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor()
):
die(
"installed Device Control Core replay checkpoint recovery mismatch"
)
legacy_descriptor = (
root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL
)
if legacy_descriptor.exists() or legacy_descriptor.is_symlink():
die("legacy Device Control Core migration recovery descriptor appeared")
evidence = {
"secondBackup": (
BACKUPS_DIR
/ DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID
)
}
runtime = (
validate_device_plane_control_core_migration_replay_preserved_runtime(
evidence
)
)
core = runtime["core"]
if (
core["imageId"] == DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
or core["status"] != "running"
or core["running"] is not True
or core["health"] != "healthy"
):
die(
"Device Control Core migration replay checkpoint recovery did "
"not converge"
)
database = (
collect_device_plane_control_core_migration_replay_database_evidence(
expected_state="final",
)
)
validate_device_manager_control_plane_runtime(
require_edge_channel=True,
core_network_mode="private-egress",
)
return {"runtime": runtime["current"], "database": database}
def emit_bounded_device_control_core_failure_logs():
core_ids = device_plane_service_container_ids("device-control-core")
if len(core_ids) != 1:
@@ -20886,6 +21334,12 @@ def component_services(component, entries=None):
):
return ("device-control-core",)
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
component,
entries,
):
return ("device-control-core",)
if is_device_plane_manager_only_release_slice(component, entries):
return ("device-manager",)
@@ -21519,6 +21973,24 @@ def component_builds(component, entries=None):
),
),)
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
component,
entries,
):
return ((
DEVICE_PLANE_ROOT,
(
"build",
"--no-cache",
"--network=host",
"-f",
"services/device-control-core/Dockerfile",
"-t",
DEVICE_PLANE_CONTROL_CORE_IMAGE,
".",
),
),)
if is_device_plane_manager_only_release_slice(component, entries):
return ((
DEVICE_PLANE_ROOT / "services/device-manager",
@@ -29715,6 +30187,7 @@ def plan_artifact(artifact):
device_plane_control_core_incident_audit_preflight = None
device_plane_control_core_migration_replay_audit_preflight = None
device_plane_control_core_migration_replay_recovery_preflight = None
device_plane_control_core_migration_replay_checkpoint_recovery_preflight = None
device_plane_backhaul_preflight = None
device_plane_backhaul_vps_enrollment_preflight = None
device_plane_runtime_before = None
@@ -30026,6 +30499,23 @@ def plan_artifact(artifact):
payload_dir
)
)
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
manifest["component"],
entries,
):
if (
manifest["id"]
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID
):
die(
"Device Control Core migration replay checkpoint recovery "
"patch id mismatch"
)
device_plane_control_core_migration_replay_checkpoint_recovery_preflight = (
validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
payload_dir
)
)
if is_device_plane_backhaul_target_slice(
manifest["component"],
entries,
@@ -32344,6 +32834,82 @@ def plan_artifact(artifact):
"device_plane_rollback="
"source+exact-degraded-predecessor-image-runtime"
)
if (
device_plane_control_core_migration_replay_checkpoint_recovery_preflight
is not None
):
recovery = (
device_plane_control_core_migration_replay_checkpoint_recovery_preflight
)
database = recovery["database"]
core = recovery["core"]
print(f"device_plane_transition={recovery['mode']}")
print(
"device_control_core_root_cause="
"restart-replay-cycles-exact-committed-migration-checkpoints"
)
print(
"device_control_core_failed_recovery_044_artifact_sha256="
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256}"
)
print(
"device_control_core_failed_recovery_044_started_apply=false"
)
print(
"device_control_core_migration_predecessor_sha256="
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256}"
)
print(
"device_control_core_migration_target_sha256="
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256}"
)
print(
"device_control_core_current_runtime="
f"{core['containerId']}:{core['imageId']}:"
f"{core['status']}:{core['health']}:"
f"restarts={core['restartCount']}"
)
print(
"device_control_core_live_command_kind_incompatible_count="
f"{database['invalidCommandKindCount']}"
)
print(
"device_control_core_triggering_newer_receipt_count="
f"{database['triggeringReceiptCount']}"
)
print(
"device_control_core_current_constraint="
f"not-validated:exact-{database['constraintPhase']}-command-kinds"
)
print(
"device_control_core_allowed_predecessor_constraints="
"exact-replay-005|replay-007|replay-009|replay-011"
)
print(
"device_control_core_target_constraint="
"validated:exact-migration-016-command-kinds"
)
emit_device_plane_control_core_migration_replay_database_evidence(
database
)
print("device_postgres_row_mutation=none")
print("device_postgres_telemetry_table=absent")
print(
"device_plane_runtime_mutation="
"build+recreate:device-control-core"
)
print(
"device_plane_runtime_services="
"preserved:device-manager,device-gateway,device-postgres,"
"device-backhaul-target"
)
print("device_gateway_public_ingress=disabled")
print("device_gateway_tcp_9921=preserved:loopback-only")
print("gelios=untouched-legacy-only")
print(
"device_plane_rollback="
"source+exact-degraded-predecessor-image-runtime"
)
if device_plane_backhaul_vps_enrollment_preflight is not None:
print(
"device_plane_transition="
@@ -32960,6 +33526,10 @@ def rollback_device_plane_apply(
"device-plane",
entries,
)
or is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
"device-plane",
entries,
)
)
if control_core_release_rollback:
before = {
@@ -34580,6 +35150,21 @@ def component_healthchecks(component, entries=None, services=None):
"commandTransport": "typed-service-ping-v1",
},
},)
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
component,
entries,
):
return ({
"url": "http://127.0.0.1:18120/healthz",
"expected_json": {
"ok": True,
"service": "nodedc-device-control-core",
"database": "ready",
"discoveryIngest": "enabled",
"managementApi": "enabled",
"commandTransport": "typed-service-ping-v1",
},
},)
if is_device_plane_manager_control_plane_slice(component, entries):
command_transport = (
"typed-service-ping-v1"
@@ -35644,6 +36229,38 @@ def run_healthchecks(component, entries=None, services=None):
accept_device_plane_control_core_migration_replay_recovery()
return
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
component,
entries,
):
if tuple(services or ()) != ("device-control-core",):
die(
"Device Control Core migration replay checkpoint recovery "
"service set mismatch"
)
try:
healthcheck_compose_service_with_grace(
"device-plane",
"device-control-core",
)
except DeployError:
emit_bounded_device_control_core_failure_logs()
raise
for service in (
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
):
healthcheck_compose_service_with_grace(
"device-plane",
service,
)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
accept_device_plane_control_core_migration_replay_checkpoint_recovery()
return
if is_device_plane_control_core_release_slice(component, entries):
if tuple(services or ()) != ("device-control-core",):
die("Device Control Core release service set mismatch")
@@ -36820,6 +37437,21 @@ def apply_artifact(artifact):
validate_device_plane_control_core_migration_replay_recovery_evidence(
payload_dir
)
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
component,
entries,
):
if (
patch_id
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID
):
die(
"Device Control Core migration replay checkpoint "
"recovery patch id mismatch"
)
validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
payload_dir
)
if is_device_plane_backhaul_target_slice(
component,
entries,
@@ -37146,6 +37778,10 @@ def apply_artifact(artifact):
component,
entries,
)
or is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
component,
entries,
)
):
inventory_services = (
*inventory_services,