fix(deploy): accept exact replay checkpoints
This commit is contained in:
@@ -480,6 +480,31 @@ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT = (
|
|||||||
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 = (
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 = (
|
||||||
"b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7"
|
"b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7"
|
||||||
)
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT = (
|
||||||
|
"nodedc-device-plane-device-control-core-migration-replay-recovery-"
|
||||||
|
"20260822-044.tgz.20260822-232936"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT = (
|
||||||
|
"2026-08-22T20:29:59+00:00"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL = (
|
||||||
|
"deployment/"
|
||||||
|
"device-control-core-migration-replay-checkpoint-recovery-v2.json"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES = (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID = (
|
||||||
|
"device-control-core-migration-replay-checkpoint-recovery-20260822-046"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT = (
|
||||||
|
"nodedc-device-plane-device-control-core-migration-replay-checkpoint-"
|
||||||
|
"recovery-20260822-046.tgz"
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT_SHA256 = (
|
||||||
|
"46000c76977fb583fc7c9cf74ecf624efd8b404f7b8d0322e0270e7b8ac6e450"
|
||||||
|
)
|
||||||
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = (
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = (
|
||||||
"751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
|
"751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
|
||||||
)
|
)
|
||||||
@@ -517,6 +542,30 @@ DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS = (
|
|||||||
"infrastructure_service_instance.ensure",
|
"infrastructure_service_instance.ensure",
|
||||||
"health_observation.record",
|
"health_observation.record",
|
||||||
)
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS = (
|
||||||
|
"owner_scope.ensure",
|
||||||
|
"project.ensure",
|
||||||
|
"collection.ensure",
|
||||||
|
"project_grant.upsert",
|
||||||
|
"adapter_package.ensure",
|
||||||
|
"adapter_version.register",
|
||||||
|
"model_profile.register",
|
||||||
|
"edge.ensure",
|
||||||
|
"route.ensure",
|
||||||
|
"enrollment_intent.ensure",
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS = (
|
||||||
|
*DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS,
|
||||||
|
"device.claim",
|
||||||
|
"device.transfer",
|
||||||
|
"discovery.reject",
|
||||||
|
"discovery.expire",
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_REPLAY_009_COMMAND_KINDS = (
|
||||||
|
*DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS,
|
||||||
|
"device_credential_binding.upsert",
|
||||||
|
"device_credential_binding.revoke",
|
||||||
|
)
|
||||||
DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS = (
|
DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS = (
|
||||||
"owner_scope.ensure",
|
"owner_scope.ensure",
|
||||||
"project.ensure",
|
"project.ensure",
|
||||||
@@ -539,6 +588,15 @@ DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS = (
|
|||||||
"device_configuration_revision.create",
|
"device_configuration_revision.create",
|
||||||
"device_configuration_desired.set",
|
"device_configuration_desired.set",
|
||||||
)
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS = (
|
||||||
|
("replay-005", DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS),
|
||||||
|
("replay-007", DEVICE_PLANE_CONTROL_CORE_REPLAY_007_COMMAND_KINDS),
|
||||||
|
("replay-009", DEVICE_PLANE_CONTROL_CORE_REPLAY_009_COMMAND_KINDS),
|
||||||
|
("replay-011", DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS),
|
||||||
|
)
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES = tuple(
|
||||||
|
phase for phase, _kinds in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS
|
||||||
|
)
|
||||||
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS = (
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS = (
|
||||||
"asset.ensure",
|
"asset.ensure",
|
||||||
"asset_binding.ensure",
|
"asset_binding.ensure",
|
||||||
@@ -4914,6 +4972,7 @@ def allowed_payload_path(component, rel):
|
|||||||
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
|
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
|
||||||
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL,
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL,
|
||||||
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
|
||||||
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
|
||||||
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
|
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
|
||||||
"packages/device-protocol-contract",
|
"packages/device-protocol-contract",
|
||||||
@@ -11404,6 +11463,21 @@ def load_artifact(artifact, work_dir):
|
|||||||
validate_device_plane_control_core_migration_replay_recovery_payload(
|
validate_device_plane_control_core_migration_replay_recovery_payload(
|
||||||
payload_dir
|
payload_dir
|
||||||
)
|
)
|
||||||
|
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
manifest["component"],
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if (
|
||||||
|
manifest["id"]
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint recovery "
|
||||||
|
"patch id mismatch"
|
||||||
|
)
|
||||||
|
validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
if is_device_plane_postgres_bootstrap_slice(
|
if is_device_plane_postgres_bootstrap_slice(
|
||||||
manifest["component"],
|
manifest["component"],
|
||||||
entries,
|
entries,
|
||||||
@@ -12085,6 +12159,18 @@ def is_device_plane_control_core_migration_replay_recovery_slice(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return (
|
||||||
|
component == "device-plane"
|
||||||
|
and entries is not None
|
||||||
|
and tuple(entries)
|
||||||
|
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def is_device_plane_manager_v2_control_plane_slice(component, entries):
|
def is_device_plane_manager_v2_control_plane_slice(component, entries):
|
||||||
return (
|
return (
|
||||||
component == "device-plane"
|
component == "device-plane"
|
||||||
@@ -12911,6 +12997,66 @@ def expected_device_plane_control_core_migration_replay_recovery_descriptor():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor():
|
||||||
|
return {
|
||||||
|
"schemaVersion": (
|
||||||
|
"nodedc.device-plane."
|
||||||
|
"device-control-core-migration-replay-checkpoint-recovery.v2"
|
||||||
|
),
|
||||||
|
"mode": "terminal-044-replay-checkpoint-forward-repair",
|
||||||
|
"failedIncidentAudit": (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID
|
||||||
|
),
|
||||||
|
"failedRecovery": {
|
||||||
|
"patchId": (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
|
||||||
|
),
|
||||||
|
"artifactSha256": (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
|
||||||
|
),
|
||||||
|
"failure": "preflight-replay-checkpoint-race",
|
||||||
|
"startedApply": False,
|
||||||
|
},
|
||||||
|
"sourcePredecessor": {
|
||||||
|
"path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
|
||||||
|
"sha256": (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"sourceTarget": {
|
||||||
|
"path": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL,
|
||||||
|
"sha256": DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256,
|
||||||
|
},
|
||||||
|
"rootCause": (
|
||||||
|
"restarting-core-cycles-exact-committed-migration-checkpoints"
|
||||||
|
),
|
||||||
|
"repair": "migration-014-add-constraint-not-valid",
|
||||||
|
"databasePreflight": (
|
||||||
|
"exact-replay-checkpoint-005-007-009-011-and-final-compatible-rows"
|
||||||
|
),
|
||||||
|
"databaseRowMutation": "none",
|
||||||
|
"databaseSchemaOutcome": (
|
||||||
|
"exact-final-migration-016-validated-command-kind-check"
|
||||||
|
),
|
||||||
|
"runtimeAction": "build+recreate-device-control-core-only",
|
||||||
|
"runtimePredecessor": (
|
||||||
|
"proven-degraded-restarting-exact-preapply-image"
|
||||||
|
),
|
||||||
|
"preservedServices": [
|
||||||
|
"device-manager",
|
||||||
|
"device-gateway",
|
||||||
|
"device-postgres",
|
||||||
|
"device-backhaul-target",
|
||||||
|
],
|
||||||
|
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
|
||||||
|
"publicIngress": "disabled",
|
||||||
|
"edgeChannel": (
|
||||||
|
"core-initiated-pinned-mtls-registered-edges-only"
|
||||||
|
),
|
||||||
|
"rollback": "source+exact-degraded-predecessor-image-runtime",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def validate_platform_device_core_hub_trust_payload(payload_dir):
|
def validate_platform_device_core_hub_trust_payload(payload_dir):
|
||||||
descriptor = read_strict_json(
|
descriptor = read_strict_json(
|
||||||
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
|
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
|
||||||
@@ -13843,6 +13989,32 @@ def validate_device_plane_control_core_migration_replay_recovery_payload(
|
|||||||
!= expected_device_plane_control_core_migration_replay_recovery_descriptor()
|
!= expected_device_plane_control_core_migration_replay_recovery_descriptor()
|
||||||
):
|
):
|
||||||
die("Device Control Core migration replay recovery descriptor mismatch")
|
die("Device Control Core migration replay recovery descriptor mismatch")
|
||||||
|
validate_device_plane_control_core_migration_014_repair(payload_dir)
|
||||||
|
return descriptor
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload(
|
||||||
|
payload_dir,
|
||||||
|
):
|
||||||
|
descriptor = read_strict_json(
|
||||||
|
payload_dir
|
||||||
|
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
|
||||||
|
"Device Control Core migration replay checkpoint recovery descriptor",
|
||||||
|
max_bytes=16 * 1024,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
descriptor
|
||||||
|
!= expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor()
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint recovery "
|
||||||
|
"descriptor mismatch"
|
||||||
|
)
|
||||||
|
validate_device_plane_control_core_migration_014_repair(payload_dir)
|
||||||
|
return descriptor
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_migration_014_repair(payload_dir):
|
||||||
migration = payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
migration = payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
||||||
try:
|
try:
|
||||||
migration_stat = migration.lstat()
|
migration_stat = migration.lstat()
|
||||||
@@ -13861,7 +14033,6 @@ def validate_device_plane_control_core_migration_replay_recovery_payload(
|
|||||||
)
|
)
|
||||||
):
|
):
|
||||||
die("Device Control Core migration 014 repair mismatch")
|
die("Device Control Core migration 014 repair mismatch")
|
||||||
return descriptor
|
|
||||||
|
|
||||||
|
|
||||||
def is_device_plane_postgres_bootstrap_slice(component, entries):
|
def is_device_plane_postgres_bootstrap_slice(component, entries):
|
||||||
@@ -16875,6 +17046,79 @@ def validate_device_plane_control_core_double_failure_evidence():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_migration_replay_recovery_failure():
|
||||||
|
failed_artifact = (
|
||||||
|
FAILED_DIR
|
||||||
|
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
failed_stat = failed_artifact.lstat()
|
||||||
|
except FileNotFoundError:
|
||||||
|
die("Device Control Core migration recovery 044 failed artifact is missing")
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(failed_stat.st_mode)
|
||||||
|
or not stat.S_ISREG(failed_stat.st_mode)
|
||||||
|
or sha256_file(failed_artifact)
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery 044 failed artifact mismatch")
|
||||||
|
if any(
|
||||||
|
value.get("id")
|
||||||
|
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
|
||||||
|
for value in load_state(STATE_FILE)
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery 044 applied journal conflict")
|
||||||
|
records = [
|
||||||
|
value
|
||||||
|
for value in load_state(FAILED_STATE_FILE)
|
||||||
|
if value.get("id")
|
||||||
|
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
|
||||||
|
]
|
||||||
|
if len(records) != 1:
|
||||||
|
die("Device Control Core migration recovery 044 failed journal count mismatch")
|
||||||
|
record = records[0]
|
||||||
|
if (
|
||||||
|
record.get("artifact")
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT
|
||||||
|
or record.get("backup_id") is not None
|
||||||
|
or record.get("component") != "device-plane"
|
||||||
|
or record.get("failed_at")
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT
|
||||||
|
or record.get("sha256")
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
|
||||||
|
or record.get("started_apply") is not False
|
||||||
|
or record.get("rollback_status") != "not-required"
|
||||||
|
or record.get("status") != "failed"
|
||||||
|
or record.get("message")
|
||||||
|
!= "Device Control Core migration recovery database invariant mismatch"
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery 044 failed journal mismatch")
|
||||||
|
with tempfile.TemporaryDirectory(
|
||||||
|
prefix="device-control-core-migration-recovery-044-failed-",
|
||||||
|
dir=TMP_DIR,
|
||||||
|
) as directory:
|
||||||
|
failed_manifest, failed_entries, failed_payload = load_artifact(
|
||||||
|
failed_artifact,
|
||||||
|
Path(directory),
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
failed_manifest.get("id")
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
|
||||||
|
or failed_manifest.get("component") != "device-plane"
|
||||||
|
or failed_manifest.get("type") != "app-overlay"
|
||||||
|
or tuple(failed_entries)
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration recovery 044 failed artifact "
|
||||||
|
"contract mismatch"
|
||||||
|
)
|
||||||
|
validate_device_plane_control_core_migration_replay_recovery_payload(
|
||||||
|
failed_payload
|
||||||
|
)
|
||||||
|
return {"artifact": failed_artifact, "record": record}
|
||||||
|
|
||||||
|
|
||||||
def device_plane_control_core_migration_replay_database_invariants_match(
|
def device_plane_control_core_migration_replay_database_invariants_match(
|
||||||
evidence,
|
evidence,
|
||||||
):
|
):
|
||||||
@@ -16883,6 +17127,7 @@ def device_plane_control_core_migration_replay_database_invariants_match(
|
|||||||
and evidence["triggeringReceiptCount"] >= 1
|
and evidence["triggeringReceiptCount"] >= 1
|
||||||
and evidence["constraintValidated"]
|
and evidence["constraintValidated"]
|
||||||
and evidence["constraintCoversFinalKinds"]
|
and evidence["constraintCoversFinalKinds"]
|
||||||
|
and evidence["constraintPhase"] == "final-016"
|
||||||
and evidence["hostTelemetryTableAbsent"]
|
and evidence["hostTelemetryTableAbsent"]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16900,6 +17145,20 @@ def device_plane_control_core_migration_replay_predecessor_invariants_match(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def device_plane_control_core_migration_replay_checkpoint_invariants_match(
|
||||||
|
evidence,
|
||||||
|
):
|
||||||
|
return (
|
||||||
|
evidence["invalidCommandKindCount"] == 0
|
||||||
|
and evidence["triggeringReceiptCount"] >= 1
|
||||||
|
and evidence["constraintValidationState"] == "false"
|
||||||
|
and not evidence["constraintCoversFinalKinds"]
|
||||||
|
and evidence["constraintPhase"]
|
||||||
|
in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES
|
||||||
|
and evidence["hostTelemetryTableAbsent"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def emit_device_plane_control_core_migration_replay_database_evidence(
|
def emit_device_plane_control_core_migration_replay_database_evidence(
|
||||||
evidence,
|
evidence,
|
||||||
):
|
):
|
||||||
@@ -16930,10 +17189,26 @@ def emit_device_plane_control_core_migration_replay_database_evidence(
|
|||||||
"device_control_core_constraint_matches_replay_011_kinds="
|
"device_control_core_constraint_matches_replay_011_kinds="
|
||||||
f"{boolean(evidence['constraintMatchesReplay011Kinds'])}"
|
f"{boolean(evidence['constraintMatchesReplay011Kinds'])}"
|
||||||
)
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_constraint_phase="
|
||||||
|
f"{evidence['constraintPhase']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_constraint_matches_known_replay_checkpoint="
|
||||||
|
f"{boolean(evidence['constraintMatchesKnownReplayCheckpoint'])}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_constraint_matches_final_016_kinds="
|
||||||
|
f"{boolean(evidence['constraintMatchesFinalKinds'])}"
|
||||||
|
)
|
||||||
print(
|
print(
|
||||||
"device_control_core_recovery_044_ready="
|
"device_control_core_recovery_044_ready="
|
||||||
f"{boolean(evidence['recovery044Ready'])}"
|
f"{boolean(evidence['recovery044Ready'])}"
|
||||||
)
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_checkpoint_recovery_ready="
|
||||||
|
f"{boolean(evidence['checkpointRecoveryReady'])}"
|
||||||
|
)
|
||||||
print(
|
print(
|
||||||
"device_control_core_recovery_final_state_ready="
|
"device_control_core_recovery_final_state_ready="
|
||||||
f"{boolean(evidence['finalStateReady'])}"
|
f"{boolean(evidence['finalStateReady'])}"
|
||||||
@@ -16944,7 +17219,11 @@ def collect_device_plane_control_core_migration_replay_database_evidence(
|
|||||||
enforce_recovery_invariants=True,
|
enforce_recovery_invariants=True,
|
||||||
expected_state="final",
|
expected_state="final",
|
||||||
):
|
):
|
||||||
if expected_state not in ("final", "replay-011-predecessor"):
|
if expected_state not in (
|
||||||
|
"final",
|
||||||
|
"replay-011-predecessor",
|
||||||
|
"replay-checkpoint-predecessor",
|
||||||
|
):
|
||||||
die("Device Control Core migration recovery database state is invalid")
|
die("Device Control Core migration recovery database state is invalid")
|
||||||
postgres_ids = device_plane_service_container_ids("device-postgres")
|
postgres_ids = device_plane_service_container_ids("device-postgres")
|
||||||
if len(postgres_ids) != 1:
|
if len(postgres_ids) != 1:
|
||||||
@@ -16957,10 +17236,6 @@ def collect_device_plane_control_core_migration_replay_database_evidence(
|
|||||||
"'" + value.replace("'", "''") + "'"
|
"'" + value.replace("'", "''") + "'"
|
||||||
for value in DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS
|
for value in DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TRIGGER_KINDS
|
||||||
)
|
)
|
||||||
replay_011 = ",".join(
|
|
||||||
"'" + value.replace("'", "''") + "'"
|
|
||||||
for value in DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS
|
|
||||||
)
|
|
||||||
query = f"""
|
query = f"""
|
||||||
with constraint_state as (
|
with constraint_state as (
|
||||||
select convalidated,
|
select convalidated,
|
||||||
@@ -16979,10 +17254,6 @@ constraint_literals as (
|
|||||||
'''([^'']+)''',
|
'''([^'']+)''',
|
||||||
'g'
|
'g'
|
||||||
) as matched(value)
|
) as matched(value)
|
||||||
),
|
|
||||||
expected_replay_011 as (
|
|
||||||
select array_agg(kind order by kind) as values
|
|
||||||
from unnest(array[{replay_011}]::text[]) as expected(kind)
|
|
||||||
)
|
)
|
||||||
select (
|
select (
|
||||||
select count(*)
|
select count(*)
|
||||||
@@ -16998,11 +17269,8 @@ select (
|
|||||||
coalesce((select covers_final::text from constraint_state), 'false'),
|
coalesce((select covers_final::text from constraint_state), 'false'),
|
||||||
(to_regclass('public.device_infrastructure_host_telemetry_samples')
|
(to_regclass('public.device_infrastructure_host_telemetry_samples')
|
||||||
is null)::text,
|
is null)::text,
|
||||||
coalesce((
|
coalesce((select array_to_json(values)::text from constraint_literals),
|
||||||
select constraint_literals.values = expected_replay_011.values
|
'[]')
|
||||||
from constraint_literals
|
|
||||||
cross join expected_replay_011
|
|
||||||
), false)::text
|
|
||||||
""".strip()
|
""".strip()
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
@@ -17038,16 +17306,55 @@ select (
|
|||||||
or values[2] not in ("true", "false", "missing")
|
or values[2] not in ("true", "false", "missing")
|
||||||
or values[3] not in ("true", "false")
|
or values[3] not in ("true", "false")
|
||||||
or values[4] not in ("true", "false")
|
or values[4] not in ("true", "false")
|
||||||
or values[5] not in ("true", "false")
|
|
||||||
):
|
):
|
||||||
die("Device Control Core migration recovery database read failed")
|
die("Device Control Core migration recovery database read failed")
|
||||||
|
try:
|
||||||
|
constraint_literal_kinds = json.loads(values[5])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
die("Device Control Core migration recovery constraint read failed")
|
||||||
|
if (
|
||||||
|
not isinstance(constraint_literal_kinds, list)
|
||||||
|
or len(constraint_literal_kinds) > 128
|
||||||
|
or any(
|
||||||
|
not isinstance(value, str)
|
||||||
|
or not re.fullmatch(r"[a-z][a-z0-9._-]{1,127}", value)
|
||||||
|
for value in constraint_literal_kinds
|
||||||
|
)
|
||||||
|
or len(set(constraint_literal_kinds)) != len(constraint_literal_kinds)
|
||||||
|
):
|
||||||
|
die("Device Control Core migration recovery constraint read failed")
|
||||||
|
constraint_literal_kinds = tuple(sorted(constraint_literal_kinds))
|
||||||
|
constraint_phase_by_kinds = {
|
||||||
|
tuple(sorted(kinds)): phase
|
||||||
|
for phase, kinds in (
|
||||||
|
*DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS,
|
||||||
|
("final-016", DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if values[2] == "missing":
|
||||||
|
constraint_phase = "missing"
|
||||||
|
else:
|
||||||
|
constraint_phase = constraint_phase_by_kinds.get(
|
||||||
|
constraint_literal_kinds,
|
||||||
|
"unknown",
|
||||||
|
)
|
||||||
evidence = {
|
evidence = {
|
||||||
"invalidCommandKindCount": int(values[0]),
|
"invalidCommandKindCount": int(values[0]),
|
||||||
"triggeringReceiptCount": int(values[1]),
|
"triggeringReceiptCount": int(values[1]),
|
||||||
|
"constraintValidationState": values[2],
|
||||||
"constraintValidated": values[2] == "true",
|
"constraintValidated": values[2] == "true",
|
||||||
"constraintCoversFinalKinds": values[3] == "true",
|
"constraintCoversFinalKinds": values[3] == "true",
|
||||||
"hostTelemetryTableAbsent": values[4] == "true",
|
"hostTelemetryTableAbsent": values[4] == "true",
|
||||||
"constraintMatchesReplay011Kinds": values[5] == "true",
|
"constraintLiteralKinds": constraint_literal_kinds,
|
||||||
|
"constraintPhase": constraint_phase,
|
||||||
|
"constraintMatchesReplay011Kinds": (
|
||||||
|
constraint_phase == "replay-011"
|
||||||
|
),
|
||||||
|
"constraintMatchesKnownReplayCheckpoint": (
|
||||||
|
constraint_phase
|
||||||
|
in DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINT_PHASES
|
||||||
|
),
|
||||||
|
"constraintMatchesFinalKinds": constraint_phase == "final-016",
|
||||||
"query": query,
|
"query": query,
|
||||||
}
|
}
|
||||||
evidence["finalStateReady"] = (
|
evidence["finalStateReady"] = (
|
||||||
@@ -17060,11 +17367,18 @@ select (
|
|||||||
evidence
|
evidence
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
expected_ready = (
|
evidence["checkpointRecoveryReady"] = (
|
||||||
evidence["recovery044Ready"]
|
device_plane_control_core_migration_replay_checkpoint_invariants_match(
|
||||||
if expected_state == "replay-011-predecessor"
|
evidence
|
||||||
else evidence["finalStateReady"]
|
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
expected_ready = {
|
||||||
|
"final": evidence["finalStateReady"],
|
||||||
|
"replay-011-predecessor": evidence["recovery044Ready"],
|
||||||
|
"replay-checkpoint-predecessor": (
|
||||||
|
evidence["checkpointRecoveryReady"]
|
||||||
|
),
|
||||||
|
}[expected_state]
|
||||||
if enforce_recovery_invariants and not expected_ready:
|
if enforce_recovery_invariants and not expected_ready:
|
||||||
emit_device_plane_control_core_migration_replay_database_evidence(
|
emit_device_plane_control_core_migration_replay_database_evidence(
|
||||||
evidence
|
evidence
|
||||||
@@ -17265,6 +17579,78 @@ def validate_device_plane_control_core_migration_replay_recovery_evidence(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
|
||||||
|
payload_dir,
|
||||||
|
):
|
||||||
|
descriptor = (
|
||||||
|
validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
|
)
|
||||||
|
evidence = validate_device_plane_control_core_double_failure_evidence()
|
||||||
|
failed_recovery = (
|
||||||
|
validate_device_plane_control_core_migration_replay_recovery_failure()
|
||||||
|
)
|
||||||
|
root = component_root("device-plane")
|
||||||
|
live_migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
||||||
|
if (
|
||||||
|
live_migration.is_symlink()
|
||||||
|
or not live_migration.is_file()
|
||||||
|
or sha256_file(live_migration)
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256
|
||||||
|
):
|
||||||
|
die("Device Control Core migration 014 predecessor mismatch")
|
||||||
|
for descriptor_rel in (
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
|
||||||
|
):
|
||||||
|
installed_descriptor = root / descriptor_rel
|
||||||
|
if installed_descriptor.exists() or installed_descriptor.is_symlink():
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint recovery "
|
||||||
|
"already installed"
|
||||||
|
)
|
||||||
|
runtime = (
|
||||||
|
validate_device_plane_control_core_migration_replay_preserved_runtime(
|
||||||
|
evidence
|
||||||
|
)
|
||||||
|
)
|
||||||
|
core = runtime["core"]
|
||||||
|
if (
|
||||||
|
core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
|
||||||
|
or core["status"] not in ("running", "restarting", "exited")
|
||||||
|
or core["health"] not in ("starting", "unhealthy", None)
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint predecessor "
|
||||||
|
"mismatch"
|
||||||
|
)
|
||||||
|
image_id = inspect_optional_local_image(
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
|
||||||
|
"Device Control Core migration replay checkpoint exact predecessor image",
|
||||||
|
)
|
||||||
|
if image_id != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID:
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint predecessor "
|
||||||
|
"image is missing"
|
||||||
|
)
|
||||||
|
database = (
|
||||||
|
collect_device_plane_control_core_migration_replay_database_evidence(
|
||||||
|
expected_state="replay-checkpoint-predecessor",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"mode": descriptor["mode"],
|
||||||
|
"descriptor": descriptor,
|
||||||
|
"firstBackup": evidence["firstBackup"],
|
||||||
|
"secondBackup": evidence["secondBackup"],
|
||||||
|
"failedRecovery": failed_recovery,
|
||||||
|
"runtime": runtime["current"],
|
||||||
|
"core": core,
|
||||||
|
"database": database,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def accept_device_plane_control_core_migration_replay_recovery():
|
def accept_device_plane_control_core_migration_replay_recovery():
|
||||||
root = component_root("device-plane")
|
root = component_root("device-plane")
|
||||||
migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
||||||
@@ -17316,6 +17702,68 @@ def accept_device_plane_control_core_migration_replay_recovery():
|
|||||||
return {"runtime": runtime["current"], "database": database}
|
return {"runtime": runtime["current"], "database": database}
|
||||||
|
|
||||||
|
|
||||||
|
def accept_device_plane_control_core_migration_replay_checkpoint_recovery():
|
||||||
|
root = component_root("device-plane")
|
||||||
|
migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
|
||||||
|
if (
|
||||||
|
migration.is_symlink()
|
||||||
|
or not migration.is_file()
|
||||||
|
or sha256_file(migration)
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256
|
||||||
|
):
|
||||||
|
die("installed Device Control Core migration 014 repair mismatch")
|
||||||
|
descriptor = read_strict_json(
|
||||||
|
root
|
||||||
|
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL,
|
||||||
|
"installed Device Control Core replay checkpoint recovery descriptor",
|
||||||
|
max_bytes=16 * 1024,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
descriptor
|
||||||
|
!= expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor()
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"installed Device Control Core replay checkpoint recovery mismatch"
|
||||||
|
)
|
||||||
|
legacy_descriptor = (
|
||||||
|
root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL
|
||||||
|
)
|
||||||
|
if legacy_descriptor.exists() or legacy_descriptor.is_symlink():
|
||||||
|
die("legacy Device Control Core migration recovery descriptor appeared")
|
||||||
|
evidence = {
|
||||||
|
"secondBackup": (
|
||||||
|
BACKUPS_DIR
|
||||||
|
/ DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID
|
||||||
|
)
|
||||||
|
}
|
||||||
|
runtime = (
|
||||||
|
validate_device_plane_control_core_migration_replay_preserved_runtime(
|
||||||
|
evidence
|
||||||
|
)
|
||||||
|
)
|
||||||
|
core = runtime["core"]
|
||||||
|
if (
|
||||||
|
core["imageId"] == DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
|
||||||
|
or core["status"] != "running"
|
||||||
|
or core["running"] is not True
|
||||||
|
or core["health"] != "healthy"
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint recovery did "
|
||||||
|
"not converge"
|
||||||
|
)
|
||||||
|
database = (
|
||||||
|
collect_device_plane_control_core_migration_replay_database_evidence(
|
||||||
|
expected_state="final",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
validate_device_manager_control_plane_runtime(
|
||||||
|
require_edge_channel=True,
|
||||||
|
core_network_mode="private-egress",
|
||||||
|
)
|
||||||
|
return {"runtime": runtime["current"], "database": database}
|
||||||
|
|
||||||
|
|
||||||
def emit_bounded_device_control_core_failure_logs():
|
def emit_bounded_device_control_core_failure_logs():
|
||||||
core_ids = device_plane_service_container_ids("device-control-core")
|
core_ids = device_plane_service_container_ids("device-control-core")
|
||||||
if len(core_ids) != 1:
|
if len(core_ids) != 1:
|
||||||
@@ -20886,6 +21334,12 @@ def component_services(component, entries=None):
|
|||||||
):
|
):
|
||||||
return ("device-control-core",)
|
return ("device-control-core",)
|
||||||
|
|
||||||
|
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return ("device-control-core",)
|
||||||
|
|
||||||
if is_device_plane_manager_only_release_slice(component, entries):
|
if is_device_plane_manager_only_release_slice(component, entries):
|
||||||
return ("device-manager",)
|
return ("device-manager",)
|
||||||
|
|
||||||
@@ -21519,6 +21973,24 @@ def component_builds(component, entries=None):
|
|||||||
),
|
),
|
||||||
),)
|
),)
|
||||||
|
|
||||||
|
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return ((
|
||||||
|
DEVICE_PLANE_ROOT,
|
||||||
|
(
|
||||||
|
"build",
|
||||||
|
"--no-cache",
|
||||||
|
"--network=host",
|
||||||
|
"-f",
|
||||||
|
"services/device-control-core/Dockerfile",
|
||||||
|
"-t",
|
||||||
|
DEVICE_PLANE_CONTROL_CORE_IMAGE,
|
||||||
|
".",
|
||||||
|
),
|
||||||
|
),)
|
||||||
|
|
||||||
if is_device_plane_manager_only_release_slice(component, entries):
|
if is_device_plane_manager_only_release_slice(component, entries):
|
||||||
return ((
|
return ((
|
||||||
DEVICE_PLANE_ROOT / "services/device-manager",
|
DEVICE_PLANE_ROOT / "services/device-manager",
|
||||||
@@ -29715,6 +30187,7 @@ def plan_artifact(artifact):
|
|||||||
device_plane_control_core_incident_audit_preflight = None
|
device_plane_control_core_incident_audit_preflight = None
|
||||||
device_plane_control_core_migration_replay_audit_preflight = None
|
device_plane_control_core_migration_replay_audit_preflight = None
|
||||||
device_plane_control_core_migration_replay_recovery_preflight = None
|
device_plane_control_core_migration_replay_recovery_preflight = None
|
||||||
|
device_plane_control_core_migration_replay_checkpoint_recovery_preflight = None
|
||||||
device_plane_backhaul_preflight = None
|
device_plane_backhaul_preflight = None
|
||||||
device_plane_backhaul_vps_enrollment_preflight = None
|
device_plane_backhaul_vps_enrollment_preflight = None
|
||||||
device_plane_runtime_before = None
|
device_plane_runtime_before = None
|
||||||
@@ -30026,6 +30499,23 @@ def plan_artifact(artifact):
|
|||||||
payload_dir
|
payload_dir
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
manifest["component"],
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if (
|
||||||
|
manifest["id"]
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint recovery "
|
||||||
|
"patch id mismatch"
|
||||||
|
)
|
||||||
|
device_plane_control_core_migration_replay_checkpoint_recovery_preflight = (
|
||||||
|
validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
|
)
|
||||||
if is_device_plane_backhaul_target_slice(
|
if is_device_plane_backhaul_target_slice(
|
||||||
manifest["component"],
|
manifest["component"],
|
||||||
entries,
|
entries,
|
||||||
@@ -32344,6 +32834,82 @@ def plan_artifact(artifact):
|
|||||||
"device_plane_rollback="
|
"device_plane_rollback="
|
||||||
"source+exact-degraded-predecessor-image-runtime"
|
"source+exact-degraded-predecessor-image-runtime"
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
device_plane_control_core_migration_replay_checkpoint_recovery_preflight
|
||||||
|
is not None
|
||||||
|
):
|
||||||
|
recovery = (
|
||||||
|
device_plane_control_core_migration_replay_checkpoint_recovery_preflight
|
||||||
|
)
|
||||||
|
database = recovery["database"]
|
||||||
|
core = recovery["core"]
|
||||||
|
print(f"device_plane_transition={recovery['mode']}")
|
||||||
|
print(
|
||||||
|
"device_control_core_root_cause="
|
||||||
|
"restart-replay-cycles-exact-committed-migration-checkpoints"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_failed_recovery_044_artifact_sha256="
|
||||||
|
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_failed_recovery_044_started_apply=false"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_migration_predecessor_sha256="
|
||||||
|
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_migration_target_sha256="
|
||||||
|
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_current_runtime="
|
||||||
|
f"{core['containerId']}:{core['imageId']}:"
|
||||||
|
f"{core['status']}:{core['health']}:"
|
||||||
|
f"restarts={core['restartCount']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_live_command_kind_incompatible_count="
|
||||||
|
f"{database['invalidCommandKindCount']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_triggering_newer_receipt_count="
|
||||||
|
f"{database['triggeringReceiptCount']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_current_constraint="
|
||||||
|
f"not-validated:exact-{database['constraintPhase']}-command-kinds"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_allowed_predecessor_constraints="
|
||||||
|
"exact-replay-005|replay-007|replay-009|replay-011"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_control_core_target_constraint="
|
||||||
|
"validated:exact-migration-016-command-kinds"
|
||||||
|
)
|
||||||
|
emit_device_plane_control_core_migration_replay_database_evidence(
|
||||||
|
database
|
||||||
|
)
|
||||||
|
print("device_postgres_row_mutation=none")
|
||||||
|
print("device_postgres_telemetry_table=absent")
|
||||||
|
print(
|
||||||
|
"device_plane_runtime_mutation="
|
||||||
|
"build+recreate:device-control-core"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"device_plane_runtime_services="
|
||||||
|
"preserved:device-manager,device-gateway,device-postgres,"
|
||||||
|
"device-backhaul-target"
|
||||||
|
)
|
||||||
|
print("device_gateway_public_ingress=disabled")
|
||||||
|
print("device_gateway_tcp_9921=preserved:loopback-only")
|
||||||
|
print("gelios=untouched-legacy-only")
|
||||||
|
print(
|
||||||
|
"device_plane_rollback="
|
||||||
|
"source+exact-degraded-predecessor-image-runtime"
|
||||||
|
)
|
||||||
if device_plane_backhaul_vps_enrollment_preflight is not None:
|
if device_plane_backhaul_vps_enrollment_preflight is not None:
|
||||||
print(
|
print(
|
||||||
"device_plane_transition="
|
"device_plane_transition="
|
||||||
@@ -32960,6 +33526,10 @@ def rollback_device_plane_apply(
|
|||||||
"device-plane",
|
"device-plane",
|
||||||
entries,
|
entries,
|
||||||
)
|
)
|
||||||
|
or is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
"device-plane",
|
||||||
|
entries,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if control_core_release_rollback:
|
if control_core_release_rollback:
|
||||||
before = {
|
before = {
|
||||||
@@ -34580,6 +35150,21 @@ def component_healthchecks(component, entries=None, services=None):
|
|||||||
"commandTransport": "typed-service-ping-v1",
|
"commandTransport": "typed-service-ping-v1",
|
||||||
},
|
},
|
||||||
},)
|
},)
|
||||||
|
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
return ({
|
||||||
|
"url": "http://127.0.0.1:18120/healthz",
|
||||||
|
"expected_json": {
|
||||||
|
"ok": True,
|
||||||
|
"service": "nodedc-device-control-core",
|
||||||
|
"database": "ready",
|
||||||
|
"discoveryIngest": "enabled",
|
||||||
|
"managementApi": "enabled",
|
||||||
|
"commandTransport": "typed-service-ping-v1",
|
||||||
|
},
|
||||||
|
},)
|
||||||
if is_device_plane_manager_control_plane_slice(component, entries):
|
if is_device_plane_manager_control_plane_slice(component, entries):
|
||||||
command_transport = (
|
command_transport = (
|
||||||
"typed-service-ping-v1"
|
"typed-service-ping-v1"
|
||||||
@@ -35644,6 +36229,38 @@ def run_healthchecks(component, entries=None, services=None):
|
|||||||
accept_device_plane_control_core_migration_replay_recovery()
|
accept_device_plane_control_core_migration_replay_recovery()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if tuple(services or ()) != ("device-control-core",):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint recovery "
|
||||||
|
"service set mismatch"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
healthcheck_compose_service_with_grace(
|
||||||
|
"device-plane",
|
||||||
|
"device-control-core",
|
||||||
|
)
|
||||||
|
except DeployError:
|
||||||
|
emit_bounded_device_control_core_failure_logs()
|
||||||
|
raise
|
||||||
|
for service in (
|
||||||
|
"device-manager",
|
||||||
|
"device-gateway",
|
||||||
|
"device-postgres",
|
||||||
|
"device-backhaul-target",
|
||||||
|
):
|
||||||
|
healthcheck_compose_service_with_grace(
|
||||||
|
"device-plane",
|
||||||
|
service,
|
||||||
|
)
|
||||||
|
for check in component_healthchecks(component, entries, services):
|
||||||
|
healthcheck_url(check)
|
||||||
|
accept_device_plane_control_core_migration_replay_checkpoint_recovery()
|
||||||
|
return
|
||||||
|
|
||||||
if is_device_plane_control_core_release_slice(component, entries):
|
if is_device_plane_control_core_release_slice(component, entries):
|
||||||
if tuple(services or ()) != ("device-control-core",):
|
if tuple(services or ()) != ("device-control-core",):
|
||||||
die("Device Control Core release service set mismatch")
|
die("Device Control Core release service set mismatch")
|
||||||
@@ -36820,6 +37437,21 @@ def apply_artifact(artifact):
|
|||||||
validate_device_plane_control_core_migration_replay_recovery_evidence(
|
validate_device_plane_control_core_migration_replay_recovery_evidence(
|
||||||
payload_dir
|
payload_dir
|
||||||
)
|
)
|
||||||
|
if is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
):
|
||||||
|
if (
|
||||||
|
patch_id
|
||||||
|
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID
|
||||||
|
):
|
||||||
|
die(
|
||||||
|
"Device Control Core migration replay checkpoint "
|
||||||
|
"recovery patch id mismatch"
|
||||||
|
)
|
||||||
|
validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
|
||||||
|
payload_dir
|
||||||
|
)
|
||||||
if is_device_plane_backhaul_target_slice(
|
if is_device_plane_backhaul_target_slice(
|
||||||
component,
|
component,
|
||||||
entries,
|
entries,
|
||||||
@@ -37146,6 +37778,10 @@ def apply_artifact(artifact):
|
|||||||
component,
|
component,
|
||||||
entries,
|
entries,
|
||||||
)
|
)
|
||||||
|
or is_device_plane_control_core_migration_replay_checkpoint_recovery_slice(
|
||||||
|
component,
|
||||||
|
entries,
|
||||||
|
)
|
||||||
):
|
):
|
||||||
inventory_services = (
|
inventory_services = (
|
||||||
*inventory_services,
|
*inventory_services,
|
||||||
|
|||||||
Reference in New Issue
Block a user