SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore(deploy): register migration invariant audit

Browse Source
This commit is contained in:
Codex
2026-08-22 22:11:35 +03:00
parent e642113124
commit 7b0e6d0bdf
2 changed files with 332 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+323 -8
View File
@@ -450,6 +450,15 @@ DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES = (
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID = (
"device-control-core-incident-audit-20260822-043"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL = (
"deployment/device-control-core-migration-replay-audit-v1.json"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_ENTRIES = (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL,
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_PATCH_ID = (
"device-control-core-migration-replay-audit-20260822-045"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL = (
"deployment/device-control-core-migration-replay-recovery-v1.json"
)
@@ -464,6 +473,13 @@ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES = (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID = (
"device-control-core-migration-replay-recovery-20260822-044"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT = (
"nodedc-device-plane-device-control-core-migration-replay-recovery-"
"20260822-044.tgz"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 = (
"b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7"
)
DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256 = (
"751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
)
@@ -4874,6 +4890,7 @@ def allowed_payload_path(component, rel):
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL,
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL,
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
@@ -11341,6 +11358,18 @@ def load_artifact(artifact, work_dir):
entries,
):
validate_device_plane_control_core_incident_audit_payload(payload_dir)
if is_device_plane_control_core_migration_replay_audit_slice(
manifest["component"],
entries,
):
if (
manifest["id"]
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_PATCH_ID
):
die("Device Control Core migration replay audit patch id mismatch")
validate_device_plane_control_core_migration_replay_audit_payload(
payload_dir
)
if is_device_plane_control_core_migration_replay_recovery_slice(
manifest["component"],
entries,
@@ -12010,6 +12039,18 @@ def is_device_plane_control_core_incident_audit_slice(component, entries):
)
def is_device_plane_control_core_migration_replay_audit_slice(
component,
entries,
):
return (
component == "device-plane"
and entries is not None
and tuple(entries)
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_ENTRIES
)
def is_device_plane_control_core_migration_replay_recovery_slice(
component,
entries,
@@ -12761,6 +12802,45 @@ def expected_device_plane_control_core_incident_audit_descriptor():
}
def expected_device_plane_control_core_migration_replay_audit_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane."
"device-control-core-migration-replay-audit.v1"
),
"mode": "rejected-recovery-044-live-invariants-read-only-audit",
"allowedOperation": "canonical-plan-only",
"applyAllowed": False,
"rejectedRecovery": {
"patchId": (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
),
"artifactSha256": (
DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
),
},
"readOnlyEvidence": [
"invalid-command-kind-count",
"triggering-receipt-count",
"constraint-validated",
"constraint-covers-final-command-kinds",
"host-telemetry-table-absent",
],
"runtimeMutation": "none",
"sourceMutation": "none",
"databaseMutation": "none",
"networkMutation": "none",
"secretRead": "none",
"preservedServices": [
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
}
def expected_device_plane_control_core_migration_replay_recovery_descriptor():
return {
"schemaVersion": (
@@ -13711,6 +13791,22 @@ def validate_device_plane_control_core_incident_audit_payload(payload_dir):
return descriptor
def validate_device_plane_control_core_migration_replay_audit_payload(
payload_dir,
):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_REL,
"Device Control Core migration replay audit descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_control_core_migration_replay_audit_descriptor()
):
die("Device Control Core migration replay audit descriptor mismatch")
return descriptor
def validate_device_plane_control_core_migration_replay_recovery_payload(
payload_dir,
):
@@ -16757,7 +16853,53 @@ def validate_device_plane_control_core_double_failure_evidence():
}
def collect_device_plane_control_core_migration_replay_database_evidence():
def device_plane_control_core_migration_replay_database_invariants_match(
evidence,
):
return (
evidence["invalidCommandKindCount"] == 0
and evidence["triggeringReceiptCount"] >= 1
and evidence["constraintValidated"]
and evidence["constraintCoversFinalKinds"]
and evidence["hostTelemetryTableAbsent"]
)
def emit_device_plane_control_core_migration_replay_database_evidence(
evidence,
):
def boolean(value):
return "true" if value else "false"
print(
"device_control_core_invalid_command_kind_count="
f"{evidence['invalidCommandKindCount']}"
)
print(
"device_control_core_triggering_receipt_count="
f"{evidence['triggeringReceiptCount']}"
)
print(
"device_control_core_constraint_validated="
f"{boolean(evidence['constraintValidated'])}"
)
print(
"device_control_core_constraint_covers_final_kinds="
f"{boolean(evidence['constraintCoversFinalKinds'])}"
)
print(
"device_control_core_host_telemetry_table_absent="
f"{boolean(evidence['hostTelemetryTableAbsent'])}"
)
print(
"device_control_core_recovery_044_ready="
f"{boolean(evidence['recovery044Ready'])}"
)
def collect_device_plane_control_core_migration_replay_database_evidence(
enforce_recovery_invariants=True,
):
postgres_ids = device_plane_service_container_ids("device-postgres")
if len(postgres_ids) != 1:
die("Device Control Core migration recovery PostgreSQL is missing")
@@ -16837,17 +16979,117 @@ select (
"hostTelemetryTableAbsent": values[4] == "true",
"query": query,
}
if (
evidence["invalidCommandKindCount"] != 0
or evidence["triggeringReceiptCount"] < 1
or not evidence["constraintValidated"]
or not evidence["constraintCoversFinalKinds"]
or not evidence["hostTelemetryTableAbsent"]
):
evidence["recovery044Ready"] = (
device_plane_control_core_migration_replay_database_invariants_match(
evidence
)
)
if enforce_recovery_invariants and not evidence["recovery044Ready"]:
emit_device_plane_control_core_migration_replay_database_evidence(
evidence
)
die("Device Control Core migration recovery database invariant mismatch")
return evidence
def validate_device_plane_control_core_migration_replay_audit_evidence(
payload_dir,
):
descriptor = (
validate_device_plane_control_core_migration_replay_audit_payload(
payload_dir
)
)
failure_evidence = validate_device_plane_control_core_double_failure_evidence()
recovery_artifact = (
INBOX_DIR
/ DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT
)
try:
recovery_stat = recovery_artifact.lstat()
except FileNotFoundError:
die("Device Control Core migration recovery staged artifact is missing")
if (
stat.S_ISLNK(recovery_stat.st_mode)
or not stat.S_ISREG(recovery_stat.st_mode)
or sha256_file(recovery_artifact)
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
):
die("Device Control Core migration recovery staged artifact mismatch")
with tempfile.TemporaryDirectory(
prefix="device-control-core-migration-replay-audit-",
dir=TMP_DIR,
) as directory:
recovery_manifest, recovery_entries, recovery_payload = load_artifact(
recovery_artifact,
Path(directory),
)
if (
recovery_manifest.get("id")
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
or recovery_manifest.get("component") != "device-plane"
or recovery_manifest.get("type") != "app-overlay"
or tuple(recovery_entries)
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES
):
die("Device Control Core migration recovery artifact contract mismatch")
validate_device_plane_control_core_migration_replay_recovery_payload(
recovery_payload
)
if any(
value.get("id")
== DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID
for value in (
*load_state(APPLIED_STATE_FILE),
*load_state(FAILED_STATE_FILE),
)
):
die("Device Control Core migration recovery has terminal journal state")
root = component_root("device-plane")
live_migration = root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
if (
live_migration.is_symlink()
or not live_migration.is_file()
or sha256_file(live_migration)
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256
):
die("Device Control Core migration replay audit source mismatch")
installed_descriptor = (
root / DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL
)
if installed_descriptor.exists() or installed_descriptor.is_symlink():
die("Device Control Core migration recovery is already installed")
runtime = (
validate_device_plane_control_core_migration_replay_preserved_runtime(
failure_evidence
)
)
core = runtime["core"]
if (
core["imageId"] != DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID
or core["status"] not in ("running", "restarting", "exited")
or core["health"] not in ("starting", "unhealthy", None)
):
die("Device Control Core migration replay audit runtime mismatch")
database = (
collect_device_plane_control_core_migration_replay_database_evidence(
enforce_recovery_invariants=False,
)
)
return {
"mode": descriptor["mode"],
"descriptor": descriptor,
"firstBackup": failure_evidence["firstBackup"],
"secondBackup": failure_evidence["secondBackup"],
"runtime": runtime["current"],
"core": core,
"database": database,
"recoveryArtifact": recovery_artifact,
}
def validate_device_plane_control_core_migration_replay_preserved_runtime(
evidence,
):
@@ -20547,6 +20789,12 @@ def component_services(component, entries=None):
if is_device_plane_control_core_incident_audit_slice(component, entries):
return ()
if is_device_plane_control_core_migration_replay_audit_slice(
component,
entries,
):
return ()
if is_device_plane_control_core_migration_replay_recovery_slice(
component,
entries,
@@ -21162,6 +21410,12 @@ def component_builds(component, entries=None):
if is_device_plane_control_core_incident_audit_slice(component, entries):
return ()
if is_device_plane_control_core_migration_replay_audit_slice(
component,
entries,
):
return ()
if is_device_plane_control_core_migration_replay_recovery_slice(
component,
entries,
@@ -29374,6 +29628,7 @@ def plan_artifact(artifact):
device_plane_manager_v2_reconciliation_preflight = None
device_plane_control_core_v3_reconciliation_preflight = None
device_plane_control_core_incident_audit_preflight = None
device_plane_control_core_migration_replay_audit_preflight = None
device_plane_control_core_migration_replay_recovery_preflight = None
device_plane_backhaul_preflight = None
device_plane_backhaul_vps_enrollment_preflight = None
@@ -29658,6 +29913,20 @@ def plan_artifact(artifact):
collect_device_plane_control_core_incident_audit()
)
device_plane_control_core_incident_audit_preflight = incident
if is_device_plane_control_core_migration_replay_audit_slice(
manifest["component"],
entries,
):
if (
manifest["id"]
!= DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_AUDIT_PATCH_ID
):
die("Device Control Core migration replay audit patch id mismatch")
device_plane_control_core_migration_replay_audit_preflight = (
validate_device_plane_control_core_migration_replay_audit_evidence(
payload_dir
)
)
if is_device_plane_control_core_migration_replay_recovery_slice(
manifest["component"],
entries,
@@ -31890,6 +32159,40 @@ def plan_artifact(artifact):
"read-only:device-control-core,device-manager,device-gateway,"
"device-postgres,device-backhaul-target"
)
if (
device_plane_control_core_migration_replay_audit_preflight
is not None
):
audit = device_plane_control_core_migration_replay_audit_preflight
database = audit["database"]
core = audit["core"]
print(f"device_plane_transition={audit['mode']}")
print("allowed_operation=canonical-plan-only")
print("apply=forbidden")
print("device_plane_build=none")
print("device_plane_runtime_mutation=none")
print("device_plane_source_mutation=none")
print("device_postgres_mutation=none")
print("device_plane_network_mutation=none")
print("device_plane_secret_read=none")
print(
"device_control_core_current_runtime="
f"{core['containerId']}:{core['imageId']}:"
f"{core['status']}:{core['health']}:"
f"restarts={core['restartCount']}"
)
print(
"device_control_core_rejected_recovery_artifact_sha256="
f"{DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256}"
)
emit_device_plane_control_core_migration_replay_database_evidence(
database
)
print(
"device_plane_runtime_services="
"read-only:device-control-core,device-manager,device-gateway,"
"device-postgres,device-backhaul-target"
)
if (
device_plane_control_core_migration_replay_recovery_preflight
is not None
@@ -36308,6 +36611,18 @@ def apply_artifact(artifact):
"apply is forbidden"
)
if is_device_plane_control_core_migration_replay_audit_slice(
component,
entries,
):
validate_device_plane_control_core_migration_replay_audit_payload(
payload_dir
)
die(
"Device Control Core migration replay audit is "
"canonical-plan-only; apply is forbidden"
)
if state_has_sha(sha):
die(f"artifact sha already applied: {sha}")
if state_has_patch_id(patch_id):