From bd715e9d71f4adc69ef124b0580b16cf56b797a7 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 22 Aug 2026 20:44:15 +0300 Subject: [PATCH] chore(deploy): register read-only Core incident audit --- infra/deploy-runner/nodedc-deploy | 459 +++++++++++++++++++++++++++++- 1 file changed, 458 insertions(+), 1 deletion(-) diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 37385da..0bbf6f4 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -441,6 +441,46 @@ DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING = ( "packages/infrastructure-telemetry-contract", DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL, ) +DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL = ( + "deployment/device-control-core-incident-audit-v1.json" +) +DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES = ( + DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL, +) +DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID = ( + "device-control-core-incident-audit-20260822-043" +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-control-core-release-v3-reconciliation-" + "20260822-042.tgz.20260822-195448" +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256 = ( + "54ab243439bce724fa0a0872b76cc32e0052ea5127153214d92872f02ae831cf" +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID = ( + "device-plane-device-control-core-release-v3-reconciliation-" + "20260822-042-20260822-195448" +) +DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_SHA256 = { + "manifest.env": ( + "37ed634d7ebde9d7da67990fe20cae7809a682c34652c37e163f9c2bc01c6a56" + ), + "files.txt": ( + "865da0d154344b2c00725f80ba904f968dfbe8da775a2a7542edc0ae04820ce7" + ), + "existing-files.txt": ( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ), + "missing-files.txt": ( + "865da0d154344b2c00725f80ba904f968dfbe8da775a2a7542edc0ae04820ce7" + ), + "source-before.tgz": ( + "12be8d9b8ed064c1d17b67d4a126128a962da0dd82aa34c66532d2d55a927124" + ), + "runtime-before.json": ( + "e65dee775511bc23f8b9238bb85bae7a1bbbc4a1d20d1329ae4b90ba60cef832" + ), +} DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_PATCH_ID = ( "device-edge-core-channel-upgrade-v4-20260812-023" ) @@ -4772,6 +4812,7 @@ def allowed_payload_path(component, rel): DEVICE_PLANE_CONTROL_CORE_RELEASE_V2_REL, DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL, DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, + DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL, DEVICE_PLANE_MANAGER_RECONCILIATION_REL, DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, "packages/device-protocol-contract", @@ -11882,6 +11923,14 @@ def is_device_plane_control_core_v3_reconciliation_slice(component, entries): ) +def is_device_plane_control_core_incident_audit_slice(component, entries): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) == DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES + ) + + def is_device_plane_manager_v2_control_plane_slice(component, entries): return ( component == "device-plane" @@ -12570,6 +12619,57 @@ def expected_device_plane_control_core_v3_reconciliation_descriptor(): } +def expected_device_plane_control_core_incident_audit_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane.device-control-core-incident-audit.v1" + ), + "mode": "double-rollback-failed-read-only-audit", + "allowedOperation": "canonical-plan-only", + "applyAllowed": False, + "failedAttempts": [ + { + "patchId": DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID, + "artifactSha256": ( + DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 + ), + "backupId": ( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID + ), + }, + { + "patchId": ( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID + ), + "artifactSha256": ( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256 + ), + "backupId": ( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID + ), + }, + ], + "readOnlyEvidence": [ + "device-control-core-runtime-inventory", + "device-control-core-bounded-container-logs", + "device-postgres-schema-presence", + "device-postgres-wait-activity", + ], + "runtimeMutation": "none", + "sourceMutation": "none", + "databaseMutation": "none", + "networkMutation": "none", + "secretRead": "none", + "preservedServices": [ + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + } + + def validate_platform_device_core_hub_trust_payload(payload_dir): descriptor = read_strict_json( payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL, @@ -13461,6 +13561,17 @@ def validate_device_plane_control_core_v3_reconciliation_payload(payload_dir): return descriptor +def validate_device_plane_control_core_incident_audit_payload(payload_dir): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL, + "Device Control Core incident audit descriptor", + max_bytes=16 * 1024, + ) + if descriptor != expected_device_plane_control_core_incident_audit_descriptor(): + die("Device Control Core incident audit descriptor mismatch") + return descriptor + + def is_device_plane_postgres_bootstrap_slice(component, entries): return ( component == "device-plane" @@ -16302,6 +16413,276 @@ def validate_device_plane_control_core_v3_reconciliation_evidence(payload_dir): } +def validate_device_plane_control_core_incident_audit_evidence(payload_dir): + descriptor = validate_device_plane_control_core_incident_audit_payload( + payload_dir + ) + first_backup = validate_device_plane_control_core_v3_reconciliation_backup() + first_failed_artifact = ( + FAILED_DIR / DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT + ) + try: + first_failed_stat = first_failed_artifact.lstat() + except FileNotFoundError: + die("Device Control Core v3 failed artifact is missing") + if ( + stat.S_ISLNK(first_failed_stat.st_mode) + or not stat.S_ISREG(first_failed_stat.st_mode) + or sha256_file(first_failed_artifact) + != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 + ): + die("Device Control Core v3 failed artifact evidence mismatch") + first_records = [ + value + for value in load_state(FAILED_STATE_FILE) + if value.get("id") == DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID + ] + if len(first_records) != 1: + die("Device Control Core v3 failed journal evidence count mismatch") + first_record = first_records[0] + if ( + first_record.get("artifact") != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT + or first_record.get("backup_id") + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID + or first_record.get("component") != "device-plane" + or first_record.get("sha256") + != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256 + or first_record.get("started_apply") is not True + or first_record.get("rollback_status") != "failed:DeployError" + or first_record.get("status") != "failed" + ): + die("Device Control Core v3 failed journal evidence mismatch") + validate_device_plane_control_core_v3_restored_source( + first_backup, + marker_installed=False, + ) + validate_device_plane_control_core_v3_reconciliation_runtime( + first_backup, + require_recovered=False, + ) + backup_dir = ( + BACKUPS_DIR + / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID + ) + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Control Core reconciliation failed-apply backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Control Core reconciliation failed-apply backup is unsafe") + expected_backup = ( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_SHA256 + ) + if {child.name for child in backup_dir.iterdir()} != set(expected_backup): + die("Device Control Core reconciliation backup file set mismatch") + for name, expected_sha256 in expected_backup.items(): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Device Control Core reconciliation backup drift detected: " + f"{name}" + ) + existing = read_backup_path_list(backup_dir / "existing-files.txt") + missing = read_backup_path_list(backup_dir / "missing-files.txt") + validate_backup_partition( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES, + existing, + missing, + "Device Control Core incident audit", + ) + if existing or tuple(missing) != ( + DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL, + ): + die("Device Control Core reconciliation backup partition mismatch") + + failed_artifact = ( + FAILED_DIR + / DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT + ) + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Control Core reconciliation failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256 + ): + die("Device Control Core reconciliation failed artifact mismatch") + records = [ + value + for value in load_state(FAILED_STATE_FILE) + if value.get("id") + == DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID + ] + if len(records) != 1: + die("Device Control Core reconciliation failed journal count mismatch") + record = records[0] + if ( + record.get("artifact") + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "failed:DeployError" + or record.get("status") != "failed" + or record.get("message") + != ( + "container healthcheck grace exhausted for " + "8b4b6c6fa1073808f0fafc24450735fe7769a2f92c36ae58dec97d3fa8a30dc0: " + "starting" + ) + ): + die("Device Control Core reconciliation failed journal mismatch") + with tempfile.TemporaryDirectory( + prefix="device-control-core-incident-audit-artifact-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, _failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES + ): + die("Device Control Core reconciliation failed artifact contract mismatch") + current = device_plane_runtime_inventory(( + "device-control-core", + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target", + )) + return { + "mode": descriptor["mode"], + "firstBackup": first_backup, + "secondBackup": backup_dir, + "runtime": current, + } + + +def collect_device_plane_control_core_incident_audit(): + core_ids = device_plane_service_container_ids("device-control-core") + postgres_ids = device_plane_service_container_ids("device-postgres") + if len(core_ids) != 1 or len(postgres_ids) != 1: + die("Device Control Core incident audit container set mismatch") + core_id = core_ids[0] + logs = subprocess.run( + [str(DOCKER), "logs", "--tail", "240", core_id], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if logs.returncode != 0: + die("Device Control Core incident audit log read failed") + bounded_logs = f"{logs.stdout}\n{logs.stderr}" + if len(bounded_logs.encode("utf-8")) > 256 * 1024: + die("Device Control Core incident audit logs exceeded bound") + error_lines = [] + for raw_line in bounded_logs.splitlines(): + line = re.sub(r"\s+", " ", raw_line.strip()) + if not line or not re.search( + r"(?:error|exception|failed|timeout|econn|enoent|eacces|" + r"postgres|relation|constraint|migration|lock|device_[a-z0-9_]+)", + line, + re.IGNORECASE, + ): + continue + if re.search( + r"(?:authorization|bearer|password|private[_ -]?key|token=|secret=)", + line, + re.IGNORECASE, + ): + line = "[sensitive log line redacted]" + line = line[:480] + if line not in error_lines: + error_lines.append(line) + error_lines = error_lines[-16:] + + activity_query = """ +select 'schema' as kind, + case when to_regclass('public.device_infrastructure_host_telemetry_samples') + is null then 'host-telemetry-table-absent' + else 'host-telemetry-table-present' end as detail +union all +select 'activity' as kind, + concat_ws(',', pid::text, state, + coalesce(wait_event_type, 'none'), coalesce(wait_event, 'none'), + case + when query ~* 'device_infrastructure_host_telemetry_samples' + then 'host-telemetry-ddl' + when query ~* '(create|alter|drop)[[:space:]]+(table|index|constraint)' + then 'schema-ddl' + when state = 'idle in transaction' then 'idle-transaction' + else 'application-query' + end, + greatest(0, floor(extract(epoch from (clock_timestamp() - query_start))))::bigint + ) as detail +from pg_stat_activity +where datname = 'device_plane' and pid <> pg_backend_pid() +order by 1, 2 +""".strip() + activity = subprocess.run( + [ + str(DOCKER), + "exec", + postgres_ids[0], + "psql", + "-X", + "-qAt", + "-v", + "ON_ERROR_STOP=1", + "-U", + "device_plane", + "-d", + "device_plane", + "-F", + "\t", + "-c", + activity_query, + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if activity.returncode != 0 or activity.stderr.strip(): + die("Device Control Core incident audit database read failed") + database_evidence = [] + for line in activity.stdout.splitlines(): + normalized = line.strip() + if not re.fullmatch(r"(?:schema|activity)\t[A-Za-z0-9_, -]{1,240}", normalized): + die("Device Control Core incident audit database output invalid") + database_evidence.append(normalized) + if not database_evidence or not any( + item.startswith("schema\t") for item in database_evidence + ): + die("Device Control Core incident audit database evidence missing") + return { + "coreContainerId": core_id, + "logSha256": hashlib.sha256(bounded_logs.encode("utf-8")).hexdigest(), + "logErrors": error_lines, + "database": database_evidence, + } + + def device_plane_service_container_ids(service): if service not in ( *DEVICE_PLANE_RUNTIME_SERVICES, @@ -19695,6 +20076,9 @@ def component_services(component, entries=None): ): return ("device-control-core",) + if is_device_plane_control_core_incident_audit_slice(component, entries): + return () + if is_device_plane_manager_only_release_slice(component, entries): return ("device-manager",) @@ -20301,6 +20685,9 @@ def component_builds(component, entries=None): ): return () + if is_device_plane_control_core_incident_audit_slice(component, entries): + return () + if is_device_plane_manager_only_release_slice(component, entries): return (( DEVICE_PLANE_ROOT / "services/device-manager", @@ -28494,6 +28881,7 @@ def plan_artifact(artifact): device_plane_manager_reconciliation_preflight = None device_plane_manager_v2_reconciliation_preflight = None device_plane_control_core_v3_reconciliation_preflight = None + device_plane_control_core_incident_audit_preflight = None device_plane_backhaul_preflight = None device_plane_backhaul_vps_enrollment_preflight = None device_plane_runtime_before = None @@ -28766,6 +29154,17 @@ def plan_artifact(artifact): payload_dir ) ) + if is_device_plane_control_core_incident_audit_slice( + manifest["component"], + entries, + ): + incident = validate_device_plane_control_core_incident_audit_evidence( + payload_dir + ) + incident["diagnostics"] = ( + collect_device_plane_control_core_incident_audit() + ) + device_plane_control_core_incident_audit_preflight = incident if is_device_plane_backhaul_target_slice( manifest["component"], entries, @@ -29059,7 +29458,10 @@ def plan_artifact(artifact): for build_root, build_args in builds: print(f"build_root={build_root}") print(f"build={' '.join((str(DOCKER),) + tuple(build_args))}") - print(f"services={' '.join(services)}") + if is_device_plane_control_core_incident_audit_slice(component, entries): + print("services=none") + else: + print(f"services={' '.join(services)}") if gitea_preflight: print("gitea_transition=fresh-install-only") print(f"gitea_root={GITEA_ROOT}:required-absent") @@ -30938,6 +31340,49 @@ def plan_artifact(artifact): print("device_gateway_tcp_9921=preserved:loopback-only") print("gelios=untouched-legacy-only") print("device_plane_rollback=marker+exact-preapply-image-runtime") + if device_plane_control_core_incident_audit_preflight is not None: + audit = device_plane_control_core_incident_audit_preflight + diagnostics = audit["diagnostics"] + runtime_by_service = { + item["service"]: item for item in audit["runtime"]["services"] + } + core = runtime_by_service["device-control-core"] + print(f"device_plane_transition={audit['mode']}") + print("allowed_operation=canonical-plan-only") + print("apply=forbidden") + print("device_plane_build=none") + print("device_plane_runtime_mutation=none") + print("device_plane_source_mutation=none") + print("device_postgres_mutation=none") + print("device_plane_network_mutation=none") + print("device_plane_secret_read=none") + print(f"first_failed_backup={audit['firstBackup'].name}") + print(f"second_failed_backup={audit['secondBackup'].name}") + print( + "device_control_core_runtime=" + f"{core['containerId']}:{core['imageId']}:" + f"{core['status']}:{core['health']}:restarts={core['restartCount']}" + ) + print( + "device_control_core_log_sha256=" + f"{diagnostics['logSha256']}" + ) + print( + "device_control_core_log_error_count=" + f"{len(diagnostics['logErrors'])}" + ) + for index, line in enumerate(diagnostics["logErrors"], start=1): + print(f"device_control_core_log_error_{index:02d}={line}") + for index, row in enumerate(diagnostics["database"], start=1): + print( + f"device_postgres_audit_{index:02d}=" + f"{row.replace(chr(9), ':')}" + ) + print( + "device_plane_runtime_services=" + "read-only:device-control-core,device-manager,device-gateway," + "device-postgres,device-backhaul-target" + ) if device_plane_backhaul_vps_enrollment_preflight is not None: print( "device_plane_transition=" @@ -35235,6 +35680,18 @@ def apply_artifact(artifact): entries, ) + if is_device_plane_control_core_incident_audit_slice( + component, + entries, + ): + validate_device_plane_control_core_incident_audit_evidence( + payload_dir + ) + die( + "Device Control Core incident audit is canonical-plan-only; " + "apply is forbidden" + ) + if state_has_sha(sha): die(f"artifact sha already applied: {sha}") if state_has_patch_id(patch_id):