SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore(deploy): register read-only Core incident audit

Browse Source
This commit is contained in:
Codex
2026-08-22 20:44:15 +03:00
parent 8d501224b4
commit bd715e9d71
1 changed files with 458 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/nodedc-deploy
+458 -1
View File
@@ -441,6 +441,46 @@ DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_MISSING = (
"packages/infrastructure-telemetry-contract",
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
)
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL = (
"deployment/device-control-core-incident-audit-v1.json"
)
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES = (
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
)
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_PATCH_ID = (
"device-control-core-incident-audit-20260822-043"
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT = (
"nodedc-device-plane-device-control-core-release-v3-reconciliation-"
"20260822-042.tgz.20260822-195448"
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256 = (
"54ab243439bce724fa0a0872b76cc32e0052ea5127153214d92872f02ae831cf"
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID = (
"device-plane-device-control-core-release-v3-reconciliation-"
"20260822-042-20260822-195448"
)
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_SHA256 = {
"manifest.env": (
"37ed634d7ebde9d7da67990fe20cae7809a682c34652c37e163f9c2bc01c6a56"
),
"files.txt": (
"865da0d154344b2c00725f80ba904f968dfbe8da775a2a7542edc0ae04820ce7"
),
"existing-files.txt": (
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
),
"missing-files.txt": (
"865da0d154344b2c00725f80ba904f968dfbe8da775a2a7542edc0ae04820ce7"
),
"source-before.tgz": (
"12be8d9b8ed064c1d17b67d4a126128a962da0dd82aa34c66532d2d55a927124"
),
"runtime-before.json": (
"e65dee775511bc23f8b9238bb85bae7a1bbbc4a1d20d1329ae4b90ba60cef832"
),
}
DEVICE_PLANE_CONTROL_CORE_RELEASE_FIRST_PREDECESSOR_PATCH_ID = (
"device-edge-core-channel-upgrade-v4-20260812-023"
)
@@ -4772,6 +4812,7 @@ def allowed_payload_path(component, rel):
DEVICE_PLANE_CONTROL_CORE_RELEASE_V2_REL,
DEVICE_PLANE_CONTROL_CORE_RELEASE_V3_REL,
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
"packages/device-protocol-contract",
@@ -11882,6 +11923,14 @@ def is_device_plane_control_core_v3_reconciliation_slice(component, entries):
)
def is_device_plane_control_core_incident_audit_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries) == DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_ENTRIES
)
def is_device_plane_manager_v2_control_plane_slice(component, entries):
return (
component == "device-plane"
@@ -12570,6 +12619,57 @@ def expected_device_plane_control_core_v3_reconciliation_descriptor():
}
def expected_device_plane_control_core_incident_audit_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane.device-control-core-incident-audit.v1"
),
"mode": "double-rollback-failed-read-only-audit",
"allowedOperation": "canonical-plan-only",
"applyAllowed": False,
"failedAttempts": [
{
"patchId": DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID,
"artifactSha256": (
DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256
),
"backupId": (
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID
),
},
{
"patchId": (
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID
),
"artifactSha256": (
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256
),
"backupId": (
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID
),
},
],
"readOnlyEvidence": [
"device-control-core-runtime-inventory",
"device-control-core-bounded-container-logs",
"device-postgres-schema-presence",
"device-postgres-wait-activity",
],
"runtimeMutation": "none",
"sourceMutation": "none",
"databaseMutation": "none",
"networkMutation": "none",
"secretRead": "none",
"preservedServices": [
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
}
def validate_platform_device_core_hub_trust_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
@@ -13461,6 +13561,17 @@ def validate_device_plane_control_core_v3_reconciliation_payload(payload_dir):
return descriptor
def validate_device_plane_control_core_incident_audit_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_CONTROL_CORE_INCIDENT_AUDIT_REL,
"Device Control Core incident audit descriptor",
max_bytes=16 * 1024,
)
if descriptor != expected_device_plane_control_core_incident_audit_descriptor():
die("Device Control Core incident audit descriptor mismatch")
return descriptor
def is_device_plane_postgres_bootstrap_slice(component, entries):
return (
component == "device-plane"
@@ -16302,6 +16413,276 @@ def validate_device_plane_control_core_v3_reconciliation_evidence(payload_dir):
}
def validate_device_plane_control_core_incident_audit_evidence(payload_dir):
descriptor = validate_device_plane_control_core_incident_audit_payload(
payload_dir
)
first_backup = validate_device_plane_control_core_v3_reconciliation_backup()
first_failed_artifact = (
FAILED_DIR / DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT
)
try:
first_failed_stat = first_failed_artifact.lstat()
except FileNotFoundError:
die("Device Control Core v3 failed artifact is missing")
if (
stat.S_ISLNK(first_failed_stat.st_mode)
or not stat.S_ISREG(first_failed_stat.st_mode)
or sha256_file(first_failed_artifact)
!= DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256
):
die("Device Control Core v3 failed artifact evidence mismatch")
first_records = [
value
for value in load_state(FAILED_STATE_FILE)
if value.get("id") == DEVICE_PLANE_CONTROL_CORE_V3_FAILED_PATCH_ID
]
if len(first_records) != 1:
die("Device Control Core v3 failed journal evidence count mismatch")
first_record = first_records[0]
if (
first_record.get("artifact") != DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT
or first_record.get("backup_id")
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_BACKUP_ID
or first_record.get("component") != "device-plane"
or first_record.get("sha256")
!= DEVICE_PLANE_CONTROL_CORE_V3_FAILED_ARTIFACT_SHA256
or first_record.get("started_apply") is not True
or first_record.get("rollback_status") != "failed:DeployError"
or first_record.get("status") != "failed"
):
die("Device Control Core v3 failed journal evidence mismatch")
validate_device_plane_control_core_v3_restored_source(
first_backup,
marker_installed=False,
)
validate_device_plane_control_core_v3_reconciliation_runtime(
first_backup,
require_recovered=False,
)
backup_dir = (
BACKUPS_DIR
/ DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID
)
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Device Control Core reconciliation failed-apply backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Device Control Core reconciliation failed-apply backup is unsafe")
expected_backup = (
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_SHA256
)
if {child.name for child in backup_dir.iterdir()} != set(expected_backup):
die("Device Control Core reconciliation backup file set mismatch")
for name, expected_sha256 in expected_backup.items():
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected_sha256
):
die(
"Device Control Core reconciliation backup drift detected: "
f"{name}"
)
existing = read_backup_path_list(backup_dir / "existing-files.txt")
missing = read_backup_path_list(backup_dir / "missing-files.txt")
validate_backup_partition(
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES,
existing,
missing,
"Device Control Core incident audit",
)
if existing or tuple(missing) != (
DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_REL,
):
die("Device Control Core reconciliation backup partition mismatch")
failed_artifact = (
FAILED_DIR
/ DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT
)
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Control Core reconciliation failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256
):
die("Device Control Core reconciliation failed artifact mismatch")
records = [
value
for value in load_state(FAILED_STATE_FILE)
if value.get("id")
== DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID
]
if len(records) != 1:
die("Device Control Core reconciliation failed journal count mismatch")
record = records[0]
if (
record.get("artifact")
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT
or record.get("backup_id")
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_BACKUP_ID
or record.get("component") != "device-plane"
or record.get("sha256")
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status") != "failed:DeployError"
or record.get("status") != "failed"
or record.get("message")
!= (
"container healthcheck grace exhausted for "
"8b4b6c6fa1073808f0fafc24450735fe7769a2f92c36ae58dec97d3fa8a30dc0: "
"starting"
)
):
die("Device Control Core reconciliation failed journal mismatch")
with tempfile.TemporaryDirectory(
prefix="device-control-core-incident-audit-artifact-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, _failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= DEVICE_PLANE_CONTROL_CORE_V3_RECONCILIATION_ENTRIES
):
die("Device Control Core reconciliation failed artifact contract mismatch")
current = device_plane_runtime_inventory((
"device-control-core",
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target",
))
return {
"mode": descriptor["mode"],
"firstBackup": first_backup,
"secondBackup": backup_dir,
"runtime": current,
}
def collect_device_plane_control_core_incident_audit():
core_ids = device_plane_service_container_ids("device-control-core")
postgres_ids = device_plane_service_container_ids("device-postgres")
if len(core_ids) != 1 or len(postgres_ids) != 1:
die("Device Control Core incident audit container set mismatch")
core_id = core_ids[0]
logs = subprocess.run(
[str(DOCKER), "logs", "--tail", "240", core_id],
check=False,
capture_output=True,
text=True,
timeout=30,
)
if logs.returncode != 0:
die("Device Control Core incident audit log read failed")
bounded_logs = f"{logs.stdout}\n{logs.stderr}"
if len(bounded_logs.encode("utf-8")) > 256 * 1024:
die("Device Control Core incident audit logs exceeded bound")
error_lines = []
for raw_line in bounded_logs.splitlines():
line = re.sub(r"\s+", " ", raw_line.strip())
if not line or not re.search(
r"(?:error|exception|failed|timeout|econn|enoent|eacces|"
r"postgres|relation|constraint|migration|lock|device_[a-z0-9_]+)",
line,
re.IGNORECASE,
):
continue
if re.search(
r"(?:authorization|bearer|password|private[_ -]?key|token=|secret=)",
line,
re.IGNORECASE,
):
line = "[sensitive log line redacted]"
line = line[:480]
if line not in error_lines:
error_lines.append(line)
error_lines = error_lines[-16:]
activity_query = """
select 'schema' as kind,
case when to_regclass('public.device_infrastructure_host_telemetry_samples')
is null then 'host-telemetry-table-absent'
else 'host-telemetry-table-present' end as detail
union all
select 'activity' as kind,
concat_ws(',', pid::text, state,
coalesce(wait_event_type, 'none'), coalesce(wait_event, 'none'),
case
when query ~* 'device_infrastructure_host_telemetry_samples'
then 'host-telemetry-ddl'
when query ~* '(create|alter|drop)[[:space:]]+(table|index|constraint)'
then 'schema-ddl'
when state = 'idle in transaction' then 'idle-transaction'
else 'application-query'
end,
greatest(0, floor(extract(epoch from (clock_timestamp() - query_start))))::bigint
) as detail
from pg_stat_activity
where datname = 'device_plane' and pid <> pg_backend_pid()
order by 1, 2
""".strip()
activity = subprocess.run(
[
str(DOCKER),
"exec",
postgres_ids[0],
"psql",
"-X",
"-qAt",
"-v",
"ON_ERROR_STOP=1",
"-U",
"device_plane",
"-d",
"device_plane",
"-F",
"\t",
"-c",
activity_query,
],
check=False,
capture_output=True,
text=True,
timeout=30,
)
if activity.returncode != 0 or activity.stderr.strip():
die("Device Control Core incident audit database read failed")
database_evidence = []
for line in activity.stdout.splitlines():
normalized = line.strip()
if not re.fullmatch(r"(?:schema|activity)\t[A-Za-z0-9_, -]{1,240}", normalized):
die("Device Control Core incident audit database output invalid")
database_evidence.append(normalized)
if not database_evidence or not any(
item.startswith("schema\t") for item in database_evidence
):
die("Device Control Core incident audit database evidence missing")
return {
"coreContainerId": core_id,
"logSha256": hashlib.sha256(bounded_logs.encode("utf-8")).hexdigest(),
"logErrors": error_lines,
"database": database_evidence,
}
def device_plane_service_container_ids(service):
if service not in (
*DEVICE_PLANE_RUNTIME_SERVICES,
@@ -19695,6 +20076,9 @@ def component_services(component, entries=None):
):
return ("device-control-core",)
if is_device_plane_control_core_incident_audit_slice(component, entries):
return ()
if is_device_plane_manager_only_release_slice(component, entries):
return ("device-manager",)
@@ -20301,6 +20685,9 @@ def component_builds(component, entries=None):
):
return ()
if is_device_plane_control_core_incident_audit_slice(component, entries):
return ()
if is_device_plane_manager_only_release_slice(component, entries):
return ((
DEVICE_PLANE_ROOT / "services/device-manager",
@@ -28494,6 +28881,7 @@ def plan_artifact(artifact):
device_plane_manager_reconciliation_preflight = None
device_plane_manager_v2_reconciliation_preflight = None
device_plane_control_core_v3_reconciliation_preflight = None
device_plane_control_core_incident_audit_preflight = None
device_plane_backhaul_preflight = None
device_plane_backhaul_vps_enrollment_preflight = None
device_plane_runtime_before = None
@@ -28766,6 +29154,17 @@ def plan_artifact(artifact):
payload_dir
)
)
if is_device_plane_control_core_incident_audit_slice(
manifest["component"],
entries,
):
incident = validate_device_plane_control_core_incident_audit_evidence(
payload_dir
)
incident["diagnostics"] = (
collect_device_plane_control_core_incident_audit()
)
device_plane_control_core_incident_audit_preflight = incident
if is_device_plane_backhaul_target_slice(
manifest["component"],
entries,
@@ -29059,7 +29458,10 @@ def plan_artifact(artifact):
for build_root, build_args in builds:
print(f"build_root={build_root}")
print(f"build={' '.join((str(DOCKER),) + tuple(build_args))}")
print(f"services={' '.join(services)}")
if is_device_plane_control_core_incident_audit_slice(component, entries):
print("services=none")
else:
print(f"services={' '.join(services)}")
if gitea_preflight:
print("gitea_transition=fresh-install-only")
print(f"gitea_root={GITEA_ROOT}:required-absent")
@@ -30938,6 +31340,49 @@ def plan_artifact(artifact):
print("device_gateway_tcp_9921=preserved:loopback-only")
print("gelios=untouched-legacy-only")
print("device_plane_rollback=marker+exact-preapply-image-runtime")
if device_plane_control_core_incident_audit_preflight is not None:
audit = device_plane_control_core_incident_audit_preflight
diagnostics = audit["diagnostics"]
runtime_by_service = {
item["service"]: item for item in audit["runtime"]["services"]
}
core = runtime_by_service["device-control-core"]
print(f"device_plane_transition={audit['mode']}")
print("allowed_operation=canonical-plan-only")
print("apply=forbidden")
print("device_plane_build=none")
print("device_plane_runtime_mutation=none")
print("device_plane_source_mutation=none")
print("device_postgres_mutation=none")
print("device_plane_network_mutation=none")
print("device_plane_secret_read=none")
print(f"first_failed_backup={audit['firstBackup'].name}")
print(f"second_failed_backup={audit['secondBackup'].name}")
print(
"device_control_core_runtime="
f"{core['containerId']}:{core['imageId']}:"
f"{core['status']}:{core['health']}:restarts={core['restartCount']}"
)
print(
"device_control_core_log_sha256="
f"{diagnostics['logSha256']}"
)
print(
"device_control_core_log_error_count="
f"{len(diagnostics['logErrors'])}"
)
for index, line in enumerate(diagnostics["logErrors"], start=1):
print(f"device_control_core_log_error_{index:02d}={line}")
for index, row in enumerate(diagnostics["database"], start=1):
print(
f"device_postgres_audit_{index:02d}="
f"{row.replace(chr(9), ':')}"
)
print(
"device_plane_runtime_services="
"read-only:device-control-core,device-manager,device-gateway,"
"device-postgres,device-backhaul-target"
)
if device_plane_backhaul_vps_enrollment_preflight is not None:
print(
"device_plane_transition="
@@ -35235,6 +35680,18 @@ def apply_artifact(artifact):
entries,
)
if is_device_plane_control_core_incident_audit_slice(
component,
entries,
):
validate_device_plane_control_core_incident_audit_evidence(
payload_dir
)
die(
"Device Control Core incident audit is canonical-plan-only; "
"apply is forbidden"
)
if state_has_sha(sha):
die(f"artifact sha already applied: {sha}")
if state_has_patch_id(patch_id):