diff --git a/deployment/device-control-core-migration-replay-recovery-v1.json b/deployment/device-control-core-migration-replay-recovery-v1.json new file mode 100644 index 0000000..b77d055 --- /dev/null +++ b/deployment/device-control-core-migration-replay-recovery-v1.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": "nodedc.device-plane.device-control-core-migration-replay-recovery.v1", + "mode": "double-rollback-failed-migration-014-forward-repair", + "failedIncidentAudit": "device-control-core-incident-audit-20260822-043", + "sourcePredecessor": { + "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql", + "sha256": "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508" + }, + "sourceTarget": { + "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql", + "sha256": "38bd86b42828d44c7101d5433ddc36018e92eedeee37b9de296432ad676edd46" + }, + "rootCause": "intermediate-command-kind-check-revalidated-historical-receipts", + "repair": "migration-014-add-constraint-not-valid", + "databasePreflight": "all-live-command-kinds-covered-by-final-migration-016", + "databaseRowMutation": "none", + "databaseSchemaOutcome": "final-migration-016-validated-command-kind-check", + "runtimeAction": "build+recreate-device-control-core-only", + "runtimePredecessor": "proven-degraded-double-rollback-state", + "preservedServices": [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target" + ], + "databaseVolume": "nodedc-device-plane-postgres-data", + "publicIngress": "disabled", + "edgeChannel": "core-initiated-pinned-mtls-registered-edges-only", + "rollback": "source+exact-degraded-predecessor-image-runtime" +} diff --git a/infra/deploy-runner/build-device-control-core-migration-replay-recovery-artifact.mjs b/infra/deploy-runner/build-device-control-core-migration-replay-recovery-artifact.mjs new file mode 100644 index 0000000..2271031 --- /dev/null +++ b/infra/deploy-runner/build-device-control-core-migration-replay-recovery-artifact.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const sourceRoot = resolve(scriptDir, "../.."); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-control-core-migration-replay-recovery-20260822-044", + ...extra +] = process.argv.slice(2); +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-control-core-migration-replay-recovery-artifact.mjs [patch-id]", + ); +} + +const entries = [ + "services/device-control-core/migrations/014_device_registry_profile_commands.sql", + "deployment/device-control-core-migration-replay-recovery-v1.json", +]; +const stage = await mkdtemp(join(tmpdir(), "nodedc-control-core-migration-replay-")); +const payload = join(stage, "payload"); +const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`); + +try { + const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entries[1]), "utf8")); + const targetBytes = await readFile(resolve(sourceRoot, entries[0])); + const targetSha256 = createHash("sha256").update(targetBytes).digest("hex"); + if ( + descriptor.schemaVersion + !== "nodedc.device-plane.device-control-core-migration-replay-recovery.v1" + || descriptor.mode !== "double-rollback-failed-migration-014-forward-repair" + || descriptor.sourceTarget.path !== entries[0] + || descriptor.sourceTarget.sha256 !== targetSha256 + || descriptor.databaseRowMutation !== "none" + || descriptor.runtimeAction !== "build+recreate-device-control-core-only" + ) { + throw new Error("device_control_core_migration_replay_recovery_descriptor_mismatch"); + } + + for (const entry of entries) { + await mkdir(dirname(join(payload, entry)), { recursive: true }); + await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true }); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, + ); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + const sha256 = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + component: "device-plane", + artifact: target, + sha256, + entries, + build: ["nodedc/device-control-core:local"], + services: ["device-control-core"], + transition: descriptor.mode, + databaseRowMutation: descriptor.databaseRowMutation, + runtimeAction: descriptor.runtimeAction, + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} diff --git a/infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py b/infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py new file mode 100644 index 0000000..f5ec7f4 --- /dev/null +++ b/infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import inspect +import json +import os +import re +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +RUNNER_PATH = Path( + os.environ.get( + "NODEDC_DEPLOY_RUNNER", + SCRIPT_DIR.parents[2] / "platform/infra/deploy-runner/nodedc-deploy", + ) +) +BUILDER = ( + SCRIPT_DIR + / "build-device-control-core-migration-replay-recovery-artifact.mjs" +) + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_control_core_migration_replay_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase): + def build(self, artifact_dir): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + [ + "node", + str(BUILDER), + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID, + ], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_artifact_is_deterministic_exact_minimal_repair(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-control-core-migration-replay-recovery-", + ) as directory: + artifact_dir = Path(directory) + first = self.build(artifact_dir) + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build(artifact_dir) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + self.assertEqual(first["services"], ["device-control-core"]) + self.assertEqual(first["databaseRowMutation"], "none") + self.assertEqual( + first["entries"], + list( + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES + ), + ) + with tarfile.open(first["artifact"], "r:gz") as archive: + descriptor = json.loads( + archive.extractfile( + "payload/" + + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL + ).read().decode("utf-8") + ) + migration = archive.extractfile( + "payload/" + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + ).read() + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_control_core_migration_replay_recovery_descriptor(), + ) + self.assertEqual( + hashlib.sha256(migration).hexdigest(), + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256, + ) + self.assertIn(b")) not valid;", migration.lower()) + self.assertIsNone( + re.search( + rb"(?im)^\s*(?:delete|update|insert|truncate)\b", + migration, + ) + ) + + def test_runner_selects_only_core_and_exact_build(self): + entries = ( + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES + ) + self.assertEqual( + RUNNER.component_services("device-plane", entries), + ("device-control-core",), + ) + builds = RUNNER.component_builds("device-plane", entries) + self.assertEqual(len(builds), 1) + self.assertEqual(builds[0][0], RUNNER.DEVICE_PLANE_ROOT) + self.assertEqual( + builds[0][1], + ( + "build", + "--no-cache", + "--network=host", + "-f", + "services/device-control-core/Dockerfile", + "-t", + RUNNER.DEVICE_PLANE_CONTROL_CORE_IMAGE, + ".", + ), + ) + + def test_database_preflight_is_read_only_and_covers_live_receipts(self): + database_result = mock.Mock( + returncode=0, + stdout="0\t3\ttrue\ttrue\ttrue\n", + stderr="", + ) + with mock.patch.object( + RUNNER, + "device_plane_service_container_ids", + return_value=("p" * 64,), + ), mock.patch.object( + RUNNER.subprocess, + "run", + return_value=database_result, + ) as run: + evidence = ( + RUNNER.collect_device_plane_control_core_migration_replay_database_evidence() + ) + + self.assertEqual(evidence["invalidCommandKindCount"], 0) + self.assertEqual(evidence["triggeringReceiptCount"], 3) + command = run.call_args.args[0] + query = command[command.index("-c") + 1] + self.assertTrue(query.lstrip().lower().startswith("with constraint_state")) + self.assertNotIn(";", query) + self.assertIsNone( + re.search( + r"(?im)^\s*(?:delete|update|insert|truncate|alter|drop|create)\b", + query, + ) + ) + + def test_rollback_accepts_exact_degraded_predecessor_boundary(self): + rollback_source = inspect.getsource(RUNNER.rollback_device_plane_apply) + acceptance_source = inspect.getsource( + RUNNER.accept_device_plane_control_core_rollback_runtime + ) + self.assertIn( + "is_device_plane_control_core_migration_replay_recovery_slice", + rollback_source, + ) + self.assertIn("retag_device_plane_control_core_image", rollback_source) + self.assertIn("not predecessor_was_healthy", acceptance_source) + self.assertIn("changed preserved service", acceptance_source) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/device-control-core/migrations/014_device_registry_profile_commands.sql b/services/device-control-core/migrations/014_device_registry_profile_commands.sql index d91e2ac..7396911 100644 --- a/services/device-control-core/migrations/014_device_registry_profile_commands.sql +++ b/services/device-control-core/migrations/014_device_registry_profile_commands.sql @@ -27,6 +27,6 @@ alter table device_management_command_receipts 'device_binding.revoke', 'device_configuration_revision.create', 'device_configuration_desired.set' - )); + )) not valid; commit; diff --git a/services/device-control-core/test/management-command-migration.test.mjs b/services/device-control-core/test/management-command-migration.test.mjs index 826a6dd..cc14fed 100644 --- a/services/device-control-core/test/management-command-migration.test.mjs +++ b/services/device-control-core/test/management-command-migration.test.mjs @@ -7,6 +7,7 @@ const replayedIntermediateConstraintMigrations = Object.freeze([ "007_device_lifecycle_commands.sql", "009_device_sensitive_reference_commands.sql", "011_device_control_resource_commands.sql", + "014_device_registry_profile_commands.sql", ]); const finalCommandKindMigration = "016_device_asset_infrastructure_ontology.sql"; const finalCommandKinds = Object.freeze([