fix(device-core): make migration replay recovery safe

deployment/device-control-core-migration-replay-recovery-v1.json

@@ -0,0 +1,30 @@
+{
+  "schemaVersion": "nodedc.device-plane.device-control-core-migration-replay-recovery.v1",
+  "mode": "double-rollback-failed-migration-014-forward-repair",
+  "failedIncidentAudit": "device-control-core-incident-audit-20260822-043",
+  "sourcePredecessor": {
+    "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
+    "sha256": "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
+  },
+  "sourceTarget": {
+    "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
+    "sha256": "38bd86b42828d44c7101d5433ddc36018e92eedeee37b9de296432ad676edd46"
+  },
+  "rootCause": "intermediate-command-kind-check-revalidated-historical-receipts",
+  "repair": "migration-014-add-constraint-not-valid",
+  "databasePreflight": "all-live-command-kinds-covered-by-final-migration-016",
+  "databaseRowMutation": "none",
+  "databaseSchemaOutcome": "final-migration-016-validated-command-kind-check",
+  "runtimeAction": "build+recreate-device-control-core-only",
+  "runtimePredecessor": "proven-degraded-double-rollback-state",
+  "preservedServices": [
+    "device-manager",
+    "device-gateway",
+    "device-postgres",
+    "device-backhaul-target"
+  ],
+  "databaseVolume": "nodedc-device-plane-postgres-data",
+  "publicIngress": "disabled",
+  "edgeChannel": "core-initiated-pinned-mtls-registered-edges-only",
+  "rollback": "source+exact-degraded-predecessor-image-runtime"
+}

infra/deploy-runner/build-device-control-core-migration-replay-recovery-artifact.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const sourceRoot = resolve(scriptDir, "../..");
+const artifactDir = resolve(
+  process.env.NODEDC_DEPLOY_ARTIFACT_DIR
+    || resolve(scriptDir, "../deploy-artifacts"),
+);
+const [
+  patchId = "device-control-core-migration-replay-recovery-20260822-044",
+  ...extra
+] = process.argv.slice(2);
+if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
+  throw new Error(
+    "usage: build-device-control-core-migration-replay-recovery-artifact.mjs [patch-id]",
+  );
+}
+
+const entries = [
+  "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
+  "deployment/device-control-core-migration-replay-recovery-v1.json",
+];
+const stage = await mkdtemp(join(tmpdir(), "nodedc-control-core-migration-replay-"));
+const payload = join(stage, "payload");
+const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`);
+
+try {
+  const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entries[1]), "utf8"));
+  const targetBytes = await readFile(resolve(sourceRoot, entries[0]));
+  const targetSha256 = createHash("sha256").update(targetBytes).digest("hex");
+  if (
+    descriptor.schemaVersion
+      !== "nodedc.device-plane.device-control-core-migration-replay-recovery.v1"
+    || descriptor.mode !== "double-rollback-failed-migration-014-forward-repair"
+    || descriptor.sourceTarget.path !== entries[0]
+    || descriptor.sourceTarget.sha256 !== targetSha256
+    || descriptor.databaseRowMutation !== "none"
+    || descriptor.runtimeAction !== "build+recreate-device-control-core-only"
+  ) {
+    throw new Error("device_control_core_migration_replay_recovery_descriptor_mismatch");
+  }
+
+  for (const entry of entries) {
+    await mkdir(dirname(join(payload, entry)), { recursive: true });
+    await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true });
+  }
+  await writeFile(
+    join(stage, "manifest.env"),
+    `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
+    "utf8",
+  );
+  await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8");
+  await mkdir(artifactDir, { recursive: true });
+  const tar = spawnSync(
+    "python3",
+    ["-c", canonicalTarScript(), target, stage],
+    { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 },
+  );
+  if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
+  const sha256 = createHash("sha256")
+    .update(await readFile(target))
+    .digest("hex");
+  console.log(JSON.stringify({
+    ok: true,
+    patchId,
+    component: "device-plane",
+    artifact: target,
+    sha256,
+    entries,
+    build: ["nodedc/device-control-core:local"],
+    services: ["device-control-core"],
+    transition: descriptor.mode,
+    databaseRowMutation: descriptor.databaseRowMutation,
+    runtimeAction: descriptor.runtimeAction,
+  }, null, 2));
+} finally {
+  await rm(stage, { recursive: true, force: true });
+}
+
+function canonicalTarScript() {
+  return [
+    "import gzip,io,pathlib,sys,tarfile",
+    "root=pathlib.Path(sys.argv[2])",
+    "with open(sys.argv[1],'wb') as out:",
+    " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
+    "  with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
+    "   for top in ('manifest.env','files.txt','payload'):",
+    "    p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
+    "    for x in paths:",
+    "     info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
+    "     with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
+  ].join("\n");
+}

infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+import hashlib
+import importlib.machinery
+import importlib.util
+import inspect
+import json
+import os
+import re
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+RUNNER_PATH = Path(
+    os.environ.get(
+        "NODEDC_DEPLOY_RUNNER",
+        SCRIPT_DIR.parents[2] / "platform/infra/deploy-runner/nodedc-deploy",
+    )
+)
+BUILDER = (
+    SCRIPT_DIR
+    / "build-device-control-core-migration-replay-recovery-artifact.mjs"
+)
+
+
+def load_runner():
+    loader = importlib.machinery.SourceFileLoader(
+        "nodedc_control_core_migration_replay_runner_under_test",
+        str(RUNNER_PATH),
+    )
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    module = importlib.util.module_from_spec(spec)
+    loader.exec_module(module)
+    return module
+
+
+RUNNER = load_runner()
+
+
+class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
+    def build(self, artifact_dir):
+        environment = os.environ.copy()
+        environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
+        result = subprocess.run(
+            [
+                "node",
+                str(BUILDER),
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID,
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+            env=environment,
+        )
+        return json.loads(result.stdout)
+
+    def test_artifact_is_deterministic_exact_minimal_repair(self):
+        with tempfile.TemporaryDirectory(
+            prefix="nodedc-control-core-migration-replay-recovery-",
+        ) as directory:
+            artifact_dir = Path(directory)
+            first = self.build(artifact_dir)
+            first_bytes = Path(first["artifact"]).read_bytes()
+            second = self.build(artifact_dir)
+            second_bytes = Path(second["artifact"]).read_bytes()
+
+            self.assertEqual(first_bytes, second_bytes)
+            self.assertEqual(
+                first["sha256"],
+                hashlib.sha256(first_bytes).hexdigest(),
+            )
+            self.assertEqual(first["services"], ["device-control-core"])
+            self.assertEqual(first["databaseRowMutation"], "none")
+            self.assertEqual(
+                first["entries"],
+                list(
+                    RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES
+                ),
+            )
+            with tarfile.open(first["artifact"], "r:gz") as archive:
+                descriptor = json.loads(
+                    archive.extractfile(
+                        "payload/"
+                        + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_REL
+                    ).read().decode("utf-8")
+                )
+                migration = archive.extractfile(
+                    "payload/" + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
+                ).read()
+            self.assertEqual(
+                descriptor,
+                RUNNER.expected_device_plane_control_core_migration_replay_recovery_descriptor(),
+            )
+            self.assertEqual(
+                hashlib.sha256(migration).hexdigest(),
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256,
+            )
+            self.assertIn(b")) not valid;", migration.lower())
+            self.assertIsNone(
+                re.search(
+                    rb"(?im)^\s*(?:delete|update|insert|truncate)\b",
+                    migration,
+                )
+            )
+
+    def test_runner_selects_only_core_and_exact_build(self):
+        entries = (
+            RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ENTRIES
+        )
+        self.assertEqual(
+            RUNNER.component_services("device-plane", entries),
+            ("device-control-core",),
+        )
+        builds = RUNNER.component_builds("device-plane", entries)
+        self.assertEqual(len(builds), 1)
+        self.assertEqual(builds[0][0], RUNNER.DEVICE_PLANE_ROOT)
+        self.assertEqual(
+            builds[0][1],
+            (
+                "build",
+                "--no-cache",
+                "--network=host",
+                "-f",
+                "services/device-control-core/Dockerfile",
+                "-t",
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_IMAGE,
+                ".",
+            ),
+        )
+
+    def test_database_preflight_is_read_only_and_covers_live_receipts(self):
+        database_result = mock.Mock(
+            returncode=0,
+            stdout="0\t3\ttrue\ttrue\ttrue\n",
+            stderr="",
+        )
+        with mock.patch.object(
+            RUNNER,
+            "device_plane_service_container_ids",
+            return_value=("p" * 64,),
+        ), mock.patch.object(
+            RUNNER.subprocess,
+            "run",
+            return_value=database_result,
+        ) as run:
+            evidence = (
+                RUNNER.collect_device_plane_control_core_migration_replay_database_evidence()
+            )
+
+        self.assertEqual(evidence["invalidCommandKindCount"], 0)
+        self.assertEqual(evidence["triggeringReceiptCount"], 3)
+        command = run.call_args.args[0]
+        query = command[command.index("-c") + 1]
+        self.assertTrue(query.lstrip().lower().startswith("with constraint_state"))
+        self.assertNotIn(";", query)
+        self.assertIsNone(
+            re.search(
+                r"(?im)^\s*(?:delete|update|insert|truncate|alter|drop|create)\b",
+                query,
+            )
+        )
+
+    def test_rollback_accepts_exact_degraded_predecessor_boundary(self):
+        rollback_source = inspect.getsource(RUNNER.rollback_device_plane_apply)
+        acceptance_source = inspect.getsource(
+            RUNNER.accept_device_plane_control_core_rollback_runtime
+        )
+        self.assertIn(
+            "is_device_plane_control_core_migration_replay_recovery_slice",
+            rollback_source,
+        )
+        self.assertIn("retag_device_plane_control_core_image", rollback_source)
+        self.assertIn("not predecessor_was_healthy", acceptance_source)
+        self.assertIn("changed preserved service", acceptance_source)
+
+
+if __name__ == "__main__":
+    unittest.main()

services/device-control-core/migrations/014_device_registry_profile_commands.sql

@@ -27,6 +27,6 @@ alter table device_management_command_receipts
     'device_binding.revoke',
     'device_configuration_revision.create',
     'device_configuration_desired.set'
-  ));
+  )) not valid;
 
 commit;

services/device-control-core/test/management-command-migration.test.mjs

@@ -7,6 +7,7 @@ const replayedIntermediateConstraintMigrations = Object.freeze([
   "007_device_lifecycle_commands.sql",
   "009_device_sensitive_reference_commands.sql",
   "011_device_control_resource_commands.sql",
+  "014_device_registry_profile_commands.sql",
 ]);
 const finalCommandKindMigration = "016_device_asset_infrastructure_ontology.sql";
 const finalCommandKinds = Object.freeze([