fix(deploy): add replay checkpoint recovery

deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json

@@ -0,0 +1,36 @@
+{
+  "schemaVersion": "nodedc.device-plane.device-control-core-migration-replay-checkpoint-recovery.v2",
+  "mode": "terminal-044-replay-checkpoint-forward-repair",
+  "failedIncidentAudit": "device-control-core-incident-audit-20260822-043",
+  "failedRecovery": {
+    "patchId": "device-control-core-migration-replay-recovery-20260822-044",
+    "artifactSha256": "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7",
+    "failure": "preflight-replay-checkpoint-race",
+    "startedApply": false
+  },
+  "sourcePredecessor": {
+    "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
+    "sha256": "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
+  },
+  "sourceTarget": {
+    "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
+    "sha256": "38bd86b42828d44c7101d5433ddc36018e92eedeee37b9de296432ad676edd46"
+  },
+  "rootCause": "restarting-core-cycles-exact-committed-migration-checkpoints",
+  "repair": "migration-014-add-constraint-not-valid",
+  "databasePreflight": "exact-replay-checkpoint-005-007-009-011-and-final-compatible-rows",
+  "databaseRowMutation": "none",
+  "databaseSchemaOutcome": "exact-final-migration-016-validated-command-kind-check",
+  "runtimeAction": "build+recreate-device-control-core-only",
+  "runtimePredecessor": "proven-degraded-restarting-exact-preapply-image",
+  "preservedServices": [
+    "device-manager",
+    "device-gateway",
+    "device-postgres",
+    "device-backhaul-target"
+  ],
+  "databaseVolume": "nodedc-device-plane-postgres-data",
+  "publicIngress": "disabled",
+  "edgeChannel": "core-initiated-pinned-mtls-registered-edges-only",
+  "rollback": "source+exact-degraded-predecessor-image-runtime"
+}

infra/deploy-runner/build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const scriptDir = dirname(fileURLToPath(import.meta.url));
+const sourceRoot = resolve(scriptDir, "../..");
+const artifactDir = resolve(
+  process.env.NODEDC_DEPLOY_ARTIFACT_DIR
+    || resolve(scriptDir, "../deploy-artifacts"),
+);
+const [
+  patchId = "device-control-core-migration-replay-checkpoint-recovery-20260822-046",
+  ...extra
+] = process.argv.slice(2);
+if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
+  throw new Error(
+    "usage: build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs [patch-id]",
+  );
+}
+
+const entries = [
+  "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
+  "deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json",
+];
+const stage = await mkdtemp(join(tmpdir(), "nodedc-control-core-replay-checkpoint-"));
+const payload = join(stage, "payload");
+const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`);
+
+try {
+  const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entries[1]), "utf8"));
+  const targetBytes = await readFile(resolve(sourceRoot, entries[0]));
+  const targetSha256 = createHash("sha256").update(targetBytes).digest("hex");
+  if (
+    descriptor.schemaVersion
+      !== "nodedc.device-plane.device-control-core-migration-replay-checkpoint-recovery.v2"
+    || descriptor.mode !== "terminal-044-replay-checkpoint-forward-repair"
+    || descriptor.failedRecovery?.patchId
+      !== "device-control-core-migration-replay-recovery-20260822-044"
+    || descriptor.failedRecovery?.artifactSha256
+      !== "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7"
+    || descriptor.failedRecovery?.startedApply !== false
+    || descriptor.sourceTarget.path !== entries[0]
+    || descriptor.sourceTarget.sha256 !== targetSha256
+    || descriptor.databaseRowMutation !== "none"
+    || descriptor.runtimeAction !== "build+recreate-device-control-core-only"
+  ) {
+    throw new Error(
+      "device_control_core_migration_replay_checkpoint_recovery_descriptor_mismatch",
+    );
+  }
+
+  for (const entry of entries) {
+    await mkdir(dirname(join(payload, entry)), { recursive: true });
+    await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true });
+  }
+  await writeFile(
+    join(stage, "manifest.env"),
+    `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
+    "utf8",
+  );
+  await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8");
+  await mkdir(artifactDir, { recursive: true });
+  const tar = spawnSync(
+    "python3",
+    ["-c", canonicalTarScript(), target, stage],
+    { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 },
+  );
+  if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
+  const sha256 = createHash("sha256")
+    .update(await readFile(target))
+    .digest("hex");
+  console.log(JSON.stringify({
+    ok: true,
+    patchId,
+    component: "device-plane",
+    artifact: target,
+    sha256,
+    entries,
+    build: ["nodedc/device-control-core:local"],
+    services: ["device-control-core"],
+    transition: descriptor.mode,
+    databaseRowMutation: descriptor.databaseRowMutation,
+    runtimeAction: descriptor.runtimeAction,
+  }, null, 2));
+} finally {
+  await rm(stage, { recursive: true, force: true });
+}
+
+function canonicalTarScript() {
+  return [
+    "import gzip,io,pathlib,sys,tarfile",
+    "root=pathlib.Path(sys.argv[2])",
+    "with open(sys.argv[1],'wb') as out:",
+    " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
+    "  with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
+    "   for top in ('manifest.env','files.txt','payload'):",
+    "    p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
+    "    for x in paths:",
+    "     info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
+    "     with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
+  ].join("\n");
+}

infra/deploy-runner/test_device_control_core_migration_replay_audit_artifact.py

@@ -94,7 +94,16 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
     def test_database_audit_returns_all_mismatches_without_mutation(self):
         database_result = mock.Mock(
             returncode=0,
-            stdout="2\t0\tfalse\tfalse\tfalse\ttrue\n",
+            stdout=(
+                "2\t0\tfalse\tfalse\tfalse\t"
+                + json.dumps(
+                    sorted(
+                        RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS
+                    ),
+                    separators=(",", ":"),
+                )
+                + "\n"
+            ),
             stderr="",
         )
         with mock.patch.object(
@@ -118,7 +127,10 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
         self.assertFalse(evidence["constraintCoversFinalKinds"])
         self.assertFalse(evidence["hostTelemetryTableAbsent"])
         self.assertTrue(evidence["constraintMatchesReplay011Kinds"])
+        self.assertEqual(evidence["constraintPhase"], "replay-011")
+        self.assertTrue(evidence["constraintMatchesKnownReplayCheckpoint"])
         self.assertFalse(evidence["recovery044Ready"])
+        self.assertFalse(evidence["checkpointRecoveryReady"])
         self.assertFalse(evidence["finalStateReady"])
         command = run.call_args.args[0]
         query = command[command.index("-c") + 1]
@@ -134,7 +146,14 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
     def test_recovery_rejection_prints_bounded_database_evidence(self):
         database_result = mock.Mock(
             returncode=0,
-            stdout="0\t0\ttrue\ttrue\ttrue\tfalse\n",
+            stdout=(
+                "0\t0\ttrue\ttrue\ttrue\t"
+                + json.dumps(
+                    sorted(RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS),
+                    separators=(",", ":"),
+                )
+                + "\n"
+            ),
             stderr="",
         )
         output = io.StringIO()
@@ -159,6 +178,8 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
             "constraint_covers_final_kinds=true",
             "host_telemetry_table_absent=true",
             "constraint_matches_replay_011_kinds=false",
+            "constraint_phase=final-016",
+            "constraint_matches_final_016_kinds=true",
             "recovery_final_state_ready=false",
         ):
             self.assertIn(marker, rendered)
@@ -197,7 +218,11 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
                 "constraintCoversFinalKinds": True,
                 "hostTelemetryTableAbsent": True,
                 "constraintMatchesReplay011Kinds": True,
+                "constraintPhase": "replay-011",
+                "constraintMatchesKnownReplayCheckpoint": True,
+                "constraintMatchesFinalKinds": False,
                 "recovery044Ready": False,
+                "checkpointRecoveryReady": False,
                 "finalStateReady": False,
             }
             failure_evidence = {

infra/deploy-runner/test_device_control_core_migration_replay_checkpoint_recovery_artifact.py

@@ -0,0 +1,442 @@
+#!/usr/bin/env python3
+import hashlib
+import importlib.machinery
+import importlib.util
+import inspect
+import json
+import os
+import re
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+RUNNER_PATH = Path(
+    os.environ.get(
+        "NODEDC_DEPLOY_RUNNER",
+        SCRIPT_DIR.parents[2] / "platform/infra/deploy-runner/nodedc-deploy",
+    )
+)
+BUILDER = (
+    SCRIPT_DIR
+    / "build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs"
+)
+LEGACY_BUILDER = (
+    SCRIPT_DIR
+    / "build-device-control-core-migration-replay-recovery-artifact.mjs"
+)
+MIGRATION_ROOT = SCRIPT_DIR.parents[1] / "services/device-control-core/migrations"
+
+
+def load_runner():
+    loader = importlib.machinery.SourceFileLoader(
+        "nodedc_control_core_migration_replay_checkpoint_runner_under_test",
+        str(RUNNER_PATH),
+    )
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    module = importlib.util.module_from_spec(spec)
+    loader.exec_module(module)
+    return module
+
+
+RUNNER = load_runner()
+
+
+def database_stdout(
+    kinds,
+    *,
+    invalid=0,
+    triggering=8,
+    validated="false",
+    covers_final="false",
+    telemetry_absent="true",
+):
+    return "\t".join(
+        (
+            str(invalid),
+            str(triggering),
+            validated,
+            covers_final,
+            telemetry_absent,
+            json.dumps(sorted(kinds), separators=(",", ":")),
+        )
+    ) + "\n"
+
+
+class DeviceControlCoreMigrationReplayCheckpointRecoveryArtifactTest(
+    unittest.TestCase
+):
+    def build(self, artifact_dir):
+        environment = os.environ.copy()
+        environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
+        result = subprocess.run(
+            [
+                "node",
+                str(BUILDER),
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID,
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+            env=environment,
+        )
+        return json.loads(result.stdout)
+
+    def test_artifact_is_deterministic_exact_new_identity(self):
+        with tempfile.TemporaryDirectory(
+            prefix="nodedc-control-core-checkpoint-recovery-",
+        ) as directory:
+            artifact_dir = Path(directory)
+            first = self.build(artifact_dir)
+            first_bytes = Path(first["artifact"]).read_bytes()
+            second = self.build(artifact_dir)
+            second_bytes = Path(second["artifact"]).read_bytes()
+
+            self.assertEqual(first_bytes, second_bytes)
+            self.assertEqual(
+                first["sha256"],
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT_SHA256,
+            )
+            self.assertEqual(first["services"], ["device-control-core"])
+            self.assertEqual(first["databaseRowMutation"], "none")
+            self.assertEqual(
+                first["entries"],
+                list(
+                    RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES
+                ),
+            )
+            with tarfile.open(first["artifact"], "r:gz") as archive:
+                descriptor = json.loads(
+                    archive.extractfile(
+                        "payload/"
+                        + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL
+                    ).read().decode("utf-8")
+                )
+                migration = archive.extractfile(
+                    "payload/" + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
+                ).read()
+            self.assertEqual(
+                descriptor,
+                RUNNER.expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor(),
+            )
+            self.assertEqual(
+                hashlib.sha256(migration).hexdigest(),
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256,
+            )
+            self.assertIn(b")) not valid;", migration.lower())
+            self.assertIsNone(
+                re.search(
+                    rb"(?im)^\s*(?:delete|update|insert|truncate)\b",
+                    migration,
+                )
+            )
+
+    def test_runner_selects_only_core_and_exact_build(self):
+        entries = (
+            RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES
+        )
+        self.assertEqual(
+            RUNNER.component_services("device-plane", entries),
+            ("device-control-core",),
+        )
+        builds = RUNNER.component_builds("device-plane", entries)
+        self.assertEqual(
+            builds,
+            ((
+                RUNNER.DEVICE_PLANE_ROOT,
+                (
+                    "build",
+                    "--no-cache",
+                    "--network=host",
+                    "-f",
+                    "services/device-control-core/Dockerfile",
+                    "-t",
+                    RUNNER.DEVICE_PLANE_CONTROL_CORE_IMAGE,
+                    ".",
+                ),
+            ),),
+        )
+
+    def test_checkpoint_sets_match_every_committed_replay_migration(self):
+        for phase, expected in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS:
+            migration_number = phase.split("-", 1)[1]
+            migration = next(MIGRATION_ROOT.glob(f"{migration_number}_*.sql"))
+            command_kinds = tuple(re.findall(r"'([^']+)'", migration.read_text()))
+            with self.subTest(phase=phase):
+                self.assertEqual(command_kinds, expected)
+                self.assertIn("not valid", migration.read_text().lower())
+
+    def test_every_exact_replay_checkpoint_is_accepted(self):
+        for phase, kinds in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS:
+            database_result = mock.Mock(
+                returncode=0,
+                stdout=database_stdout(kinds),
+                stderr="",
+            )
+            with self.subTest(phase=phase), mock.patch.object(
+                RUNNER,
+                "device_plane_service_container_ids",
+                return_value=("p" * 64,),
+            ), mock.patch.object(
+                RUNNER.subprocess,
+                "run",
+                return_value=database_result,
+            ):
+                evidence = (
+                    RUNNER.collect_device_plane_control_core_migration_replay_database_evidence(
+                        expected_state="replay-checkpoint-predecessor",
+                    )
+                )
+                self.assertEqual(evidence["constraintPhase"], phase)
+                self.assertTrue(
+                    evidence["constraintMatchesKnownReplayCheckpoint"]
+                )
+                self.assertTrue(evidence["checkpointRecoveryReady"])
+                self.assertFalse(evidence["finalStateReady"])
+
+    def test_unknown_missing_and_validated_checkpoint_states_are_rejected(self):
+        cases = (
+            (
+                "unknown",
+                database_stdout(("owner_scope.ensure", "unknown.ensure")),
+            ),
+            (
+                "missing",
+                database_stdout((), validated="missing"),
+            ),
+            (
+                "validated-checkpoint",
+                database_stdout(
+                    RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS,
+                    validated="true",
+                ),
+            ),
+        )
+        for name, stdout in cases:
+            database_result = mock.Mock(
+                returncode=0,
+                stdout=stdout,
+                stderr="",
+            )
+            with self.subTest(name=name), mock.patch.object(
+                RUNNER,
+                "device_plane_service_container_ids",
+                return_value=("p" * 64,),
+            ), mock.patch.object(
+                RUNNER.subprocess,
+                "run",
+                return_value=database_result,
+            ), self.assertRaises(RUNNER.DeployError):
+                RUNNER.collect_device_plane_control_core_migration_replay_database_evidence(
+                    expected_state="replay-checkpoint-predecessor",
+                )
+
+    def test_final_acceptance_requires_exact_migration_016_set(self):
+        database_result = mock.Mock(
+            returncode=0,
+            stdout=database_stdout(
+                RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS,
+                validated="true",
+                covers_final="true",
+            ),
+            stderr="",
+        )
+        with mock.patch.object(
+            RUNNER,
+            "device_plane_service_container_ids",
+            return_value=("p" * 64,),
+        ), mock.patch.object(
+            RUNNER.subprocess,
+            "run",
+            return_value=database_result,
+        ):
+            evidence = (
+                RUNNER.collect_device_plane_control_core_migration_replay_database_evidence(
+                    expected_state="final",
+                )
+            )
+        self.assertEqual(evidence["constraintPhase"], "final-016")
+        self.assertTrue(evidence["constraintMatchesFinalKinds"])
+        self.assertTrue(evidence["finalStateReady"])
+        self.assertFalse(evidence["checkpointRecoveryReady"])
+
+    def test_terminal_044_failure_evidence_is_exact_and_preapply(self):
+        with tempfile.TemporaryDirectory(
+            prefix="nodedc-control-core-recovery-044-failure-",
+        ) as directory:
+            root = Path(directory)
+            artifact_dir = root / "artifact"
+            failed_dir = root / "failed"
+            runner_tmp = root / "tmp"
+            state_file = root / "applied.jsonl"
+            failed_state_file = root / "failed.jsonl"
+            artifact_dir.mkdir()
+            failed_dir.mkdir()
+            runner_tmp.mkdir()
+            environment = os.environ.copy()
+            environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
+            subprocess.run(
+                [
+                    "node",
+                    str(LEGACY_BUILDER),
+                    RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID,
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+                env=environment,
+            )
+            source = (
+                artifact_dir
+                / RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT
+            )
+            failed_artifact = (
+                failed_dir
+                / RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT
+            )
+            source.replace(failed_artifact)
+            state_file.write_text("", encoding="utf-8")
+            record = {
+                "artifact": failed_artifact.name,
+                "backup_id": None,
+                "component": "device-plane",
+                "failed_at": (
+                    RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT
+                ),
+                "id": RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID,
+                "message": (
+                    "Device Control Core migration recovery database invariant mismatch"
+                ),
+                "rollback_status": "not-required",
+                "sha256": (
+                    RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
+                ),
+                "started_apply": False,
+                "status": "failed",
+            }
+            failed_state_file.write_text(
+                json.dumps(record) + "\n",
+                encoding="utf-8",
+            )
+
+            with mock.patch.object(RUNNER, "FAILED_DIR", failed_dir), mock.patch.object(
+                RUNNER,
+                "STATE_FILE",
+                state_file,
+            ), mock.patch.object(
+                RUNNER,
+                "FAILED_STATE_FILE",
+                failed_state_file,
+            ), mock.patch.object(RUNNER, "TMP_DIR", runner_tmp):
+                evidence = (
+                    RUNNER.validate_device_plane_control_core_migration_replay_recovery_failure()
+                )
+            self.assertEqual(evidence["record"], record)
+
+    def test_checkpoint_recovery_preflight_reaches_database_collector(self):
+        with tempfile.TemporaryDirectory(
+            prefix="nodedc-control-core-checkpoint-preflight-",
+        ) as directory:
+            root = Path(directory)
+            migration = (
+                root / RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
+            )
+            migration.parent.mkdir(parents=True)
+            migration.write_bytes(b"migration-014-predecessor")
+            core = {
+                "containerId": "c" * 64,
+                "imageId": RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
+                "status": "restarting",
+                "health": "unhealthy",
+                "restartCount": 280,
+            }
+            runtime = {"core": core, "current": {"services": [core]}}
+            database = {
+                "constraintPhase": "replay-007",
+                "checkpointRecoveryReady": True,
+            }
+            descriptor = (
+                RUNNER.expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor()
+            )
+
+            with mock.patch.object(
+                RUNNER,
+                "validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload",
+                return_value=descriptor,
+            ), mock.patch.object(
+                RUNNER,
+                "validate_device_plane_control_core_double_failure_evidence",
+                return_value={
+                    "firstBackup": root / "first",
+                    "secondBackup": root / "second",
+                },
+            ), mock.patch.object(
+                RUNNER,
+                "validate_device_plane_control_core_migration_replay_recovery_failure",
+                return_value={"record": {"started_apply": False}},
+            ), mock.patch.object(
+                RUNNER,
+                "component_root",
+                return_value=root,
+            ), mock.patch.object(
+                RUNNER,
+                "DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256",
+                hashlib.sha256(migration.read_bytes()).hexdigest(),
+            ), mock.patch.object(
+                RUNNER,
+                "validate_device_plane_control_core_migration_replay_preserved_runtime",
+                return_value=runtime,
+            ), mock.patch.object(
+                RUNNER,
+                "inspect_optional_local_image",
+                return_value=RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
+            ), mock.patch.object(
+                RUNNER,
+                "collect_device_plane_control_core_migration_replay_database_evidence",
+                return_value=database,
+            ) as collector:
+                evidence = (
+                    RUNNER.validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
+                        root
+                    )
+                )
+
+            self.assertEqual(evidence["database"], database)
+            collector.assert_called_once_with(
+                expected_state="replay-checkpoint-predecessor",
+            )
+
+    def test_v2_rollback_and_acceptance_are_registered(self):
+        rollback_source = inspect.getsource(RUNNER.rollback_device_plane_apply)
+        health_source = inspect.getsource(RUNNER.run_healthchecks)
+        preflight_source = inspect.getsource(
+            RUNNER.validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence
+        )
+        acceptance_source = inspect.getsource(
+            RUNNER.accept_device_plane_control_core_migration_replay_checkpoint_recovery
+        )
+        self.assertIn(
+            "is_device_plane_control_core_migration_replay_checkpoint_recovery_slice",
+            rollback_source,
+        )
+        self.assertIn(
+            "accept_device_plane_control_core_migration_replay_checkpoint_recovery",
+            health_source,
+        )
+        self.assertIn(
+            'expected_state="replay-checkpoint-predecessor"',
+            preflight_source,
+        )
+        self.assertIn(
+            "validate_device_plane_control_core_migration_replay_recovery_failure",
+            preflight_source,
+        )
+        self.assertIn('expected_state="final"', acceptance_source)
+
+
+if __name__ == "__main__":
+    unittest.main()

infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py

@@ -139,7 +139,16 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
     def test_database_preflight_is_read_only_and_covers_live_receipts(self):
         database_result = mock.Mock(
             returncode=0,
-            stdout="0\t8\tfalse\tfalse\ttrue\ttrue\n",
+            stdout=(
+                "0\t8\tfalse\tfalse\ttrue\t"
+                + json.dumps(
+                    sorted(
+                        RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS
+                    ),
+                    separators=(",", ":"),
+                )
+                + "\n"
+            ),
             stderr="",
         )
         with mock.patch.object(
@@ -162,7 +171,9 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
         self.assertFalse(evidence["constraintValidated"])
         self.assertFalse(evidence["constraintCoversFinalKinds"])
         self.assertTrue(evidence["constraintMatchesReplay011Kinds"])
+        self.assertEqual(evidence["constraintPhase"], "replay-011")
         self.assertTrue(evidence["recovery044Ready"])
+        self.assertTrue(evidence["checkpointRecoveryReady"])
         self.assertFalse(evidence["finalStateReady"])
         command = run.call_args.args[0]
         query = command[command.index("-c") + 1]
@@ -175,8 +186,7 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
             )
         )
         self.assertIn("regexp_matches", query)
-        for command_kind in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS:
+        self.assertIn("array_to_json", query)
-            self.assertIn(f"'{command_kind}'", query)
 
     def test_runner_predecessor_kind_set_matches_migration_011(self):
         migration = MIGRATION_011.read_text(encoding="utf-8")
@@ -190,7 +200,14 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
     def test_database_acceptance_requires_final_validated_constraint(self):
         database_result = mock.Mock(
             returncode=0,
-            stdout="0\t8\ttrue\ttrue\ttrue\tfalse\n",
+            stdout=(
+                "0\t8\ttrue\ttrue\ttrue\t"
+                + json.dumps(
+                    sorted(RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS),
+                    separators=(",", ":"),
+                )
+                + "\n"
+            ),
             stderr="",
         )
         with mock.patch.object(
@@ -211,13 +228,20 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
         self.assertTrue(evidence["constraintValidated"])
         self.assertTrue(evidence["constraintCoversFinalKinds"])
         self.assertFalse(evidence["constraintMatchesReplay011Kinds"])
+        self.assertEqual(evidence["constraintPhase"], "final-016")
+        self.assertTrue(evidence["constraintMatchesFinalKinds"])
         self.assertFalse(evidence["recovery044Ready"])
+        self.assertFalse(evidence["checkpointRecoveryReady"])
         self.assertTrue(evidence["finalStateReady"])
 
     def test_database_preflight_rejects_ambiguous_constraint_shape(self):
         database_result = mock.Mock(
             returncode=0,
-            stdout="0\t8\tfalse\tfalse\ttrue\tfalse\n",
+            stdout=(
+                "0\t8\tfalse\tfalse\ttrue\t"
+                + json.dumps(["owner_scope.ensure", "unknown.ensure"])
+                + "\n"
+            ),
             stderr="",
         )
         with mock.patch.object(