From 5d58dd6c37b3d54a3efd0d6fed3e0ce83bab9eb7 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Sat, 22 Aug 2026 23:58:17 +0300 Subject: [PATCH] fix(deploy): add replay checkpoint recovery --- ...gration-replay-checkpoint-recovery-v2.json | 36 ++ ...on-replay-checkpoint-recovery-artifact.mjs | 106 +++++ ...ol_core_migration_replay_audit_artifact.py | 29 +- ...ion_replay_checkpoint_recovery_artifact.py | 442 ++++++++++++++++++ ...core_migration_replay_recovery_artifact.py | 34 +- 5 files changed, 640 insertions(+), 7 deletions(-) create mode 100644 deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json create mode 100644 infra/deploy-runner/build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs create mode 100644 infra/deploy-runner/test_device_control_core_migration_replay_checkpoint_recovery_artifact.py diff --git a/deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json b/deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json new file mode 100644 index 0000000..4e19e7d --- /dev/null +++ b/deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": "nodedc.device-plane.device-control-core-migration-replay-checkpoint-recovery.v2", + "mode": "terminal-044-replay-checkpoint-forward-repair", + "failedIncidentAudit": "device-control-core-incident-audit-20260822-043", + "failedRecovery": { + "patchId": "device-control-core-migration-replay-recovery-20260822-044", + "artifactSha256": "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7", + "failure": "preflight-replay-checkpoint-race", + "startedApply": false + }, + "sourcePredecessor": { + "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql", + "sha256": "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508" + }, + "sourceTarget": { + "path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql", + "sha256": "38bd86b42828d44c7101d5433ddc36018e92eedeee37b9de296432ad676edd46" + }, + "rootCause": "restarting-core-cycles-exact-committed-migration-checkpoints", + "repair": "migration-014-add-constraint-not-valid", + "databasePreflight": "exact-replay-checkpoint-005-007-009-011-and-final-compatible-rows", + "databaseRowMutation": "none", + "databaseSchemaOutcome": "exact-final-migration-016-validated-command-kind-check", + "runtimeAction": "build+recreate-device-control-core-only", + "runtimePredecessor": "proven-degraded-restarting-exact-preapply-image", + "preservedServices": [ + "device-manager", + "device-gateway", + "device-postgres", + "device-backhaul-target" + ], + "databaseVolume": "nodedc-device-plane-postgres-data", + "publicIngress": "disabled", + "edgeChannel": "core-initiated-pinned-mtls-registered-edges-only", + "rollback": "source+exact-degraded-predecessor-image-runtime" +} diff --git a/infra/deploy-runner/build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs b/infra/deploy-runner/build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs new file mode 100644 index 0000000..d5c3134 --- /dev/null +++ b/infra/deploy-runner/build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs @@ -0,0 +1,106 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const sourceRoot = resolve(scriptDir, "../.."); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-control-core-migration-replay-checkpoint-recovery-20260822-046", + ...extra +] = process.argv.slice(2); +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs [patch-id]", + ); +} + +const entries = [ + "services/device-control-core/migrations/014_device_registry_profile_commands.sql", + "deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json", +]; +const stage = await mkdtemp(join(tmpdir(), "nodedc-control-core-replay-checkpoint-")); +const payload = join(stage, "payload"); +const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`); + +try { + const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entries[1]), "utf8")); + const targetBytes = await readFile(resolve(sourceRoot, entries[0])); + const targetSha256 = createHash("sha256").update(targetBytes).digest("hex"); + if ( + descriptor.schemaVersion + !== "nodedc.device-plane.device-control-core-migration-replay-checkpoint-recovery.v2" + || descriptor.mode !== "terminal-044-replay-checkpoint-forward-repair" + || descriptor.failedRecovery?.patchId + !== "device-control-core-migration-replay-recovery-20260822-044" + || descriptor.failedRecovery?.artifactSha256 + !== "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7" + || descriptor.failedRecovery?.startedApply !== false + || descriptor.sourceTarget.path !== entries[0] + || descriptor.sourceTarget.sha256 !== targetSha256 + || descriptor.databaseRowMutation !== "none" + || descriptor.runtimeAction !== "build+recreate-device-control-core-only" + ) { + throw new Error( + "device_control_core_migration_replay_checkpoint_recovery_descriptor_mismatch", + ); + } + + for (const entry of entries) { + await mkdir(dirname(join(payload, entry)), { recursive: true }); + await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true }); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, + ); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + const sha256 = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + component: "device-plane", + artifact: target, + sha256, + entries, + build: ["nodedc/device-control-core:local"], + services: ["device-control-core"], + transition: descriptor.mode, + databaseRowMutation: descriptor.databaseRowMutation, + runtimeAction: descriptor.runtimeAction, + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} diff --git a/infra/deploy-runner/test_device_control_core_migration_replay_audit_artifact.py b/infra/deploy-runner/test_device_control_core_migration_replay_audit_artifact.py index efdc370..c86e353 100644 --- a/infra/deploy-runner/test_device_control_core_migration_replay_audit_artifact.py +++ b/infra/deploy-runner/test_device_control_core_migration_replay_audit_artifact.py @@ -94,7 +94,16 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase): def test_database_audit_returns_all_mismatches_without_mutation(self): database_result = mock.Mock( returncode=0, - stdout="2\t0\tfalse\tfalse\tfalse\ttrue\n", + stdout=( + "2\t0\tfalse\tfalse\tfalse\t" + + json.dumps( + sorted( + RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS + ), + separators=(",", ":"), + ) + + "\n" + ), stderr="", ) with mock.patch.object( @@ -118,7 +127,10 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase): self.assertFalse(evidence["constraintCoversFinalKinds"]) self.assertFalse(evidence["hostTelemetryTableAbsent"]) self.assertTrue(evidence["constraintMatchesReplay011Kinds"]) + self.assertEqual(evidence["constraintPhase"], "replay-011") + self.assertTrue(evidence["constraintMatchesKnownReplayCheckpoint"]) self.assertFalse(evidence["recovery044Ready"]) + self.assertFalse(evidence["checkpointRecoveryReady"]) self.assertFalse(evidence["finalStateReady"]) command = run.call_args.args[0] query = command[command.index("-c") + 1] @@ -134,7 +146,14 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase): def test_recovery_rejection_prints_bounded_database_evidence(self): database_result = mock.Mock( returncode=0, - stdout="0\t0\ttrue\ttrue\ttrue\tfalse\n", + stdout=( + "0\t0\ttrue\ttrue\ttrue\t" + + json.dumps( + sorted(RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS), + separators=(",", ":"), + ) + + "\n" + ), stderr="", ) output = io.StringIO() @@ -159,6 +178,8 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase): "constraint_covers_final_kinds=true", "host_telemetry_table_absent=true", "constraint_matches_replay_011_kinds=false", + "constraint_phase=final-016", + "constraint_matches_final_016_kinds=true", "recovery_final_state_ready=false", ): self.assertIn(marker, rendered) @@ -197,7 +218,11 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase): "constraintCoversFinalKinds": True, "hostTelemetryTableAbsent": True, "constraintMatchesReplay011Kinds": True, + "constraintPhase": "replay-011", + "constraintMatchesKnownReplayCheckpoint": True, + "constraintMatchesFinalKinds": False, "recovery044Ready": False, + "checkpointRecoveryReady": False, "finalStateReady": False, } failure_evidence = { diff --git a/infra/deploy-runner/test_device_control_core_migration_replay_checkpoint_recovery_artifact.py b/infra/deploy-runner/test_device_control_core_migration_replay_checkpoint_recovery_artifact.py new file mode 100644 index 0000000..9276a0a --- /dev/null +++ b/infra/deploy-runner/test_device_control_core_migration_replay_checkpoint_recovery_artifact.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import inspect +import json +import os +import re +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +RUNNER_PATH = Path( + os.environ.get( + "NODEDC_DEPLOY_RUNNER", + SCRIPT_DIR.parents[2] / "platform/infra/deploy-runner/nodedc-deploy", + ) +) +BUILDER = ( + SCRIPT_DIR + / "build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs" +) +LEGACY_BUILDER = ( + SCRIPT_DIR + / "build-device-control-core-migration-replay-recovery-artifact.mjs" +) +MIGRATION_ROOT = SCRIPT_DIR.parents[1] / "services/device-control-core/migrations" + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_control_core_migration_replay_checkpoint_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +def database_stdout( + kinds, + *, + invalid=0, + triggering=8, + validated="false", + covers_final="false", + telemetry_absent="true", +): + return "\t".join( + ( + str(invalid), + str(triggering), + validated, + covers_final, + telemetry_absent, + json.dumps(sorted(kinds), separators=(",", ":")), + ) + ) + "\n" + + +class DeviceControlCoreMigrationReplayCheckpointRecoveryArtifactTest( + unittest.TestCase +): + def build(self, artifact_dir): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + [ + "node", + str(BUILDER), + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID, + ], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_artifact_is_deterministic_exact_new_identity(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-control-core-checkpoint-recovery-", + ) as directory: + artifact_dir = Path(directory) + first = self.build(artifact_dir) + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build(artifact_dir) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT_SHA256, + ) + self.assertEqual(first["services"], ["device-control-core"]) + self.assertEqual(first["databaseRowMutation"], "none") + self.assertEqual( + first["entries"], + list( + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES + ), + ) + with tarfile.open(first["artifact"], "r:gz") as archive: + descriptor = json.loads( + archive.extractfile( + "payload/" + + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL + ).read().decode("utf-8") + ) + migration = archive.extractfile( + "payload/" + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + ).read() + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor(), + ) + self.assertEqual( + hashlib.sha256(migration).hexdigest(), + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256, + ) + self.assertIn(b")) not valid;", migration.lower()) + self.assertIsNone( + re.search( + rb"(?im)^\s*(?:delete|update|insert|truncate)\b", + migration, + ) + ) + + def test_runner_selects_only_core_and_exact_build(self): + entries = ( + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES + ) + self.assertEqual( + RUNNER.component_services("device-plane", entries), + ("device-control-core",), + ) + builds = RUNNER.component_builds("device-plane", entries) + self.assertEqual( + builds, + (( + RUNNER.DEVICE_PLANE_ROOT, + ( + "build", + "--no-cache", + "--network=host", + "-f", + "services/device-control-core/Dockerfile", + "-t", + RUNNER.DEVICE_PLANE_CONTROL_CORE_IMAGE, + ".", + ), + ),), + ) + + def test_checkpoint_sets_match_every_committed_replay_migration(self): + for phase, expected in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS: + migration_number = phase.split("-", 1)[1] + migration = next(MIGRATION_ROOT.glob(f"{migration_number}_*.sql")) + command_kinds = tuple(re.findall(r"'([^']+)'", migration.read_text())) + with self.subTest(phase=phase): + self.assertEqual(command_kinds, expected) + self.assertIn("not valid", migration.read_text().lower()) + + def test_every_exact_replay_checkpoint_is_accepted(self): + for phase, kinds in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS: + database_result = mock.Mock( + returncode=0, + stdout=database_stdout(kinds), + stderr="", + ) + with self.subTest(phase=phase), mock.patch.object( + RUNNER, + "device_plane_service_container_ids", + return_value=("p" * 64,), + ), mock.patch.object( + RUNNER.subprocess, + "run", + return_value=database_result, + ): + evidence = ( + RUNNER.collect_device_plane_control_core_migration_replay_database_evidence( + expected_state="replay-checkpoint-predecessor", + ) + ) + self.assertEqual(evidence["constraintPhase"], phase) + self.assertTrue( + evidence["constraintMatchesKnownReplayCheckpoint"] + ) + self.assertTrue(evidence["checkpointRecoveryReady"]) + self.assertFalse(evidence["finalStateReady"]) + + def test_unknown_missing_and_validated_checkpoint_states_are_rejected(self): + cases = ( + ( + "unknown", + database_stdout(("owner_scope.ensure", "unknown.ensure")), + ), + ( + "missing", + database_stdout((), validated="missing"), + ), + ( + "validated-checkpoint", + database_stdout( + RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS, + validated="true", + ), + ), + ) + for name, stdout in cases: + database_result = mock.Mock( + returncode=0, + stdout=stdout, + stderr="", + ) + with self.subTest(name=name), mock.patch.object( + RUNNER, + "device_plane_service_container_ids", + return_value=("p" * 64,), + ), mock.patch.object( + RUNNER.subprocess, + "run", + return_value=database_result, + ), self.assertRaises(RUNNER.DeployError): + RUNNER.collect_device_plane_control_core_migration_replay_database_evidence( + expected_state="replay-checkpoint-predecessor", + ) + + def test_final_acceptance_requires_exact_migration_016_set(self): + database_result = mock.Mock( + returncode=0, + stdout=database_stdout( + RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS, + validated="true", + covers_final="true", + ), + stderr="", + ) + with mock.patch.object( + RUNNER, + "device_plane_service_container_ids", + return_value=("p" * 64,), + ), mock.patch.object( + RUNNER.subprocess, + "run", + return_value=database_result, + ): + evidence = ( + RUNNER.collect_device_plane_control_core_migration_replay_database_evidence( + expected_state="final", + ) + ) + self.assertEqual(evidence["constraintPhase"], "final-016") + self.assertTrue(evidence["constraintMatchesFinalKinds"]) + self.assertTrue(evidence["finalStateReady"]) + self.assertFalse(evidence["checkpointRecoveryReady"]) + + def test_terminal_044_failure_evidence_is_exact_and_preapply(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-control-core-recovery-044-failure-", + ) as directory: + root = Path(directory) + artifact_dir = root / "artifact" + failed_dir = root / "failed" + runner_tmp = root / "tmp" + state_file = root / "applied.jsonl" + failed_state_file = root / "failed.jsonl" + artifact_dir.mkdir() + failed_dir.mkdir() + runner_tmp.mkdir() + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + subprocess.run( + [ + "node", + str(LEGACY_BUILDER), + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID, + ], + check=True, + capture_output=True, + text=True, + env=environment, + ) + source = ( + artifact_dir + / RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT + ) + failed_artifact = ( + failed_dir + / RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT + ) + source.replace(failed_artifact) + state_file.write_text("", encoding="utf-8") + record = { + "artifact": failed_artifact.name, + "backup_id": None, + "component": "device-plane", + "failed_at": ( + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT + ), + "id": RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID, + "message": ( + "Device Control Core migration recovery database invariant mismatch" + ), + "rollback_status": "not-required", + "sha256": ( + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256 + ), + "started_apply": False, + "status": "failed", + } + failed_state_file.write_text( + json.dumps(record) + "\n", + encoding="utf-8", + ) + + with mock.patch.object(RUNNER, "FAILED_DIR", failed_dir), mock.patch.object( + RUNNER, + "STATE_FILE", + state_file, + ), mock.patch.object( + RUNNER, + "FAILED_STATE_FILE", + failed_state_file, + ), mock.patch.object(RUNNER, "TMP_DIR", runner_tmp): + evidence = ( + RUNNER.validate_device_plane_control_core_migration_replay_recovery_failure() + ) + self.assertEqual(evidence["record"], record) + + def test_checkpoint_recovery_preflight_reaches_database_collector(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-control-core-checkpoint-preflight-", + ) as directory: + root = Path(directory) + migration = ( + root / RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL + ) + migration.parent.mkdir(parents=True) + migration.write_bytes(b"migration-014-predecessor") + core = { + "containerId": "c" * 64, + "imageId": RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + "status": "restarting", + "health": "unhealthy", + "restartCount": 280, + } + runtime = {"core": core, "current": {"services": [core]}} + database = { + "constraintPhase": "replay-007", + "checkpointRecoveryReady": True, + } + descriptor = ( + RUNNER.expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor() + ) + + with mock.patch.object( + RUNNER, + "validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload", + return_value=descriptor, + ), mock.patch.object( + RUNNER, + "validate_device_plane_control_core_double_failure_evidence", + return_value={ + "firstBackup": root / "first", + "secondBackup": root / "second", + }, + ), mock.patch.object( + RUNNER, + "validate_device_plane_control_core_migration_replay_recovery_failure", + return_value={"record": {"started_apply": False}}, + ), mock.patch.object( + RUNNER, + "component_root", + return_value=root, + ), mock.patch.object( + RUNNER, + "DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256", + hashlib.sha256(migration.read_bytes()).hexdigest(), + ), mock.patch.object( + RUNNER, + "validate_device_plane_control_core_migration_replay_preserved_runtime", + return_value=runtime, + ), mock.patch.object( + RUNNER, + "inspect_optional_local_image", + return_value=RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID, + ), mock.patch.object( + RUNNER, + "collect_device_plane_control_core_migration_replay_database_evidence", + return_value=database, + ) as collector: + evidence = ( + RUNNER.validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence( + root + ) + ) + + self.assertEqual(evidence["database"], database) + collector.assert_called_once_with( + expected_state="replay-checkpoint-predecessor", + ) + + def test_v2_rollback_and_acceptance_are_registered(self): + rollback_source = inspect.getsource(RUNNER.rollback_device_plane_apply) + health_source = inspect.getsource(RUNNER.run_healthchecks) + preflight_source = inspect.getsource( + RUNNER.validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence + ) + acceptance_source = inspect.getsource( + RUNNER.accept_device_plane_control_core_migration_replay_checkpoint_recovery + ) + self.assertIn( + "is_device_plane_control_core_migration_replay_checkpoint_recovery_slice", + rollback_source, + ) + self.assertIn( + "accept_device_plane_control_core_migration_replay_checkpoint_recovery", + health_source, + ) + self.assertIn( + 'expected_state="replay-checkpoint-predecessor"', + preflight_source, + ) + self.assertIn( + "validate_device_plane_control_core_migration_replay_recovery_failure", + preflight_source, + ) + self.assertIn('expected_state="final"', acceptance_source) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py b/infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py index 8514fb9..98e0281 100644 --- a/infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py +++ b/infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py @@ -139,7 +139,16 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase): def test_database_preflight_is_read_only_and_covers_live_receipts(self): database_result = mock.Mock( returncode=0, - stdout="0\t8\tfalse\tfalse\ttrue\ttrue\n", + stdout=( + "0\t8\tfalse\tfalse\ttrue\t" + + json.dumps( + sorted( + RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS + ), + separators=(",", ":"), + ) + + "\n" + ), stderr="", ) with mock.patch.object( @@ -162,7 +171,9 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase): self.assertFalse(evidence["constraintValidated"]) self.assertFalse(evidence["constraintCoversFinalKinds"]) self.assertTrue(evidence["constraintMatchesReplay011Kinds"]) + self.assertEqual(evidence["constraintPhase"], "replay-011") self.assertTrue(evidence["recovery044Ready"]) + self.assertTrue(evidence["checkpointRecoveryReady"]) self.assertFalse(evidence["finalStateReady"]) command = run.call_args.args[0] query = command[command.index("-c") + 1] @@ -175,8 +186,7 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase): ) ) self.assertIn("regexp_matches", query) - for command_kind in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS: - self.assertIn(f"'{command_kind}'", query) + self.assertIn("array_to_json", query) def test_runner_predecessor_kind_set_matches_migration_011(self): migration = MIGRATION_011.read_text(encoding="utf-8") @@ -190,7 +200,14 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase): def test_database_acceptance_requires_final_validated_constraint(self): database_result = mock.Mock( returncode=0, - stdout="0\t8\ttrue\ttrue\ttrue\tfalse\n", + stdout=( + "0\t8\ttrue\ttrue\ttrue\t" + + json.dumps( + sorted(RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS), + separators=(",", ":"), + ) + + "\n" + ), stderr="", ) with mock.patch.object( @@ -211,13 +228,20 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase): self.assertTrue(evidence["constraintValidated"]) self.assertTrue(evidence["constraintCoversFinalKinds"]) self.assertFalse(evidence["constraintMatchesReplay011Kinds"]) + self.assertEqual(evidence["constraintPhase"], "final-016") + self.assertTrue(evidence["constraintMatchesFinalKinds"]) self.assertFalse(evidence["recovery044Ready"]) + self.assertFalse(evidence["checkpointRecoveryReady"]) self.assertTrue(evidence["finalStateReady"]) def test_database_preflight_rejects_ambiguous_constraint_shape(self): database_result = mock.Mock( returncode=0, - stdout="0\t8\tfalse\tfalse\ttrue\tfalse\n", + stdout=( + "0\t8\tfalse\tfalse\ttrue\t" + + json.dumps(["owner_scope.ensure", "unknown.ensure"]) + + "\n" + ), stderr="", ) with mock.patch.object(