SILVER/NODEDC_DEVICE_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(deploy): add replay checkpoint recovery

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 23:58:17 +03:00
parent 11ddacd9cd
commit 5d58dd6c37
5 changed files with 640 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json
+36
View File
@@ -0,0 +1,36 @@
{
"schemaVersion": "nodedc.device-plane.device-control-core-migration-replay-checkpoint-recovery.v2",
"mode": "terminal-044-replay-checkpoint-forward-repair",
"failedIncidentAudit": "device-control-core-incident-audit-20260822-043",
"failedRecovery": {
"patchId": "device-control-core-migration-replay-recovery-20260822-044",
"artifactSha256": "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7",
"failure": "preflight-replay-checkpoint-race",
"startedApply": false
},
"sourcePredecessor": {
"path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
"sha256": "751accf346b34d2774cc7b9572640d2c25fdb0b1db793ac32183b56f48e26508"
},
"sourceTarget": {
"path": "services/device-control-core/migrations/014_device_registry_profile_commands.sql",
"sha256": "38bd86b42828d44c7101d5433ddc36018e92eedeee37b9de296432ad676edd46"
},
"rootCause": "restarting-core-cycles-exact-committed-migration-checkpoints",
"repair": "migration-014-add-constraint-not-valid",
"databasePreflight": "exact-replay-checkpoint-005-007-009-011-and-final-compatible-rows",
"databaseRowMutation": "none",
"databaseSchemaOutcome": "exact-final-migration-016-validated-command-kind-check",
"runtimeAction": "build+recreate-device-control-core-only",
"runtimePredecessor": "proven-degraded-restarting-exact-preapply-image",
"preservedServices": [
"device-manager",
"device-gateway",
"device-postgres",
"device-backhaul-target"
],
"databaseVolume": "nodedc-device-plane-postgres-data",
"publicIngress": "disabled",
"edgeChannel": "core-initiated-pinned-mtls-registered-edges-only",
"rollback": "source+exact-degraded-predecessor-image-runtime"
}
infra/deploy-runner/build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const sourceRoot = resolve(scriptDir, "../..");
const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"),
);
const [
patchId = "device-control-core-migration-replay-checkpoint-recovery-20260822-046",
...extra
] = process.argv.slice(2);
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
throw new Error(
"usage: build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs [patch-id]",
);
}
const entries = [
"services/device-control-core/migrations/014_device_registry_profile_commands.sql",
"deployment/device-control-core-migration-replay-checkpoint-recovery-v2.json",
];
const stage = await mkdtemp(join(tmpdir(), "nodedc-control-core-replay-checkpoint-"));
const payload = join(stage, "payload");
const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`);
try {
const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entries[1]), "utf8"));
const targetBytes = await readFile(resolve(sourceRoot, entries[0]));
const targetSha256 = createHash("sha256").update(targetBytes).digest("hex");
if (
descriptor.schemaVersion
!== "nodedc.device-plane.device-control-core-migration-replay-checkpoint-recovery.v2"
|| descriptor.mode !== "terminal-044-replay-checkpoint-forward-repair"
|| descriptor.failedRecovery?.patchId
!== "device-control-core-migration-replay-recovery-20260822-044"
|| descriptor.failedRecovery?.artifactSha256
!== "b893d8c90f98943797d32f486d6477d58a3be69eb1291e28c4a4bbd2e96774b7"
|| descriptor.failedRecovery?.startedApply !== false
|| descriptor.sourceTarget.path !== entries[0]
|| descriptor.sourceTarget.sha256 !== targetSha256
|| descriptor.databaseRowMutation !== "none"
|| descriptor.runtimeAction !== "build+recreate-device-control-core-only"
) {
throw new Error(
"device_control_core_migration_replay_checkpoint_recovery_descriptor_mismatch",
);
}
for (const entry of entries) {
await mkdir(dirname(join(payload, entry)), { recursive: true });
await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true });
}
await writeFile(
join(stage, "manifest.env"),
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
"utf8",
);
await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8");
await mkdir(artifactDir, { recursive: true });
const tar = spawnSync(
"python3",
["-c", canonicalTarScript(), target, stage],
{ encoding: "utf8", maxBuffer: 16 * 1024 * 1024 },
);
if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
const sha256 = createHash("sha256")
.update(await readFile(target))
.digest("hex");
console.log(JSON.stringify({
ok: true,
patchId,
component: "device-plane",
artifact: target,
sha256,
entries,
build: ["nodedc/device-control-core:local"],
services: ["device-control-core"],
transition: descriptor.mode,
databaseRowMutation: descriptor.databaseRowMutation,
runtimeAction: descriptor.runtimeAction,
}, null, 2));
} finally {
await rm(stage, { recursive: true, force: true });
}
function canonicalTarScript() {
return [
"import gzip,io,pathlib,sys,tarfile",
"root=pathlib.Path(sys.argv[2])",
"with open(sys.argv[1],'wb') as out:",
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
" for top in ('manifest.env','files.txt','payload'):",
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
" for x in paths:",
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
].join("\n");
}
infra/deploy-runner/test_device_control_core_migration_replay_audit_artifact.py
+27 -2
View File
@@ -94,7 +94,16 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
def test_database_audit_returns_all_mismatches_without_mutation(self):
database_result = mock.Mock(
returncode=0,
stdout="2\t0\tfalse\tfalse\tfalse\ttrue\n",
stdout=(
"2\t0\tfalse\tfalse\tfalse\t"
+ json.dumps(
sorted(
RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS
),
separators=(",", ":"),
)
+ "\n"
),
stderr="",
)
with mock.patch.object(
@@ -118,7 +127,10 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
self.assertFalse(evidence["constraintCoversFinalKinds"])
self.assertFalse(evidence["hostTelemetryTableAbsent"])
self.assertTrue(evidence["constraintMatchesReplay011Kinds"])
self.assertEqual(evidence["constraintPhase"], "replay-011")
self.assertTrue(evidence["constraintMatchesKnownReplayCheckpoint"])
self.assertFalse(evidence["recovery044Ready"])
self.assertFalse(evidence["checkpointRecoveryReady"])
self.assertFalse(evidence["finalStateReady"])
command = run.call_args.args[0]
query = command[command.index("-c") + 1]
@@ -134,7 +146,14 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
def test_recovery_rejection_prints_bounded_database_evidence(self):
database_result = mock.Mock(
returncode=0,
stdout="0\t0\ttrue\ttrue\ttrue\tfalse\n",
stdout=(
"0\t0\ttrue\ttrue\ttrue\t"
+ json.dumps(
sorted(RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS),
separators=(",", ":"),
)
+ "\n"
),
stderr="",
)
output = io.StringIO()
@@ -159,6 +178,8 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
"constraint_covers_final_kinds=true",
"host_telemetry_table_absent=true",
"constraint_matches_replay_011_kinds=false",
"constraint_phase=final-016",
"constraint_matches_final_016_kinds=true",
"recovery_final_state_ready=false",
):
self.assertIn(marker, rendered)
@@ -197,7 +218,11 @@ class DeviceControlCoreMigrationReplayAuditArtifactTest(unittest.TestCase):
"constraintCoversFinalKinds": True,
"hostTelemetryTableAbsent": True,
"constraintMatchesReplay011Kinds": True,
"constraintPhase": "replay-011",
"constraintMatchesKnownReplayCheckpoint": True,
"constraintMatchesFinalKinds": False,
"recovery044Ready": False,
"checkpointRecoveryReady": False,
"finalStateReady": False,
}
failure_evidence = {
infra/deploy-runner/test_device_control_core_migration_replay_checkpoint_recovery_artifact.py
+442
View File
@@ -0,0 +1,442 @@
#!/usr/bin/env python3
import hashlib
import importlib.machinery
import importlib.util
import inspect
import json
import os
import re
import subprocess
import tarfile
import tempfile
import unittest
from pathlib import Path
from unittest import mock
SCRIPT_DIR = Path(__file__).resolve().parent
RUNNER_PATH = Path(
os.environ.get(
"NODEDC_DEPLOY_RUNNER",
SCRIPT_DIR.parents[2] / "platform/infra/deploy-runner/nodedc-deploy",
)
)
BUILDER = (
SCRIPT_DIR
/ "build-device-control-core-migration-replay-checkpoint-recovery-artifact.mjs"
)
LEGACY_BUILDER = (
SCRIPT_DIR
/ "build-device-control-core-migration-replay-recovery-artifact.mjs"
)
MIGRATION_ROOT = SCRIPT_DIR.parents[1] / "services/device-control-core/migrations"
def load_runner():
loader = importlib.machinery.SourceFileLoader(
"nodedc_control_core_migration_replay_checkpoint_runner_under_test",
str(RUNNER_PATH),
)
spec = importlib.util.spec_from_loader(loader.name, loader)
module = importlib.util.module_from_spec(spec)
loader.exec_module(module)
return module
RUNNER = load_runner()
def database_stdout(
kinds,
*,
invalid=0,
triggering=8,
validated="false",
covers_final="false",
telemetry_absent="true",
):
return "\t".join(
(
str(invalid),
str(triggering),
validated,
covers_final,
telemetry_absent,
json.dumps(sorted(kinds), separators=(",", ":")),
)
) + "\n"
class DeviceControlCoreMigrationReplayCheckpointRecoveryArtifactTest(
unittest.TestCase
):
def build(self, artifact_dir):
environment = os.environ.copy()
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
result = subprocess.run(
[
"node",
str(BUILDER),
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_PATCH_ID,
],
check=True,
capture_output=True,
text=True,
env=environment,
)
return json.loads(result.stdout)
def test_artifact_is_deterministic_exact_new_identity(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-control-core-checkpoint-recovery-",
) as directory:
artifact_dir = Path(directory)
first = self.build(artifact_dir)
first_bytes = Path(first["artifact"]).read_bytes()
second = self.build(artifact_dir)
second_bytes = Path(second["artifact"]).read_bytes()
self.assertEqual(first_bytes, second_bytes)
self.assertEqual(
first["sha256"],
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ARTIFACT_SHA256,
)
self.assertEqual(first["services"], ["device-control-core"])
self.assertEqual(first["databaseRowMutation"], "none")
self.assertEqual(
first["entries"],
list(
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES
),
)
with tarfile.open(first["artifact"], "r:gz") as archive:
descriptor = json.loads(
archive.extractfile(
"payload/"
+ RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_REL
).read().decode("utf-8")
)
migration = archive.extractfile(
"payload/" + RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
).read()
self.assertEqual(
descriptor,
RUNNER.expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor(),
)
self.assertEqual(
hashlib.sha256(migration).hexdigest(),
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_TARGET_SHA256,
)
self.assertIn(b")) not valid;", migration.lower())
self.assertIsNone(
re.search(
rb"(?im)^\s*(?:delete|update|insert|truncate)\b",
migration,
)
)
def test_runner_selects_only_core_and_exact_build(self):
entries = (
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_CHECKPOINT_RECOVERY_ENTRIES
)
self.assertEqual(
RUNNER.component_services("device-plane", entries),
("device-control-core",),
)
builds = RUNNER.component_builds("device-plane", entries)
self.assertEqual(
builds,
((
RUNNER.DEVICE_PLANE_ROOT,
(
"build",
"--no-cache",
"--network=host",
"-f",
"services/device-control-core/Dockerfile",
"-t",
RUNNER.DEVICE_PLANE_CONTROL_CORE_IMAGE,
".",
),
),),
)
def test_checkpoint_sets_match_every_committed_replay_migration(self):
for phase, expected in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS:
migration_number = phase.split("-", 1)[1]
migration = next(MIGRATION_ROOT.glob(f"{migration_number}_*.sql"))
command_kinds = tuple(re.findall(r"'([^']+)'", migration.read_text()))
with self.subTest(phase=phase):
self.assertEqual(command_kinds, expected)
self.assertIn("not valid", migration.read_text().lower())
def test_every_exact_replay_checkpoint_is_accepted(self):
for phase, kinds in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_CHECKPOINTS:
database_result = mock.Mock(
returncode=0,
stdout=database_stdout(kinds),
stderr="",
)
with self.subTest(phase=phase), mock.patch.object(
RUNNER,
"device_plane_service_container_ids",
return_value=("p" * 64,),
), mock.patch.object(
RUNNER.subprocess,
"run",
return_value=database_result,
):
evidence = (
RUNNER.collect_device_plane_control_core_migration_replay_database_evidence(
expected_state="replay-checkpoint-predecessor",
)
)
self.assertEqual(evidence["constraintPhase"], phase)
self.assertTrue(
evidence["constraintMatchesKnownReplayCheckpoint"]
)
self.assertTrue(evidence["checkpointRecoveryReady"])
self.assertFalse(evidence["finalStateReady"])
def test_unknown_missing_and_validated_checkpoint_states_are_rejected(self):
cases = (
(
"unknown",
database_stdout(("owner_scope.ensure", "unknown.ensure")),
),
(
"missing",
database_stdout((), validated="missing"),
),
(
"validated-checkpoint",
database_stdout(
RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_005_COMMAND_KINDS,
validated="true",
),
),
)
for name, stdout in cases:
database_result = mock.Mock(
returncode=0,
stdout=stdout,
stderr="",
)
with self.subTest(name=name), mock.patch.object(
RUNNER,
"device_plane_service_container_ids",
return_value=("p" * 64,),
), mock.patch.object(
RUNNER.subprocess,
"run",
return_value=database_result,
), self.assertRaises(RUNNER.DeployError):
RUNNER.collect_device_plane_control_core_migration_replay_database_evidence(
expected_state="replay-checkpoint-predecessor",
)
def test_final_acceptance_requires_exact_migration_016_set(self):
database_result = mock.Mock(
returncode=0,
stdout=database_stdout(
RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS,
validated="true",
covers_final="true",
),
stderr="",
)
with mock.patch.object(
RUNNER,
"device_plane_service_container_ids",
return_value=("p" * 64,),
), mock.patch.object(
RUNNER.subprocess,
"run",
return_value=database_result,
):
evidence = (
RUNNER.collect_device_plane_control_core_migration_replay_database_evidence(
expected_state="final",
)
)
self.assertEqual(evidence["constraintPhase"], "final-016")
self.assertTrue(evidence["constraintMatchesFinalKinds"])
self.assertTrue(evidence["finalStateReady"])
self.assertFalse(evidence["checkpointRecoveryReady"])
def test_terminal_044_failure_evidence_is_exact_and_preapply(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-control-core-recovery-044-failure-",
) as directory:
root = Path(directory)
artifact_dir = root / "artifact"
failed_dir = root / "failed"
runner_tmp = root / "tmp"
state_file = root / "applied.jsonl"
failed_state_file = root / "failed.jsonl"
artifact_dir.mkdir()
failed_dir.mkdir()
runner_tmp.mkdir()
environment = os.environ.copy()
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
subprocess.run(
[
"node",
str(LEGACY_BUILDER),
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID,
],
check=True,
capture_output=True,
text=True,
env=environment,
)
source = (
artifact_dir
/ RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT
)
failed_artifact = (
failed_dir
/ RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_ARTIFACT
)
source.replace(failed_artifact)
state_file.write_text("", encoding="utf-8")
record = {
"artifact": failed_artifact.name,
"backup_id": None,
"component": "device-plane",
"failed_at": (
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_FAILED_AT
),
"id": RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_PATCH_ID,
"message": (
"Device Control Core migration recovery database invariant mismatch"
),
"rollback_status": "not-required",
"sha256": (
RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_REPLAY_RECOVERY_ARTIFACT_SHA256
),
"started_apply": False,
"status": "failed",
}
failed_state_file.write_text(
json.dumps(record) + "\n",
encoding="utf-8",
)
with mock.patch.object(RUNNER, "FAILED_DIR", failed_dir), mock.patch.object(
RUNNER,
"STATE_FILE",
state_file,
), mock.patch.object(
RUNNER,
"FAILED_STATE_FILE",
failed_state_file,
), mock.patch.object(RUNNER, "TMP_DIR", runner_tmp):
evidence = (
RUNNER.validate_device_plane_control_core_migration_replay_recovery_failure()
)
self.assertEqual(evidence["record"], record)
def test_checkpoint_recovery_preflight_reaches_database_collector(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-control-core-checkpoint-preflight-",
) as directory:
root = Path(directory)
migration = (
root / RUNNER.DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_REL
)
migration.parent.mkdir(parents=True)
migration.write_bytes(b"migration-014-predecessor")
core = {
"containerId": "c" * 64,
"imageId": RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
"status": "restarting",
"health": "unhealthy",
"restartCount": 280,
}
runtime = {"core": core, "current": {"services": [core]}}
database = {
"constraintPhase": "replay-007",
"checkpointRecoveryReady": True,
}
descriptor = (
RUNNER.expected_device_plane_control_core_migration_replay_checkpoint_recovery_descriptor()
)
with mock.patch.object(
RUNNER,
"validate_device_plane_control_core_migration_replay_checkpoint_recovery_payload",
return_value=descriptor,
), mock.patch.object(
RUNNER,
"validate_device_plane_control_core_double_failure_evidence",
return_value={
"firstBackup": root / "first",
"secondBackup": root / "second",
},
), mock.patch.object(
RUNNER,
"validate_device_plane_control_core_migration_replay_recovery_failure",
return_value={"record": {"started_apply": False}},
), mock.patch.object(
RUNNER,
"component_root",
return_value=root,
), mock.patch.object(
RUNNER,
"DEVICE_PLANE_CONTROL_CORE_MIGRATION_014_PREDECESSOR_SHA256",
hashlib.sha256(migration.read_bytes()).hexdigest(),
), mock.patch.object(
RUNNER,
"validate_device_plane_control_core_migration_replay_preserved_runtime",
return_value=runtime,
), mock.patch.object(
RUNNER,
"inspect_optional_local_image",
return_value=RUNNER.DEVICE_PLANE_CONTROL_CORE_V3_PREAPPLY_IMAGE_ID,
), mock.patch.object(
RUNNER,
"collect_device_plane_control_core_migration_replay_database_evidence",
return_value=database,
) as collector:
evidence = (
RUNNER.validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence(
root
)
)
self.assertEqual(evidence["database"], database)
collector.assert_called_once_with(
expected_state="replay-checkpoint-predecessor",
)
def test_v2_rollback_and_acceptance_are_registered(self):
rollback_source = inspect.getsource(RUNNER.rollback_device_plane_apply)
health_source = inspect.getsource(RUNNER.run_healthchecks)
preflight_source = inspect.getsource(
RUNNER.validate_device_plane_control_core_migration_replay_checkpoint_recovery_evidence
)
acceptance_source = inspect.getsource(
RUNNER.accept_device_plane_control_core_migration_replay_checkpoint_recovery
)
self.assertIn(
"is_device_plane_control_core_migration_replay_checkpoint_recovery_slice",
rollback_source,
)
self.assertIn(
"accept_device_plane_control_core_migration_replay_checkpoint_recovery",
health_source,
)
self.assertIn(
'expected_state="replay-checkpoint-predecessor"',
preflight_source,
)
self.assertIn(
"validate_device_plane_control_core_migration_replay_recovery_failure",
preflight_source,
)
self.assertIn('expected_state="final"', acceptance_source)
if __name__ == "__main__":
unittest.main()
infra/deploy-runner/test_device_control_core_migration_replay_recovery_artifact.py
+29 -5
View File
@@ -139,7 +139,16 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
def test_database_preflight_is_read_only_and_covers_live_receipts(self):
database_result = mock.Mock(
returncode=0,
stdout="0\t8\tfalse\tfalse\ttrue\ttrue\n",
stdout=(
"0\t8\tfalse\tfalse\ttrue\t"
+ json.dumps(
sorted(
RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS
),
separators=(",", ":"),
)
+ "\n"
),
stderr="",
)
with mock.patch.object(
@@ -162,7 +171,9 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
self.assertFalse(evidence["constraintValidated"])
self.assertFalse(evidence["constraintCoversFinalKinds"])
self.assertTrue(evidence["constraintMatchesReplay011Kinds"])
self.assertEqual(evidence["constraintPhase"], "replay-011")
self.assertTrue(evidence["recovery044Ready"])
self.assertTrue(evidence["checkpointRecoveryReady"])
self.assertFalse(evidence["finalStateReady"])
command = run.call_args.args[0]
query = command[command.index("-c") + 1]
@@ -175,8 +186,7 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
)
)
self.assertIn("regexp_matches", query)
for command_kind in RUNNER.DEVICE_PLANE_CONTROL_CORE_REPLAY_011_COMMAND_KINDS:
self.assertIn(f"'{command_kind}'", query)
self.assertIn("array_to_json", query)
def test_runner_predecessor_kind_set_matches_migration_011(self):
migration = MIGRATION_011.read_text(encoding="utf-8")
@@ -190,7 +200,14 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
def test_database_acceptance_requires_final_validated_constraint(self):
database_result = mock.Mock(
returncode=0,
stdout="0\t8\ttrue\ttrue\ttrue\tfalse\n",
stdout=(
"0\t8\ttrue\ttrue\ttrue\t"
+ json.dumps(
sorted(RUNNER.DEVICE_PLANE_CONTROL_CORE_FINAL_COMMAND_KINDS),
separators=(",", ":"),
)
+ "\n"
),
stderr="",
)
with mock.patch.object(
@@ -211,13 +228,20 @@ class DeviceControlCoreMigrationReplayRecoveryArtifactTest(unittest.TestCase):
self.assertTrue(evidence["constraintValidated"])
self.assertTrue(evidence["constraintCoversFinalKinds"])
self.assertFalse(evidence["constraintMatchesReplay011Kinds"])
self.assertEqual(evidence["constraintPhase"], "final-016")
self.assertTrue(evidence["constraintMatchesFinalKinds"])
self.assertFalse(evidence["recovery044Ready"])
self.assertFalse(evidence["checkpointRecoveryReady"])
self.assertTrue(evidence["finalStateReady"])
def test_database_preflight_rejects_ambiguous_constraint_shape(self):
database_result = mock.Mock(
returncode=0,
stdout="0\t8\tfalse\tfalse\ttrue\tfalse\n",
stdout=(
"0\t8\tfalse\tfalse\ttrue\t"
+ json.dumps(["owner_scope.ensure", "unknown.ensure"])
+ "\n"
),
stderr="",
)
with mock.patch.object(