From 37bdbebb4ef0714a95e84224bfa784e8ba291a29 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 11 Aug 2026 09:35:58 +0300 Subject: [PATCH] fix(device-plane): reconcile failed manager activation --- ...er-control-plane-v2-reconciliation-v1.json | 24 ++ .../src/credential-reference.mjs | 29 ++ .../src/sensitive-reference-management.mjs | 2 +- .../sensitive-reference-management.test.mjs | 52 +++ ...-device-manager-control-plane-artifact.mjs | 23 +- ...ntrol-plane-v2-reconciliation-artifact.mjs | 111 +++++ infra/deploy-runner/nodedc-deploy | 396 ++++++++++++++++++ ..._device_manager_control_plane_artifacts.py | 4 + ...r_control_plane_reconciliation_artifact.py | 112 ++++- 9 files changed, 735 insertions(+), 18 deletions(-) create mode 100644 device-plane/deployment/device-manager-control-plane-v2-reconciliation-v1.json create mode 100644 device-plane/services/device-control-core/src/credential-reference.mjs create mode 100644 infra/deploy-runner/build-device-manager-control-plane-v2-reconciliation-artifact.mjs diff --git a/device-plane/deployment/device-manager-control-plane-v2-reconciliation-v1.json b/device-plane/deployment/device-manager-control-plane-v2-reconciliation-v1.json new file mode 100644 index 0000000..8099437 --- /dev/null +++ b/device-plane/deployment/device-manager-control-plane-v2-reconciliation-v1.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": "nodedc.device-plane.device-manager-control-plane-v2-reconciliation.v1", + "mode": "failed-v2-control-plane-baseline-adoption", + "failedPatchId": "device-manager-control-plane-20260811-003", + "failedArtifactSha256": "ba29618ffbfed55448768794f28b18dda439ddb39a1d2a4f1dece19de7f29990", + "backupId": "device-plane-device-manager-control-plane-20260811-003-20260811-012505", + "failureClass": "deterministic-runtime-module-resolution", + "missingModule": "/packages/external-provider-contract/src/credential-reference.mjs", + "correctiveAction": "runtime-local-contract-adapter+staged-module-import-gate", + "sourceAction": "publish-reconciliation-marker-only", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres", + "device-backhaul-target" + ], + "absentService": "device-manager", + "databaseVolume": "nodedc-device-plane-postgres-data", + "publicIngress": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "rollback": "marker-only-runtime-unchanged" +} diff --git a/device-plane/services/device-control-core/src/credential-reference.mjs b/device-plane/services/device-control-core/src/credential-reference.mjs new file mode 100644 index 0000000..85861c3 --- /dev/null +++ b/device-plane/services/device-control-core/src/credential-reference.mjs @@ -0,0 +1,29 @@ +export const NDC_CREDENTIAL_REFERENCE_OWNER = "ndc_l2_credentials"; + +const CREDENTIAL_REFERENCE_PATTERN = + /^ndc-credref:[A-Za-z0-9][A-Za-z0-9._:-]{7,240}$/; + +export function normalizeNdcCredentialReference(input) { + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new TypeError("ndc_credential_reference_invalid"); + } + for (const key of Object.keys(input)) { + if (!new Set(["owner", "reference"]).has(key)) { + throw new TypeError(`ndc_credential_reference_field_unexpected:${key}`); + } + } + if (input.owner !== NDC_CREDENTIAL_REFERENCE_OWNER) { + throw new TypeError("ndc_credential_reference_owner_invalid"); + } + if (!isNdcCredentialReferenceValue(input.reference)) { + throw new TypeError("ndc_credential_reference_value_invalid"); + } + return Object.freeze({ + owner: NDC_CREDENTIAL_REFERENCE_OWNER, + reference: input.reference, + }); +} + +export function isNdcCredentialReferenceValue(value) { + return typeof value === "string" && CREDENTIAL_REFERENCE_PATTERN.test(value); +} diff --git a/device-plane/services/device-control-core/src/sensitive-reference-management.mjs b/device-plane/services/device-control-core/src/sensitive-reference-management.mjs index 5184c5a..97bd246 100644 --- a/device-plane/services/device-control-core/src/sensitive-reference-management.mjs +++ b/device-plane/services/device-control-core/src/sensitive-reference-management.mjs @@ -1,6 +1,6 @@ import { normalizeNdcCredentialReference, -} from "../../../../packages/external-provider-contract/src/credential-reference.mjs"; +} from "./credential-reference.mjs"; export const DEVICE_SENSITIVE_REFERENCE_COMMAND_KINDS = Object.freeze([ "device_credential_binding.upsert", diff --git a/device-plane/services/device-control-core/test/sensitive-reference-management.test.mjs b/device-plane/services/device-control-core/test/sensitive-reference-management.test.mjs index 9bd9cd2..7c44748 100644 --- a/device-plane/services/device-control-core/test/sensitive-reference-management.test.mjs +++ b/device-plane/services/device-control-core/test/sensitive-reference-management.test.mjs @@ -9,6 +9,12 @@ import { ALL_DEVICE_MANAGEMENT_COMMAND_KINDS, normalizeDeviceManagementCommand, } from "../src/management-command.mjs"; +import { + normalizeNdcCredentialReference as normalizeRuntimeCredentialReference, +} from "../src/credential-reference.mjs"; +import { + normalizeNdcCredentialReference as normalizePlatformCredentialReference, +} from "../../../../packages/external-provider-contract/src/credential-reference.mjs"; const projectRef = "project:11111111-1111-4111-8111-111111111111"; const deviceRef = "device:22222222-2222-4222-8222-222222222222"; @@ -65,6 +71,52 @@ test("credential binding accepts only the platform canonical opaque ref", () => ); }); +test("runtime credential reference adapter matches the platform contract", () => { + const accepted = [ + { + owner: "ndc_l2_credentials", + reference: "ndc-credref:pilot-command-0001", + }, + { + owner: "ndc_l2_credentials", + reference: "ndc-credref:A1234567", + }, + ]; + for (const input of accepted) { + assert.deepEqual( + normalizeRuntimeCredentialReference(input), + normalizePlatformCredentialReference(input), + ); + } + + const rejected = [ + null, + [], + { owner: "device_core", reference: "ndc-credref:pilot-command-0001" }, + { owner: "ndc_l2_credentials", reference: "secret:test" }, + { + owner: "ndc_l2_credentials", + reference: "ndc-credref:pilot-command-0001", + token: "forbidden", + }, + ]; + for (const input of rejected) { + let runtimeError; + let platformError; + try { + normalizeRuntimeCredentialReference(input); + } catch (error) { + runtimeError = error; + } + try { + normalizePlatformCredentialReference(input); + } catch (error) { + platformError = error; + } + assert.equal(runtimeError?.message, platformError?.message); + } +}); + test("credential binding rejects raw secret-shaped fields", () => { for (const field of ["password", "token", "secretValue", "endpoint"]) { assert.throws( diff --git a/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs b/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs index d41dc9a..b36c5a3 100644 --- a/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs +++ b/infra/deploy-runner/build-device-manager-control-plane-artifact.mjs @@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process"; import { cp, lstat, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const platformRoot = resolve(scriptDir, "../.."); @@ -50,6 +50,27 @@ try { } await copySafe(resolve(devicePlaneRoot, entry), join(payload, entry), devicePlaneRoot); } + const coreImport = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import(${JSON.stringify(pathToFileURL(join( + payload, + "services/device-control-core/src/sensitive-reference-management.mjs", + )).href)})`, + ], + { + cwd: payload, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }, + ); + if (coreImport.status !== 0) { + throw new Error( + `device_control_core_staged_module_import_failed:${coreImport.stderr || coreImport.stdout}`, + ); + } const compose = await readFile(join(payload, "docker-compose.device-manager.yml"), "utf8"); for (const required of [ "device-manager:", diff --git a/infra/deploy-runner/build-device-manager-control-plane-v2-reconciliation-artifact.mjs b/infra/deploy-runner/build-device-manager-control-plane-v2-reconciliation-artifact.mjs new file mode 100644 index 0000000..5de795b --- /dev/null +++ b/infra/deploy-runner/build-device-manager-control-plane-v2-reconciliation-artifact.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const sourceRoot = resolve(platformRoot, "device-plane"); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-manager-control-plane-v2-reconciliation-20260811-004", + ...extra +] = process.argv.slice(2); +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-manager-control-plane-v2-reconciliation-artifact.mjs [patch-id]", + ); +} + +const entry = "deployment/device-manager-control-plane-v2-reconciliation-v1.json"; +const stage = await mkdtemp(join(tmpdir(), "nodedc-device-manager-v2-reconciliation-")); +const payload = join(stage, "payload"); +const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`); + +try { + const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entry), "utf8")); + const expected = { + schemaVersion: "nodedc.device-plane.device-manager-control-plane-v2-reconciliation.v1", + mode: "failed-v2-control-plane-baseline-adoption", + failedPatchId: "device-manager-control-plane-20260811-003", + failedArtifactSha256: + "ba29618ffbfed55448768794f28b18dda439ddb39a1d2a4f1dece19de7f29990", + backupId: + "device-plane-device-manager-control-plane-20260811-003-20260811-012505", + failureClass: "deterministic-runtime-module-resolution", + missingModule: "/packages/external-provider-contract/src/credential-reference.mjs", + correctiveAction: "runtime-local-contract-adapter+staged-module-import-gate", + sourceAction: "publish-reconciliation-marker-only", + runtimeAction: "read-only-acceptance", + preservedServices: [ + "device-control-core", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + absentService: "device-manager", + databaseVolume: "nodedc-device-plane-postgres-data", + publicIngress: "disabled", + commandTransport: "disabled", + gelios: "untouched", + rollback: "marker-only-runtime-unchanged", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error("device_manager_v2_reconciliation_descriptor_mismatch"); + } + + await mkdir(dirname(join(payload, entry)), { recursive: true }); + await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true }); + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${entry}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, + ); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + const sha256 = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + component: "device-plane", + artifact: target, + sha256, + entries: [entry], + build: [], + services: [], + transition: "failed-v2-control-plane-baseline-adoption", + runtimeAction: "read-only-acceptance", + sourceAction: "publish-reconciliation-marker-only", + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 4879c5a..bafa1b4 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -315,6 +315,64 @@ DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING = ( "services/device-manager", DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_REL, ) +DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL = ( + "deployment/device-manager-control-plane-v2-reconciliation-v1.json" +) +DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES = ( + DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, +) +DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID = ( + "device-manager-control-plane-20260811-003" +) +DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-manager-control-plane-20260811-003.tgz." + "20260811-012505" +) +DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 = ( + "ba29618ffbfed55448768794f28b18dda439ddb39a1d2a4f1dece19de7f29990" +) +DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID = ( + "device-manager-control-plane-v2-reconciliation-20260811-004" +) +DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID = ( + "device-plane-device-manager-control-plane-20260811-003-" + "20260811-012505" +) +DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_SHA256 = { + "manifest.env": ( + "4f7535e5eeee0bc3339173c2425787c153528d26f2c60f6cdd386ad3359e05cf" + ), + "files.txt": ( + "191e81991258673881435083e4a6733062ec0e30afb9b80a875c1a6581dd18a2" + ), + "existing-files.txt": ( + "b605a86ce5ed6c0adb7b4cbeb5758795fe4219367df2c18db9411825f232e308" + ), + "missing-files.txt": ( + "c1ab9da97b257b467fd222eda0e34b5a7b29b628c71f10a5f4069b2991b3d39d" + ), + "source-before.tgz": ( + "32d6d6341e410806bcff7007c5eacec34090bc337483222f396f19ca2edad9be" + ), + "runtime-before.json": ( + "af8f2c71caeedcd61ebf02de98ff6c61d35c832fbd446a6b38a7e7c16150c2bd" + ), +} +DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING = ( + ".dockerignore", + "package.json", + "package-lock.json", + "packages/device-protocol-contract", + "packages/arusnavi-b2-adapter", + "services/device-control-core", + "services/device-gateway/package.json", + "services/device-edge-relay/package.json", +) +DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING = ( + DEVICE_PLANE_MANAGER_COMPOSE_REL, + "services/device-manager", + DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL, +) DEVICE_PLANE_FOUNDATION_RECOVERY_REL = ( "deployment/device-plane-foundation-recovery-v1.json" ) @@ -3202,6 +3260,7 @@ def allowed_payload_path(component, rel): DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_REL, DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL, DEVICE_PLANE_MANAGER_RECONCILIATION_REL, + DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, "packages/device-protocol-contract", "packages/arusnavi-b2-adapter", "services/device-control-core", @@ -8259,6 +8318,11 @@ def load_artifact(artifact, work_dir): entries, ): validate_device_plane_manager_reconciliation_payload(payload_dir) + if is_device_plane_manager_v2_reconciliation_slice( + manifest["component"], + entries, + ): + validate_device_plane_manager_v2_reconciliation_payload(payload_dir) if is_device_plane_postgres_bootstrap_slice( manifest["component"], entries, @@ -8610,6 +8674,14 @@ def reject_terminal_device_plane_manager_artifact( "Device Manager control-plane 001 is terminal failed; " "use reconciliation followed by the exact v2 activation successor" ) + if ( + manifest.get("id") == DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID + or sha256 == DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 + ): + die( + "Device Manager control-plane 003 is terminal failed; " + "use the exact v2 reconciliation successor" + ) def reject_terminal_device_plane_backhaul_artifact(manifest, sha256): @@ -8690,6 +8762,15 @@ def is_device_plane_manager_reconciliation_slice(component, entries): ) +def is_device_plane_manager_v2_reconciliation_slice(component, entries): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) + == DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES + ) + + def expected_platform_device_core_hub_trust_descriptor(): return { "schemaVersion": "nodedc.platform.device-core-hub-trust.v1", @@ -8780,6 +8861,43 @@ def expected_device_plane_manager_reconciliation_descriptor(): } +def expected_device_plane_manager_v2_reconciliation_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane.device-manager-control-plane-" + "v2-reconciliation.v1" + ), + "mode": "failed-v2-control-plane-baseline-adoption", + "failedPatchId": DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID, + "failedArtifactSha256": ( + DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 + ), + "backupId": DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID, + "failureClass": "deterministic-runtime-module-resolution", + "missingModule": ( + "/packages/external-provider-contract/src/" + "credential-reference.mjs" + ), + "correctiveAction": ( + "runtime-local-contract-adapter+staged-module-import-gate" + ), + "sourceAction": "publish-reconciliation-marker-only", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres", + "device-backhaul-target", + ], + "absentService": "device-manager", + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "publicIngress": "disabled", + "commandTransport": "disabled", + "gelios": "untouched", + "rollback": "marker-only-runtime-unchanged", + } + + def validate_platform_device_core_hub_trust_payload(payload_dir): descriptor = read_strict_json( payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL, @@ -8889,6 +9007,23 @@ def validate_device_plane_manager_reconciliation_payload(payload_dir): return descriptor +def validate_device_plane_manager_v2_reconciliation_payload(payload_dir): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL, + "Device Manager v2 control-plane reconciliation descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_manager_v2_reconciliation_descriptor() + ): + die( + "Device Manager v2 control-plane reconciliation " + "descriptor mismatch" + ) + return descriptor + + def is_device_plane_postgres_bootstrap_slice(component, entries): return ( component == "device-plane" @@ -10369,6 +10504,185 @@ def validate_device_plane_manager_activation_predecessor(payload_dir): } +def validate_device_plane_manager_v2_reconciliation_backup(): + backup_dir = ( + BACKUPS_DIR / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID + ) + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Manager v2 failed-apply backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Manager v2 failed-apply backup is unsafe") + expected = DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_SHA256 + if {child.name for child in backup_dir.iterdir()} != set(expected): + die("Device Manager v2 failed-apply backup file set mismatch") + for name, expected_sha256 in expected.items(): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Device Manager v2 failed-apply backup drift detected: " + f"{name}" + ) + existing = tuple(read_backup_path_list( + backup_dir / "existing-files.txt" + )) + missing = tuple(read_backup_path_list( + backup_dir / "missing-files.txt" + )) + validate_backup_partition( + DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES, + existing, + missing, + "Device Manager v2 reconciliation", + ) + if ( + existing != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING + or missing != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING + ): + die("Device Manager v2 failed-apply backup partition mismatch") + return backup_dir + + +def validate_device_plane_manager_v2_reconciled_baseline( + backup_dir, + *, + marker_installed, +): + prior_backup = validate_device_plane_manager_reconciliation_backup() + runtime = validate_device_plane_manager_reconciled_baseline( + prior_backup, + marker_installed=True, + ) + root = component_root("device-plane") + with tempfile.TemporaryDirectory( + prefix="device-manager-v2-reconciled-baseline-", + dir=TMP_DIR, + ) as directory: + backup_root = Path(directory) + materialize_backup_tree( + backup_dir / "source-before.tgz", + backup_root, + set(DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING), + ) + backup_source = collect_exact_files( + backup_root, + DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING, + "Device Manager v2 pre-apply source", + ) + live_source = collect_exact_files( + root, + DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING, + "Device Manager v2 reconciled live source", + ) + if live_source != backup_source: + die("Device Manager v2 rollback source does not match backup") + for rel in DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING: + path = root / rel + if path.exists() or path.is_symlink(): + die( + "Device Manager v2 candidate-only source remains installed: " + f"{rel}" + ) + + marker = root / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL + if marker_installed: + descriptor = read_strict_json( + marker, + "installed Device Manager v2 reconciliation descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_manager_v2_reconciliation_descriptor() + ): + die("installed Device Manager v2 reconciliation descriptor mismatch") + elif marker.exists() or marker.is_symlink(): + die("Device Manager v2 reconciliation descriptor already installed") + return runtime + + +def validate_device_plane_manager_v2_reconciliation_evidence(payload_dir): + descriptor = validate_device_plane_manager_v2_reconciliation_payload( + payload_dir + ) + backup_dir = validate_device_plane_manager_v2_reconciliation_backup() + failed_artifact = FAILED_DIR / DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Manager v2 failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 + ): + die("Device Manager v2 failed artifact evidence mismatch") + + records = [ + value + for value in load_state(FAILED_STATE_FILE) + if value.get("id") == DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID + ] + if len(records) != 1: + die("Device Manager v2 failed journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") != DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "failed:DeployError" + or record.get("status") != "failed" + or record.get("message") + != ( + "container healthcheck grace exhausted for " + "f8593303db6b71468716ed2428e54aab18d7303b34adbe1a93cd6df86effa1cf: " + "unhealthy" + ) + ): + die("Device Manager v2 failed journal evidence mismatch") + + with tempfile.TemporaryDirectory( + prefix="device-manager-v2-failed-artifact-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, _failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") != DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES + ): + die("Device Manager v2 failed artifact contract mismatch") + + runtime = validate_device_plane_manager_v2_reconciled_baseline( + backup_dir, + marker_installed=False, + ) + return { + "mode": descriptor["mode"], + "backup": backup_dir, + "failedArtifact": failed_artifact, + "runtime": runtime, + } + + def device_plane_service_container_ids(service): if service not in ( *DEVICE_PLANE_RUNTIME_SERVICES, @@ -13558,6 +13872,11 @@ def component_services(component, entries=None): # evidence and the already-restored baseline. Runtime is read-only. return () + if is_device_plane_manager_v2_reconciliation_slice(component, entries): + # The failed v2 apply already restored the prior source and later + # converged to the healthy baseline. Publish evidence only. + return () + if is_device_plane_postgres_bootstrap_slice(component, entries): # This exact one-time transition is the only Device Plane artifact that # may select durable state. Its preflight requires both container and @@ -13976,6 +14295,9 @@ def component_builds(component, entries=None): if is_device_plane_manager_reconciliation_slice(component, entries): return () + if is_device_plane_manager_v2_reconciliation_slice(component, entries): + return () + if is_device_plane_backhaul_vps_enrollment_slice(component, entries): return () @@ -16000,6 +16322,7 @@ def plan_artifact(artifact): device_plane_b2_recovery_preflight = None device_plane_manager_activation_preflight = None device_plane_manager_reconciliation_preflight = None + device_plane_manager_v2_reconciliation_preflight = None device_plane_backhaul_preflight = None device_plane_backhaul_vps_enrollment_preflight = None device_plane_runtime_before = None @@ -16227,6 +16550,15 @@ def plan_artifact(artifact): payload_dir ) ) + if is_device_plane_manager_v2_reconciliation_slice( + manifest["component"], + entries, + ): + device_plane_manager_v2_reconciliation_preflight = ( + validate_device_plane_manager_v2_reconciliation_evidence( + payload_dir + ) + ) if is_device_plane_backhaul_target_slice( manifest["component"], entries, @@ -17841,6 +18173,42 @@ def plan_artifact(artifact): print("device_gateway_tcp_9921=preserved:loopback-only") print("device_manager_public_route=unchanged:absent") print("device_plane_rollback=marker-only-runtime-unchanged") + if device_plane_manager_v2_reconciliation_preflight is not None: + print( + "device_plane_transition=" + f"{device_plane_manager_v2_reconciliation_preflight['mode']}" + ) + print( + "failed_patch=" + f"{DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID}" + ) + print( + "failed_artifact_sha256=" + f"{DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256}" + ) + print( + "recovery_backup=" + f"{device_plane_manager_v2_reconciliation_preflight['backup'].name}" + ) + print( + "device_plane_failure_class=" + "deterministic-runtime-module-resolution" + ) + print("device_plane_build=none") + print("device_plane_runtime_mutation=none") + print( + "device_plane_runtime_services=" + "preserved:device-control-core,device-gateway," + "device-postgres,device-backhaul-target" + ) + print("device_manager=absent") + print( + "device_plane_source_action=" + "publish-v2-reconciliation-marker-only" + ) + print("device_gateway_tcp_9921=preserved:loopback-only") + print("device_manager_public_route=unchanged:absent") + print("device_plane_rollback=marker-only-runtime-unchanged") if device_plane_backhaul_vps_enrollment_preflight is not None: print( "device_plane_transition=" @@ -19356,6 +19724,8 @@ def run_component_runtime(component, entries, services): return if is_device_plane_manager_reconciliation_slice(component, entries): return + if is_device_plane_manager_v2_reconciliation_slice(component, entries): + return if is_device_plane_foundation_recovery_slice(component, entries): return if is_device_plane_foundation_network_publication_slice( @@ -19386,6 +19756,8 @@ def run_device_plane_runtime_for_apply( "device-plane", entries ) or is_device_plane_manager_reconciliation_slice( "device-plane", entries + ) or is_device_plane_manager_v2_reconciliation_slice( + "device-plane", entries ): return # Build and runtime preparation are pre-runtime phases. A failure here @@ -20132,6 +20504,23 @@ def run_healthchecks(component, entries=None, services=None): ) return + if is_device_plane_manager_v2_reconciliation_slice(component, entries): + if tuple(services or ()) != (): + die("Device Manager v2 reconciliation service set mismatch") + for service in DEVICE_PLANE_RUNTIME_SERVICES: + healthcheck_compose_service_with_grace( + "device-plane", + service, + ) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + backup_dir = validate_device_plane_manager_v2_reconciliation_backup() + validate_device_plane_manager_v2_reconciled_baseline( + backup_dir, + marker_installed=True, + ) + return + if is_device_plane_backhaul_vps_enrollment_slice(component, entries): if tuple(services or ()) != (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,): die("Device Plane VPS enrollment service set mismatch") @@ -21112,6 +21501,13 @@ def apply_artifact(artifact): validate_device_plane_manager_reconciliation_evidence( payload_dir ) + if is_device_plane_manager_v2_reconciliation_slice( + component, + entries, + ): + validate_device_plane_manager_v2_reconciliation_evidence( + payload_dir + ) if is_device_plane_backhaul_target_slice( component, entries, diff --git a/infra/deploy-runner/test_device_manager_control_plane_artifacts.py b/infra/deploy-runner/test_device_manager_control_plane_artifacts.py index 6da2a4c..c621977 100644 --- a/infra/deploy-runner/test_device_manager_control_plane_artifacts.py +++ b/infra/deploy-runner/test_device_manager_control_plane_artifacts.py @@ -156,6 +156,10 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase): "payload/services/device-manager/server/device-manager-server.mjs", names, ) + self.assertIn( + "payload/services/device-control-core/src/credential-reference.mjs", + names, + ) self.assertFalse(any(name.endswith(".test.mjs") for name in names)) self.assertEqual(result["services"], ["device-control-core", "device-manager"]) self.assertNotIn("device-postgres", result["services"]) diff --git a/infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py b/infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py index af47f6f..2f7cbf8 100644 --- a/infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py +++ b/infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py @@ -18,6 +18,10 @@ BUILDER = ( SCRIPT_DIR / "build-device-manager-control-plane-reconciliation-artifact.mjs" ) +V2_BUILDER = ( + SCRIPT_DIR + / "build-device-manager-control-plane-v2-reconciliation-artifact.mjs" +) def load_runner(): @@ -35,11 +39,11 @@ RUNNER = load_runner() class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase): - def build(self, artifact_dir, patch_id): + def build(self, artifact_dir, patch_id, builder=BUILDER): environment = os.environ.copy() environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) result = subprocess.run( - ["node", str(BUILDER), patch_id], + ["node", str(builder), patch_id], check=True, capture_output=True, text=True, @@ -145,6 +149,79 @@ class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase): RUNNER.DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_ENTRIES, ) + def test_v2_reconciliation_is_marker_only_exact_and_deterministic(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-manager-v2-reconciliation-artifact-", + ) as directory: + artifact_dir = Path(directory) + first = self.build( + artifact_dir, + "device-manager-control-plane-v2-reconciliation-unit-004", + V2_BUILDER, + ) + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build( + artifact_dir, + "device-manager-control-plane-v2-reconciliation-unit-004", + V2_BUILDER, + ) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + self.assertEqual( + first["entries"], + list(RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES), + ) + self.assertEqual(first["build"], []) + self.assertEqual(first["services"], []) + with tarfile.open(first["artifact"], "r:gz") as archive: + descriptor = json.loads( + archive.extractfile( + "payload/" + + RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL + ).read().decode("utf-8") + ) + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_manager_v2_reconciliation_descriptor(), + ) + self.assertEqual( + RUNNER.component_services( + "device-plane", + RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES, + ), + (), + ) + self.assertEqual( + RUNNER.component_builds( + "device-plane", + RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES, + ), + (), + ) + + def test_failed_v2_control_plane_is_terminal(self): + with self.assertRaisesRegex( + RUNNER.DeployError, + "exact v2 reconciliation successor", + ): + RUNNER.reject_terminal_device_plane_manager_artifact( + {"id": RUNNER.DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID}, + "0" * 64, + ) + with self.assertRaisesRegex( + RUNNER.DeployError, + "exact v2 reconciliation successor", + ): + RUNNER.reject_terminal_device_plane_manager_artifact( + {"id": "different"}, + RUNNER.DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256, + ) + def test_health_grace_waits_through_unhealthy_and_exited(self): results = [ mock.Mock(stdout="unhealthy\n", stderr="", returncode=0), @@ -266,21 +343,24 @@ class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase): generic_health.assert_not_called() def test_reconciliation_runtime_phase_is_read_only(self): - entries = RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES - with ( - mock.patch.object(RUNNER, "run_build") as build, - mock.patch.object(RUNNER, "prepare_component_runtime") as prepare, - mock.patch.object(RUNNER, "run_compose") as compose, + for entries in ( + RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES, + RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES, ): - RUNNER.run_component_runtime("device-plane", entries, ()) - RUNNER.run_device_plane_runtime_for_apply( - entries, - (), - mock.Mock(), - ) - build.assert_not_called() - prepare.assert_not_called() - compose.assert_not_called() + with ( + mock.patch.object(RUNNER, "run_build") as build, + mock.patch.object(RUNNER, "prepare_component_runtime") as prepare, + mock.patch.object(RUNNER, "run_compose") as compose, + ): + RUNNER.run_component_runtime("device-plane", entries, ()) + RUNNER.run_device_plane_runtime_for_apply( + entries, + (), + mock.Mock(), + ) + build.assert_not_called() + prepare.assert_not_called() + compose.assert_not_called() if __name__ == "__main__":