SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(device-plane): reconcile failed manager activation

Browse Source
This commit is contained in:
Codex
2026-08-11 09:35:58 +03:00
parent 8defd55715
commit 37bdbebb4e
9 changed files with 735 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
device-plane/deployment/device-manager-control-plane-v2-reconciliation-v1.json
+24
View File
@@ -0,0 +1,24 @@
{
"schemaVersion": "nodedc.device-plane.device-manager-control-plane-v2-reconciliation.v1",
"mode": "failed-v2-control-plane-baseline-adoption",
"failedPatchId": "device-manager-control-plane-20260811-003",
"failedArtifactSha256": "ba29618ffbfed55448768794f28b18dda439ddb39a1d2a4f1dece19de7f29990",
"backupId": "device-plane-device-manager-control-plane-20260811-003-20260811-012505",
"failureClass": "deterministic-runtime-module-resolution",
"missingModule": "/packages/external-provider-contract/src/credential-reference.mjs",
"correctiveAction": "runtime-local-contract-adapter+staged-module-import-gate",
"sourceAction": "publish-reconciliation-marker-only",
"runtimeAction": "read-only-acceptance",
"preservedServices": [
"device-control-core",
"device-gateway",
"device-postgres",
"device-backhaul-target"
],
"absentService": "device-manager",
"databaseVolume": "nodedc-device-plane-postgres-data",
"publicIngress": "disabled",
"commandTransport": "disabled",
"gelios": "untouched",
"rollback": "marker-only-runtime-unchanged"
}
device-plane/services/device-control-core/src/credential-reference.mjs
+29
View File
@@ -0,0 +1,29 @@
export const NDC_CREDENTIAL_REFERENCE_OWNER = "ndc_l2_credentials";
const CREDENTIAL_REFERENCE_PATTERN =
/^ndc-credref:[A-Za-z0-9][A-Za-z0-9._:-]{7,240}$/;
export function normalizeNdcCredentialReference(input) {
if (!input || typeof input !== "object" || Array.isArray(input)) {
throw new TypeError("ndc_credential_reference_invalid");
}
for (const key of Object.keys(input)) {
if (!new Set(["owner", "reference"]).has(key)) {
throw new TypeError(`ndc_credential_reference_field_unexpected:${key}`);
}
}
if (input.owner !== NDC_CREDENTIAL_REFERENCE_OWNER) {
throw new TypeError("ndc_credential_reference_owner_invalid");
}
if (!isNdcCredentialReferenceValue(input.reference)) {
throw new TypeError("ndc_credential_reference_value_invalid");
}
return Object.freeze({
owner: NDC_CREDENTIAL_REFERENCE_OWNER,
reference: input.reference,
});
}
export function isNdcCredentialReferenceValue(value) {
return typeof value === "string" && CREDENTIAL_REFERENCE_PATTERN.test(value);
}
device-plane/services/device-control-core/src/sensitive-reference-management.mjs
+1 -1
View File
@@ -1,6 +1,6 @@
import {
normalizeNdcCredentialReference,
} from "../../../../packages/external-provider-contract/src/credential-reference.mjs";
} from "./credential-reference.mjs";
export const DEVICE_SENSITIVE_REFERENCE_COMMAND_KINDS = Object.freeze([
"device_credential_binding.upsert",
device-plane/services/device-control-core/test/sensitive-reference-management.test.mjs
+52
View File
@@ -9,6 +9,12 @@ import {
ALL_DEVICE_MANAGEMENT_COMMAND_KINDS,
normalizeDeviceManagementCommand,
} from "../src/management-command.mjs";
import {
normalizeNdcCredentialReference as normalizeRuntimeCredentialReference,
} from "../src/credential-reference.mjs";
import {
normalizeNdcCredentialReference as normalizePlatformCredentialReference,
} from "../../../../packages/external-provider-contract/src/credential-reference.mjs";
const projectRef = "project:11111111-1111-4111-8111-111111111111";
const deviceRef = "device:22222222-2222-4222-8222-222222222222";
@@ -65,6 +71,52 @@ test("credential binding accepts only the platform canonical opaque ref", () =>
);
});
test("runtime credential reference adapter matches the platform contract", () => {
const accepted = [
{
owner: "ndc_l2_credentials",
reference: "ndc-credref:pilot-command-0001",
},
{
owner: "ndc_l2_credentials",
reference: "ndc-credref:A1234567",
},
];
for (const input of accepted) {
assert.deepEqual(
normalizeRuntimeCredentialReference(input),
normalizePlatformCredentialReference(input),
);
}
const rejected = [
null,
[],
{ owner: "device_core", reference: "ndc-credref:pilot-command-0001" },
{ owner: "ndc_l2_credentials", reference: "secret:test" },
{
owner: "ndc_l2_credentials",
reference: "ndc-credref:pilot-command-0001",
token: "forbidden",
},
];
for (const input of rejected) {
let runtimeError;
let platformError;
try {
normalizeRuntimeCredentialReference(input);
} catch (error) {
runtimeError = error;
}
try {
normalizePlatformCredentialReference(input);
} catch (error) {
platformError = error;
}
assert.equal(runtimeError?.message, platformError?.message);
}
});
test("credential binding rejects raw secret-shaped fields", () => {
for (const field of ["password", "token", "secretValue", "endpoint"]) {
assert.throws(
infra/deploy-runner/build-device-manager-control-plane-artifact.mjs
+22 -1
View File
@@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process";
import { cp, lstat, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { fileURLToPath, pathToFileURL } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../..");
@@ -50,6 +50,27 @@ try {
}
await copySafe(resolve(devicePlaneRoot, entry), join(payload, entry), devicePlaneRoot);
}
const coreImport = spawnSync(
process.execPath,
[
"--input-type=module",
"--eval",
`import(${JSON.stringify(pathToFileURL(join(
payload,
"services/device-control-core/src/sensitive-reference-management.mjs",
)).href)})`,
],
{
cwd: payload,
encoding: "utf8",
maxBuffer: 16 * 1024 * 1024,
},
);
if (coreImport.status !== 0) {
throw new Error(
`device_control_core_staged_module_import_failed:${coreImport.stderr || coreImport.stdout}`,
);
}
const compose = await readFile(join(payload, "docker-compose.device-manager.yml"), "utf8");
for (const required of [
"device-manager:",
infra/deploy-runner/build-device-manager-control-plane-v2-reconciliation-artifact.mjs
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../..");
const sourceRoot = resolve(platformRoot, "device-plane");
const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"),
);
const [
patchId = "device-manager-control-plane-v2-reconciliation-20260811-004",
...extra
] = process.argv.slice(2);
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
throw new Error(
"usage: build-device-manager-control-plane-v2-reconciliation-artifact.mjs [patch-id]",
);
}
const entry = "deployment/device-manager-control-plane-v2-reconciliation-v1.json";
const stage = await mkdtemp(join(tmpdir(), "nodedc-device-manager-v2-reconciliation-"));
const payload = join(stage, "payload");
const target = resolve(artifactDir, `nodedc-device-plane-${patchId}.tgz`);
try {
const descriptor = JSON.parse(await readFile(resolve(sourceRoot, entry), "utf8"));
const expected = {
schemaVersion: "nodedc.device-plane.device-manager-control-plane-v2-reconciliation.v1",
mode: "failed-v2-control-plane-baseline-adoption",
failedPatchId: "device-manager-control-plane-20260811-003",
failedArtifactSha256:
"ba29618ffbfed55448768794f28b18dda439ddb39a1d2a4f1dece19de7f29990",
backupId:
"device-plane-device-manager-control-plane-20260811-003-20260811-012505",
failureClass: "deterministic-runtime-module-resolution",
missingModule: "/packages/external-provider-contract/src/credential-reference.mjs",
correctiveAction: "runtime-local-contract-adapter+staged-module-import-gate",
sourceAction: "publish-reconciliation-marker-only",
runtimeAction: "read-only-acceptance",
preservedServices: [
"device-control-core",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
absentService: "device-manager",
databaseVolume: "nodedc-device-plane-postgres-data",
publicIngress: "disabled",
commandTransport: "disabled",
gelios: "untouched",
rollback: "marker-only-runtime-unchanged",
};
if (JSON.stringify(descriptor) !== JSON.stringify(expected)) {
throw new Error("device_manager_v2_reconciliation_descriptor_mismatch");
}
await mkdir(dirname(join(payload, entry)), { recursive: true });
await cp(resolve(sourceRoot, entry), join(payload, entry), { force: true });
await writeFile(
join(stage, "manifest.env"),
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
"utf8",
);
await writeFile(join(stage, "files.txt"), `${entry}\n`, "utf8");
await mkdir(artifactDir, { recursive: true });
const tar = spawnSync(
"python3",
["-c", canonicalTarScript(), target, stage],
{ encoding: "utf8", maxBuffer: 16 * 1024 * 1024 },
);
if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
const sha256 = createHash("sha256")
.update(await readFile(target))
.digest("hex");
console.log(JSON.stringify({
ok: true,
patchId,
component: "device-plane",
artifact: target,
sha256,
entries: [entry],
build: [],
services: [],
transition: "failed-v2-control-plane-baseline-adoption",
runtimeAction: "read-only-acceptance",
sourceAction: "publish-reconciliation-marker-only",
}, null, 2));
} finally {
await rm(stage, { recursive: true, force: true });
}
function canonicalTarScript() {
return [
"import gzip,io,pathlib,sys,tarfile",
"root=pathlib.Path(sys.argv[2])",
"with open(sys.argv[1],'wb') as out:",
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
" for top in ('manifest.env','files.txt','payload'):",
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
" for x in paths:",
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix()); info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
].join("\n");
}
infra/deploy-runner/nodedc-deploy
+396
View File
@@ -315,6 +315,64 @@ DEVICE_PLANE_MANAGER_RECONCILIATION_MISSING = (
"services/device-manager",
DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_REL,
)
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL = (
"deployment/device-manager-control-plane-v2-reconciliation-v1.json"
)
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES = (
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
)
DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID = (
"device-manager-control-plane-20260811-003"
)
DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT = (
"nodedc-device-plane-device-manager-control-plane-20260811-003.tgz."
"20260811-012505"
)
DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256 = (
"ba29618ffbfed55448768794f28b18dda439ddb39a1d2a4f1dece19de7f29990"
)
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_PATCH_ID = (
"device-manager-control-plane-v2-reconciliation-20260811-004"
)
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID = (
"device-plane-device-manager-control-plane-20260811-003-"
"20260811-012505"
)
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_SHA256 = {
"manifest.env": (
"4f7535e5eeee0bc3339173c2425787c153528d26f2c60f6cdd386ad3359e05cf"
),
"files.txt": (
"191e81991258673881435083e4a6733062ec0e30afb9b80a875c1a6581dd18a2"
),
"existing-files.txt": (
"b605a86ce5ed6c0adb7b4cbeb5758795fe4219367df2c18db9411825f232e308"
),
"missing-files.txt": (
"c1ab9da97b257b467fd222eda0e34b5a7b29b628c71f10a5f4069b2991b3d39d"
),
"source-before.tgz": (
"32d6d6341e410806bcff7007c5eacec34090bc337483222f396f19ca2edad9be"
),
"runtime-before.json": (
"af8f2c71caeedcd61ebf02de98ff6c61d35c832fbd446a6b38a7e7c16150c2bd"
),
}
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING = (
".dockerignore",
"package.json",
"package-lock.json",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway/package.json",
"services/device-edge-relay/package.json",
)
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING = (
DEVICE_PLANE_MANAGER_COMPOSE_REL,
"services/device-manager",
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
)
DEVICE_PLANE_FOUNDATION_RECOVERY_REL = (
"deployment/device-plane-foundation-recovery-v1.json"
)
@@ -3202,6 +3260,7 @@ def allowed_payload_path(component, rel):
DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_REL,
DEVICE_PLANE_MANAGER_CONTROL_PLANE_REL,
DEVICE_PLANE_MANAGER_RECONCILIATION_REL,
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
@@ -8259,6 +8318,11 @@ def load_artifact(artifact, work_dir):
entries,
):
validate_device_plane_manager_reconciliation_payload(payload_dir)
if is_device_plane_manager_v2_reconciliation_slice(
manifest["component"],
entries,
):
validate_device_plane_manager_v2_reconciliation_payload(payload_dir)
if is_device_plane_postgres_bootstrap_slice(
manifest["component"],
entries,
@@ -8610,6 +8674,14 @@ def reject_terminal_device_plane_manager_artifact(
"Device Manager control-plane 001 is terminal failed; "
"use reconciliation followed by the exact v2 activation successor"
)
if (
manifest.get("id") == DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID
or sha256 == DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
):
die(
"Device Manager control-plane 003 is terminal failed; "
"use the exact v2 reconciliation successor"
)
def reject_terminal_device_plane_backhaul_artifact(manifest, sha256):
@@ -8690,6 +8762,15 @@ def is_device_plane_manager_reconciliation_slice(component, entries):
)
def is_device_plane_manager_v2_reconciliation_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries)
== DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES
)
def expected_platform_device_core_hub_trust_descriptor():
return {
"schemaVersion": "nodedc.platform.device-core-hub-trust.v1",
@@ -8780,6 +8861,43 @@ def expected_device_plane_manager_reconciliation_descriptor():
}
def expected_device_plane_manager_v2_reconciliation_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane.device-manager-control-plane-"
"v2-reconciliation.v1"
),
"mode": "failed-v2-control-plane-baseline-adoption",
"failedPatchId": DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID,
"failedArtifactSha256": (
DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
),
"backupId": DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID,
"failureClass": "deterministic-runtime-module-resolution",
"missingModule": (
"/packages/external-provider-contract/src/"
"credential-reference.mjs"
),
"correctiveAction": (
"runtime-local-contract-adapter+staged-module-import-gate"
),
"sourceAction": "publish-reconciliation-marker-only",
"runtimeAction": "read-only-acceptance",
"preservedServices": [
"device-control-core",
"device-gateway",
"device-postgres",
"device-backhaul-target",
],
"absentService": "device-manager",
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
"publicIngress": "disabled",
"commandTransport": "disabled",
"gelios": "untouched",
"rollback": "marker-only-runtime-unchanged",
}
def validate_platform_device_core_hub_trust_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / PLATFORM_DEVICE_CORE_HUB_TRUST_REL,
@@ -8889,6 +9007,23 @@ def validate_device_plane_manager_reconciliation_payload(payload_dir):
return descriptor
def validate_device_plane_manager_v2_reconciliation_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL,
"Device Manager v2 control-plane reconciliation descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_manager_v2_reconciliation_descriptor()
):
die(
"Device Manager v2 control-plane reconciliation "
"descriptor mismatch"
)
return descriptor
def is_device_plane_postgres_bootstrap_slice(component, entries):
return (
component == "device-plane"
@@ -10369,6 +10504,185 @@ def validate_device_plane_manager_activation_predecessor(payload_dir):
}
def validate_device_plane_manager_v2_reconciliation_backup():
backup_dir = (
BACKUPS_DIR / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID
)
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Device Manager v2 failed-apply backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Device Manager v2 failed-apply backup is unsafe")
expected = DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_SHA256
if {child.name for child in backup_dir.iterdir()} != set(expected):
die("Device Manager v2 failed-apply backup file set mismatch")
for name, expected_sha256 in expected.items():
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected_sha256
):
die(
"Device Manager v2 failed-apply backup drift detected: "
f"{name}"
)
existing = tuple(read_backup_path_list(
backup_dir / "existing-files.txt"
))
missing = tuple(read_backup_path_list(
backup_dir / "missing-files.txt"
))
validate_backup_partition(
DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES,
existing,
missing,
"Device Manager v2 reconciliation",
)
if (
existing != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING
or missing != DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING
):
die("Device Manager v2 failed-apply backup partition mismatch")
return backup_dir
def validate_device_plane_manager_v2_reconciled_baseline(
backup_dir,
*,
marker_installed,
):
prior_backup = validate_device_plane_manager_reconciliation_backup()
runtime = validate_device_plane_manager_reconciled_baseline(
prior_backup,
marker_installed=True,
)
root = component_root("device-plane")
with tempfile.TemporaryDirectory(
prefix="device-manager-v2-reconciled-baseline-",
dir=TMP_DIR,
) as directory:
backup_root = Path(directory)
materialize_backup_tree(
backup_dir / "source-before.tgz",
backup_root,
set(DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING),
)
backup_source = collect_exact_files(
backup_root,
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING,
"Device Manager v2 pre-apply source",
)
live_source = collect_exact_files(
root,
DEVICE_PLANE_MANAGER_V2_RECONCILIATION_EXISTING,
"Device Manager v2 reconciled live source",
)
if live_source != backup_source:
die("Device Manager v2 rollback source does not match backup")
for rel in DEVICE_PLANE_MANAGER_V2_RECONCILIATION_MISSING:
path = root / rel
if path.exists() or path.is_symlink():
die(
"Device Manager v2 candidate-only source remains installed: "
f"{rel}"
)
marker = root / DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL
if marker_installed:
descriptor = read_strict_json(
marker,
"installed Device Manager v2 reconciliation descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_manager_v2_reconciliation_descriptor()
):
die("installed Device Manager v2 reconciliation descriptor mismatch")
elif marker.exists() or marker.is_symlink():
die("Device Manager v2 reconciliation descriptor already installed")
return runtime
def validate_device_plane_manager_v2_reconciliation_evidence(payload_dir):
descriptor = validate_device_plane_manager_v2_reconciliation_payload(
payload_dir
)
backup_dir = validate_device_plane_manager_v2_reconciliation_backup()
failed_artifact = FAILED_DIR / DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Manager v2 failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
):
die("Device Manager v2 failed artifact evidence mismatch")
records = [
value
for value in load_state(FAILED_STATE_FILE)
if value.get("id") == DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID
]
if len(records) != 1:
die("Device Manager v2 failed journal evidence count mismatch")
record = records[0]
if (
record.get("artifact") != DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT
or record.get("backup_id")
!= DEVICE_PLANE_MANAGER_V2_RECONCILIATION_BACKUP_ID
or record.get("component") != "device-plane"
or record.get("sha256")
!= DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status") != "failed:DeployError"
or record.get("status") != "failed"
or record.get("message")
!= (
"container healthcheck grace exhausted for "
"f8593303db6b71468716ed2428e54aab18d7303b34adbe1a93cd6df86effa1cf: "
"unhealthy"
)
):
die("Device Manager v2 failed journal evidence mismatch")
with tempfile.TemporaryDirectory(
prefix="device-manager-v2-failed-artifact-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, _failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id") != DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= DEVICE_PLANE_MANAGER_CONTROL_PLANE_ENTRIES
):
die("Device Manager v2 failed artifact contract mismatch")
runtime = validate_device_plane_manager_v2_reconciled_baseline(
backup_dir,
marker_installed=False,
)
return {
"mode": descriptor["mode"],
"backup": backup_dir,
"failedArtifact": failed_artifact,
"runtime": runtime,
}
def device_plane_service_container_ids(service):
if service not in (
*DEVICE_PLANE_RUNTIME_SERVICES,
@@ -13558,6 +13872,11 @@ def component_services(component, entries=None):
# evidence and the already-restored baseline. Runtime is read-only.
return ()
if is_device_plane_manager_v2_reconciliation_slice(component, entries):
# The failed v2 apply already restored the prior source and later
# converged to the healthy baseline. Publish evidence only.
return ()
if is_device_plane_postgres_bootstrap_slice(component, entries):
# This exact one-time transition is the only Device Plane artifact that
# may select durable state. Its preflight requires both container and
@@ -13976,6 +14295,9 @@ def component_builds(component, entries=None):
if is_device_plane_manager_reconciliation_slice(component, entries):
return ()
if is_device_plane_manager_v2_reconciliation_slice(component, entries):
return ()
if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
return ()
@@ -16000,6 +16322,7 @@ def plan_artifact(artifact):
device_plane_b2_recovery_preflight = None
device_plane_manager_activation_preflight = None
device_plane_manager_reconciliation_preflight = None
device_plane_manager_v2_reconciliation_preflight = None
device_plane_backhaul_preflight = None
device_plane_backhaul_vps_enrollment_preflight = None
device_plane_runtime_before = None
@@ -16227,6 +16550,15 @@ def plan_artifact(artifact):
payload_dir
)
)
if is_device_plane_manager_v2_reconciliation_slice(
manifest["component"],
entries,
):
device_plane_manager_v2_reconciliation_preflight = (
validate_device_plane_manager_v2_reconciliation_evidence(
payload_dir
)
)
if is_device_plane_backhaul_target_slice(
manifest["component"],
entries,
@@ -17841,6 +18173,42 @@ def plan_artifact(artifact):
print("device_gateway_tcp_9921=preserved:loopback-only")
print("device_manager_public_route=unchanged:absent")
print("device_plane_rollback=marker-only-runtime-unchanged")
if device_plane_manager_v2_reconciliation_preflight is not None:
print(
"device_plane_transition="
f"{device_plane_manager_v2_reconciliation_preflight['mode']}"
)
print(
"failed_patch="
f"{DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID}"
)
print(
"failed_artifact_sha256="
f"{DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256}"
)
print(
"recovery_backup="
f"{device_plane_manager_v2_reconciliation_preflight['backup'].name}"
)
print(
"device_plane_failure_class="
"deterministic-runtime-module-resolution"
)
print("device_plane_build=none")
print("device_plane_runtime_mutation=none")
print(
"device_plane_runtime_services="
"preserved:device-control-core,device-gateway,"
"device-postgres,device-backhaul-target"
)
print("device_manager=absent")
print(
"device_plane_source_action="
"publish-v2-reconciliation-marker-only"
)
print("device_gateway_tcp_9921=preserved:loopback-only")
print("device_manager_public_route=unchanged:absent")
print("device_plane_rollback=marker-only-runtime-unchanged")
if device_plane_backhaul_vps_enrollment_preflight is not None:
print(
"device_plane_transition="
@@ -19356,6 +19724,8 @@ def run_component_runtime(component, entries, services):
return
if is_device_plane_manager_reconciliation_slice(component, entries):
return
if is_device_plane_manager_v2_reconciliation_slice(component, entries):
return
if is_device_plane_foundation_recovery_slice(component, entries):
return
if is_device_plane_foundation_network_publication_slice(
@@ -19386,6 +19756,8 @@ def run_device_plane_runtime_for_apply(
"device-plane", entries
) or is_device_plane_manager_reconciliation_slice(
"device-plane", entries
) or is_device_plane_manager_v2_reconciliation_slice(
"device-plane", entries
):
return
# Build and runtime preparation are pre-runtime phases. A failure here
@@ -20132,6 +20504,23 @@ def run_healthchecks(component, entries=None, services=None):
)
return
if is_device_plane_manager_v2_reconciliation_slice(component, entries):
if tuple(services or ()) != ():
die("Device Manager v2 reconciliation service set mismatch")
for service in DEVICE_PLANE_RUNTIME_SERVICES:
healthcheck_compose_service_with_grace(
"device-plane",
service,
)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
backup_dir = validate_device_plane_manager_v2_reconciliation_backup()
validate_device_plane_manager_v2_reconciled_baseline(
backup_dir,
marker_installed=True,
)
return
if is_device_plane_backhaul_vps_enrollment_slice(component, entries):
if tuple(services or ()) != (DEVICE_PLANE_BACKHAUL_TARGET_SERVICE,):
die("Device Plane VPS enrollment service set mismatch")
@@ -21112,6 +21501,13 @@ def apply_artifact(artifact):
validate_device_plane_manager_reconciliation_evidence(
payload_dir
)
if is_device_plane_manager_v2_reconciliation_slice(
component,
entries,
):
validate_device_plane_manager_v2_reconciliation_evidence(
payload_dir
)
if is_device_plane_backhaul_target_slice(
component,
entries,
infra/deploy-runner/test_device_manager_control_plane_artifacts.py
+4
View File
@@ -156,6 +156,10 @@ class DeviceManagerControlPlaneArtifactsTest(unittest.TestCase):
"payload/services/device-manager/server/device-manager-server.mjs",
names,
)
self.assertIn(
"payload/services/device-control-core/src/credential-reference.mjs",
names,
)
self.assertFalse(any(name.endswith(".test.mjs") for name in names))
self.assertEqual(result["services"], ["device-control-core", "device-manager"])
self.assertNotIn("device-postgres", result["services"])
infra/deploy-runner/test_device_manager_control_plane_reconciliation_artifact.py
+96 -16
View File
@@ -18,6 +18,10 @@ BUILDER = (
SCRIPT_DIR
/ "build-device-manager-control-plane-reconciliation-artifact.mjs"
)
V2_BUILDER = (
SCRIPT_DIR
/ "build-device-manager-control-plane-v2-reconciliation-artifact.mjs"
)
def load_runner():
@@ -35,11 +39,11 @@ RUNNER = load_runner()
class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase):
def build(self, artifact_dir, patch_id):
def build(self, artifact_dir, patch_id, builder=BUILDER):
environment = os.environ.copy()
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
result = subprocess.run(
["node", str(BUILDER), patch_id],
["node", str(builder), patch_id],
check=True,
capture_output=True,
text=True,
@@ -145,6 +149,79 @@ class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase):
RUNNER.DEVICE_PLANE_MANAGER_FAILED_CONTROL_PLANE_ENTRIES,
)
def test_v2_reconciliation_is_marker_only_exact_and_deterministic(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-device-manager-v2-reconciliation-artifact-",
) as directory:
artifact_dir = Path(directory)
first = self.build(
artifact_dir,
"device-manager-control-plane-v2-reconciliation-unit-004",
V2_BUILDER,
)
first_bytes = Path(first["artifact"]).read_bytes()
second = self.build(
artifact_dir,
"device-manager-control-plane-v2-reconciliation-unit-004",
V2_BUILDER,
)
second_bytes = Path(second["artifact"]).read_bytes()
self.assertEqual(first_bytes, second_bytes)
self.assertEqual(
first["sha256"],
hashlib.sha256(first_bytes).hexdigest(),
)
self.assertEqual(
first["entries"],
list(RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES),
)
self.assertEqual(first["build"], [])
self.assertEqual(first["services"], [])
with tarfile.open(first["artifact"], "r:gz") as archive:
descriptor = json.loads(
archive.extractfile(
"payload/"
+ RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_REL
).read().decode("utf-8")
)
self.assertEqual(
descriptor,
RUNNER.expected_device_plane_manager_v2_reconciliation_descriptor(),
)
self.assertEqual(
RUNNER.component_services(
"device-plane",
RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES,
),
(),
)
self.assertEqual(
RUNNER.component_builds(
"device-plane",
RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES,
),
(),
)
def test_failed_v2_control_plane_is_terminal(self):
with self.assertRaisesRegex(
RUNNER.DeployError,
"exact v2 reconciliation successor",
):
RUNNER.reject_terminal_device_plane_manager_artifact(
{"id": RUNNER.DEVICE_PLANE_MANAGER_V2_FAILED_PATCH_ID},
"0" * 64,
)
with self.assertRaisesRegex(
RUNNER.DeployError,
"exact v2 reconciliation successor",
):
RUNNER.reject_terminal_device_plane_manager_artifact(
{"id": "different"},
RUNNER.DEVICE_PLANE_MANAGER_V2_FAILED_ARTIFACT_SHA256,
)
def test_health_grace_waits_through_unhealthy_and_exited(self):
results = [
mock.Mock(stdout="unhealthy\n", stderr="", returncode=0),
@@ -266,21 +343,24 @@ class DeviceManagerControlPlaneReconciliationArtifactTest(unittest.TestCase):
generic_health.assert_not_called()
def test_reconciliation_runtime_phase_is_read_only(self):
entries = RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES
with (
mock.patch.object(RUNNER, "run_build") as build,
mock.patch.object(RUNNER, "prepare_component_runtime") as prepare,
mock.patch.object(RUNNER, "run_compose") as compose,
for entries in (
RUNNER.DEVICE_PLANE_MANAGER_RECONCILIATION_ENTRIES,
RUNNER.DEVICE_PLANE_MANAGER_V2_RECONCILIATION_ENTRIES,
):
RUNNER.run_component_runtime("device-plane", entries, ())
RUNNER.run_device_plane_runtime_for_apply(
entries,
(),
mock.Mock(),
)
build.assert_not_called()
prepare.assert_not_called()
compose.assert_not_called()
with (
mock.patch.object(RUNNER, "run_build") as build,
mock.patch.object(RUNNER, "prepare_component_runtime") as prepare,
mock.patch.object(RUNNER, "run_compose") as compose,
):
RUNNER.run_component_runtime("device-plane", entries, ())
RUNNER.run_device_plane_runtime_for_apply(
entries,
(),
mock.Mock(),
)
build.assert_not_called()
prepare.assert_not_called()
compose.assert_not_called()
if __name__ == "__main__":