SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(device-plane): add fail-closed deploy foundation

Browse Source
This commit is contained in:
Codex
2026-07-25 21:29:05 +03:00
parent e9e03143cd
commit e217723784
36 changed files with 3729 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/deploy-runner/README.md
+55
View File
@@ -30,6 +30,7 @@ Supported components in this source:
- `bim-viewer`
- `n8n-private-extension`
- `module-foundry`
- `device-plane`
- `proxy-contur`
- `dc-amd-proxy`
@@ -429,3 +430,57 @@ Normal service deploys must still use explicit artifacts:
sudo /usr/local/sbin/nodedc-deploy plan /volume1/docker/nodedc-deploy/inbox/<artifact>.tgz
sudo /usr/local/sbin/nodedc-deploy apply /volume1/docker/nodedc-deploy/inbox/<artifact>.tgz
```
## Device Plane foundation
`device-plane` is an additive component rooted at:
```text
/volume1/docker/nodedc-device-plane
```
Its fixed Compose project is `nodedc-device-plane`. Ordinary application
artifacts may select only `device-control-core` and `device-gateway`, always
with `--no-deps`. `device-postgres` and the named
`nodedc-device-plane-postgres-data` volume are durable prerequisites and are
never selected or recreated by an application overlay.
The sole exception is the exact one-time bootstrap artifact containing only
the reviewed Compose file and
`deployment/device-postgres-bootstrap-v1.json`. Its preflight requires both the
Compose database container and named volume to be absent. It selects only
`device-postgres`; a failed activation may remove that candidate container but
never the volume. Any pre-existing container or volume is an ambiguity and
fails closed.
The runner creates or validates three root-owned secret files outside the
artifact: the PostgreSQL password, Gateway-to-Core token and restricted
identifier pepper. The manifest cannot choose their paths or values.
The foundation publishes only loopback health endpoints on `18120` and
`18121`. Raw device ingress `9921`, discovery ingest and outbound command
transport remain disabled. The first application artifact must not be built or
staged until this runner candidate is separately reviewed, promoted and proven
by a fresh `verify-install`.
Build and test the deterministic data-only artifact contract locally:
```bash
python3 -m unittest -v \
infra.deploy-runner.test_device_plane_registry \
infra.deploy-runner.test_device_plane_artifact
node infra/deploy-runner/build-device-plane-artifact.mjs \
device-plane-foundation-YYYYMMDD-NNN
```
Any failed first activation removes only candidate Core/Gateway containers,
never volumes, restores the source overlay and retains PostgreSQL state.
After the runner is promoted and freshly verified, bootstrap the durable
prerequisite with a separate artifact before planning the application:
```bash
node infra/deploy-runner/build-device-plane-postgres-bootstrap-artifact.mjs \
device-plane-postgres-bootstrap-YYYYMMDD-NNN
```
infra/deploy-runner/build-device-plane-artifact.mjs
+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import {
cp,
lstat,
mkdir,
mkdtemp,
readFile,
readdir,
rm,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../..");
const sourceRoot = resolve(platformRoot, "device-plane");
const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"),
);
const [patchId = "device-plane-foundation-20260725-001", ...extra] =
process.argv.slice(2);
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
throw new Error("usage: build-device-plane-artifact.mjs [patch-id]");
}
const files = [
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway",
];
const ignoredBasenames = new Set([
".DS_Store",
".git",
"node_modules",
]);
const ignoredDirectoryNames = new Set(["test"]);
const stage = await mkdtemp(join(tmpdir(), "nodedc-device-plane-artifact-"));
const payload = join(stage, "payload");
const target = join(artifactDir, `nodedc-device-plane-${patchId}.tgz`);
await assertSourceBoundary();
try {
await mkdir(payload, { recursive: true });
for (const sourceRelative of files) {
await copySafe(
resolve(sourceRoot, sourceRelative),
join(payload, sourceRelative),
);
}
await writeFile(
join(stage, "manifest.env"),
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
"utf8",
);
await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8");
await mkdir(artifactDir, { recursive: true });
const tar = spawnSync(
"python3",
["-c", canonicalTarScript(), target, stage],
{
encoding: "utf8",
maxBuffer: 128 * 1024 * 1024,
},
);
if (tar.status !== 0) {
throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
}
const digest = createHash("sha256")
.update(await readFile(target))
.digest("hex");
console.log(JSON.stringify({
ok: true,
patchId,
artifact: target,
sha256: digest,
component: "device-plane",
entries: files,
services: ["device-control-core", "device-gateway"],
preserved: [
"device-postgres",
"nodedc-device-plane-postgres-data",
"Gelios",
],
excluded: [
".env*",
"node_modules",
"**/test",
"docs",
"runtime",
"secrets",
],
}, null, 2));
} finally {
await rm(stage, { recursive: true, force: true });
}
async function assertSourceBoundary() {
const compose = await readFile(
resolve(sourceRoot, "docker-compose.device-plane.yml"),
"utf8",
);
for (const fragment of [
'DEVICE_DISCOVERY_INGEST_ENABLED: "false"',
'DEVICE_GATEWAY_LISTEN_ENABLED: "false"',
'"127.0.0.1:18120:18120"',
'"127.0.0.1:18121:18121"',
"source: /volume1/docker/nodedc-device-plane/secrets/postgres-password",
"create_host_path: false",
"name: nodedc-device-plane-postgres-data",
"pull_policy: never",
]) {
if (!compose.includes(fragment)) {
throw new Error(`device_plane_compose_boundary_missing:${fragment}`);
}
}
for (const forbidden of [
"9921:9921",
"0.0.0.0:9921",
"DEVICE_DISCOVERY_INGEST_ENABLED: \"true\"",
"DEVICE_GATEWAY_LISTEN_ENABLED: \"true\"",
"POSTGRES_PASSWORD:",
]) {
if (compose.includes(forbidden)) {
throw new Error(`device_plane_compose_boundary_violation:${forbidden}`);
}
}
}
function canonicalTarScript() {
return [
"import gzip,io,pathlib,sys,tarfile",
"root=pathlib.Path(sys.argv[2])",
"with open(sys.argv[1],'wb') as out:",
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
" for top in ('manifest.env','files.txt','payload'):",
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
" for x in paths:",
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
].join("\n");
}
async function copySafe(source, destination) {
const sourceStat = await lstat(source);
if (sourceStat.isSymbolicLink()) {
throw new Error(
`source_symlink_rejected:${relative(sourceRoot, source)}`,
);
}
if (sourceStat.isFile()) {
await mkdir(dirname(destination), { recursive: true });
await cp(source, destination, { force: true, verbatimSymlinks: true });
return;
}
if (!sourceStat.isDirectory()) {
throw new Error(`source_type_rejected:${source}`);
}
await mkdir(destination, { recursive: true });
for (const entry of await readdir(source, { withFileTypes: true })) {
if (
ignoredBasenames.has(entry.name)
|| entry.name.startsWith(".env")
|| (entry.isDirectory() && ignoredDirectoryNames.has(entry.name))
) {
continue;
}
const childSource = join(source, entry.name);
const childDestination = join(destination, entry.name);
if (entry.isSymbolicLink()) {
throw new Error(
`source_symlink_rejected:${relative(sourceRoot, childSource)}`,
);
}
await copySafe(childSource, childDestination);
}
}
infra/deploy-runner/build-device-plane-postgres-bootstrap-artifact.mjs
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import {
cp,
lstat,
mkdir,
mkdtemp,
readFile,
rm,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../..");
const sourceRoot = resolve(platformRoot, "device-plane");
const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"),
);
const [patchId = "device-plane-postgres-bootstrap-20260725-001", ...extra] =
process.argv.slice(2);
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
throw new Error(
"usage: build-device-plane-postgres-bootstrap-artifact.mjs [patch-id]",
);
}
const files = [
"docker-compose.device-plane.yml",
"deployment/device-postgres-bootstrap-v1.json",
];
const stage = await mkdtemp(
join(tmpdir(), "nodedc-device-plane-postgres-bootstrap-"),
);
const payload = join(stage, "payload");
const target = join(
artifactDir,
`nodedc-device-plane-${patchId}.tgz`,
);
await assertSourceBoundary();
try {
await mkdir(payload, { recursive: true });
for (const relativePath of files) {
const source = resolve(sourceRoot, relativePath);
const sourceStat = await lstat(source);
if (!sourceStat.isFile() || sourceStat.isSymbolicLink()) {
throw new Error(`bootstrap_source_file_required:${relativePath}`);
}
const destination = join(payload, relativePath);
await mkdir(dirname(destination), { recursive: true });
await cp(source, destination, {
force: true,
verbatimSymlinks: false,
});
}
await writeFile(
join(stage, "manifest.env"),
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
"utf8",
);
await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8");
await mkdir(artifactDir, { recursive: true });
const tar = spawnSync(
"python3",
["-c", canonicalTarScript(), target, stage],
{
encoding: "utf8",
maxBuffer: 128 * 1024 * 1024,
},
);
if (tar.status !== 0) {
throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
}
const sha256 = createHash("sha256")
.update(await readFile(target))
.digest("hex");
console.log(JSON.stringify({
ok: true,
patchId,
artifact: target,
sha256,
component: "device-plane",
entries: files,
services: ["device-postgres"],
mode: "create-if-absent",
rollbackVolumePolicy: "preserve",
}, null, 2));
} finally {
await rm(stage, { recursive: true, force: true });
}
async function assertSourceBoundary() {
const descriptor = JSON.parse(
await readFile(
resolve(
sourceRoot,
"deployment/device-postgres-bootstrap-v1.json",
),
"utf8",
),
);
const expected = {
schemaVersion: "nodedc.device-plane.postgres-bootstrap.v1",
service: "device-postgres",
volume: "nodedc-device-plane-postgres-data",
mode: "create-if-absent",
ordinaryApplicationSelection: "forbidden",
rollbackVolumePolicy: "preserve",
};
if (JSON.stringify(descriptor) !== JSON.stringify(expected)) {
throw new Error("device_plane_postgres_bootstrap_descriptor_mismatch");
}
const compose = await readFile(
resolve(sourceRoot, "docker-compose.device-plane.yml"),
"utf8",
);
for (const required of [
"device-postgres:",
"name: nodedc-device-plane-postgres-data",
"POSTGRES_PASSWORD_FILE: /run/nodedc-secrets/postgres-password",
"create_host_path: false",
]) {
if (!compose.includes(required)) {
throw new Error(`device_plane_postgres_boundary_missing:${required}`);
}
}
const postgresStart = compose.indexOf(" device-postgres:");
const postgresEnd = compose.indexOf("\n device-control-core:");
if (
postgresStart < 0
|| postgresEnd <= postgresStart
|| compose.slice(postgresStart, postgresEnd).includes("\n ports:")
) {
throw new Error("device_plane_postgres_host_port_forbidden");
}
}
function canonicalTarScript() {
return [
"import gzip,io,pathlib,sys,tarfile",
"root=pathlib.Path(sys.argv[2])",
"with open(sys.argv[1],'wb') as out:",
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
" for top in ('manifest.env','files.txt','payload'):",
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
" for x in paths:",
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
].join("\n");
}
infra/deploy-runner/nodedc-deploy
+408
View File
@@ -37,6 +37,21 @@ MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret"
MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token"
PROXY_CONTUR_ENV_FILE = Path("/volume1/docker/proxy-contur/.env")
DC_AMD_PROXY_RUNTIME_DIR = Path("/volume1/docker/dc-amd-proxy/runtime")
DEVICE_PLANE_ROOT = Path("/volume1/docker/nodedc-device-plane")
DEVICE_PLANE_SECRET_DIR = DEVICE_PLANE_ROOT / "secrets"
DEVICE_PLANE_POSTGRES_PASSWORD_FILE = DEVICE_PLANE_SECRET_DIR / "postgres-password"
DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE = DEVICE_PLANE_SECRET_DIR / "gateway-core-token"
DEVICE_PLANE_IDENTIFIER_PEPPER_FILE = DEVICE_PLANE_SECRET_DIR / "identifier-pepper"
DEVICE_PLANE_CONTROL_CORE_IMAGE = "nodedc/device-control-core:local"
DEVICE_PLANE_GATEWAY_IMAGE = "nodedc/device-gateway:local"
DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL = (
"deployment/device-postgres-bootstrap-v1.json"
)
DEVICE_PLANE_POSTGRES_BOOTSTRAP_ENTRIES = (
"docker-compose.device-plane.yml",
DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
)
DEVICE_PLANE_POSTGRES_VOLUME = "nodedc-device-plane-postgres-data"
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-provisioner"
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "token"
ENGINE_CREDENTIAL_PROVISIONER_PRIVATE_KEY_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "engine-credential-provisioner-ed25519.pem"
@@ -1140,6 +1155,19 @@ COMPONENTS = {
"http://172.22.0.222:9920/healthz",
),
},
"device-plane": {
"payload_root": DEVICE_PLANE_ROOT,
"compose_root": DEVICE_PLANE_ROOT,
"compose_project": "nodedc-device-plane",
"compose_files": (
DEVICE_PLANE_ROOT / "docker-compose.device-plane.yml",
),
"bootstrap_root": True,
"compose_no_deps": True,
# PostgreSQL is durable state infrastructure. Normal application
# overlays can rebuild/recreate only these two stateless services.
"services": ("device-control-core", "device-gateway"),
},
"proxy-contur": {
"payload_root": Path("/volume1/docker/proxy-contur"),
"compose_root": Path("/volume1/docker/proxy-contur"),
@@ -2297,6 +2325,12 @@ def denied_payload_path(component, rel):
"infra/docker-compose.module-foundry.yml",
):
pass
elif component == "device-plane" and rel in (
"docker-compose.device-plane.yml",
"services/device-control-core/Dockerfile",
"services/device-gateway/Dockerfile",
):
pass
elif component == "proxy-contur" and rel in (
"Dockerfile",
"docker-compose.yml",
@@ -2426,6 +2460,25 @@ def denied_payload_path(component, rel):
return "module-foundry env file"
if rel.startswith(("Dockerfile.bak", "infra/docker-compose.module-foundry.yml.bak")):
return "module-foundry backup file"
elif component == "device-plane":
runtime_prefixes = (
"runtime",
"secrets",
"data",
"logs",
"backups",
"node_modules",
)
if rel == ".env" or rel.startswith(".env."):
return "device-plane env file"
if "test" in parts:
return "device-plane test path"
if rel.startswith((
"docker-compose.device-plane.yml.bak",
"services/device-control-core/Dockerfile.bak",
"services/device-gateway/Dockerfile.bak",
)):
return "device-plane backup file"
elif component == "n8n-private-extension":
# An extension release is inert data at this boundary. Activation is
# Engine-owned and must never be smuggled into the artifact as a script,
@@ -2676,6 +2729,27 @@ def allowed_payload_path(component, rel):
)):
return True
if component == "device-plane":
if rel in (
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway",
):
return True
if rel.startswith((
"packages/device-protocol-contract/",
"packages/arusnavi-b2-adapter/",
"services/device-control-core/",
"services/device-gateway/",
)):
return True
if component == "n8n-private-extension":
parts = PurePosixPath(rel).parts
if (
@@ -7681,6 +7755,11 @@ def load_artifact(artifact, work_dir):
die(f"files.txt entry missing in payload: {rel}")
validate_payload_tree(manifest["component"], payload_dir, entries)
if is_device_plane_postgres_bootstrap_slice(
manifest["component"],
entries,
):
validate_device_plane_postgres_bootstrap_payload(payload_dir)
if manifest["component"] == "n8n-private-extension":
validate_n8n_private_extension_release(payload_dir, entries)
if manifest["component"] == "engine":
@@ -7972,6 +8051,79 @@ def component_root(component):
return COMPONENTS[component]["payload_root"]
def is_device_plane_postgres_bootstrap_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries) == DEVICE_PLANE_POSTGRES_BOOTSTRAP_ENTRIES
)
def validate_device_plane_postgres_bootstrap_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
"Device Plane PostgreSQL bootstrap descriptor",
max_bytes=8 * 1024,
)
expected = {
"schemaVersion": "nodedc.device-plane.postgres-bootstrap.v1",
"service": "device-postgres",
"volume": DEVICE_PLANE_POSTGRES_VOLUME,
"mode": "create-if-absent",
"ordinaryApplicationSelection": "forbidden",
"rollbackVolumePolicy": "preserve",
}
if descriptor != expected:
die("Device Plane PostgreSQL bootstrap descriptor mismatch")
return descriptor
def preflight_device_plane_postgres_bootstrap():
container_result = subprocess.run(
[
str(DOCKER),
"container",
"ls",
"-a",
"--filter",
"label=com.docker.compose.project=nodedc-device-plane",
"--filter",
"label=com.docker.compose.service=device-postgres",
"--format",
"{{.ID}}",
],
check=False,
capture_output=True,
text=True,
)
if container_result.returncode != 0:
die("Device Plane PostgreSQL container preflight failed")
container_ids = [
line.strip()
for line in container_result.stdout.splitlines()
if line.strip()
]
if container_ids:
die("Device Plane PostgreSQL container already exists")
volume_result = subprocess.run(
[
str(DOCKER),
"volume",
"inspect",
DEVICE_PLANE_POSTGRES_VOLUME,
],
check=False,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
if volume_result.returncode == 0:
die("Device Plane PostgreSQL volume already exists")
if volume_result.returncode != 1:
die("Device Plane PostgreSQL volume preflight failed")
return "absent"
def component_compose_root(component):
return COMPONENTS[component].get("compose_root", component_root(component))
@@ -9491,6 +9643,13 @@ def is_platform_provider_catalog_only(entries):
def component_services(component, entries=None):
if is_device_plane_postgres_bootstrap_slice(component, entries):
# This exact one-time transition is the only Device Plane artifact that
# may select durable state. Its preflight requires both container and
# named volume to be absent, so --force-recreate cannot touch an
# installed database.
return ("device-postgres",)
if is_engine_l2_closed_loop_slice(component, entries):
# The failed 030 apply published both the backend source and the built
# UI before Compose rejected the descriptor/source mismatch. The exact
@@ -9549,6 +9708,45 @@ def component_services(component, entries=None):
# authorization normalization inside the already-active backend.
return ("nodedc-backend",)
if component == "device-plane" and entries is not None:
selected = []
def add(*services):
for service in services:
if service not in selected:
selected.append(service)
touches_common = any(
rel in (
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
)
or rel.startswith((
"packages/device-protocol-contract/",
"packages/arusnavi-b2-adapter/",
))
for rel in entries
)
touches_core = any(
rel == "services/device-control-core"
or rel.startswith("services/device-control-core/")
for rel in entries
)
touches_gateway = any(
rel == "services/device-gateway"
or rel.startswith("services/device-gateway/")
for rel in entries
)
if touches_common or touches_core:
add("device-control-core")
if touches_common or touches_gateway:
add("device-gateway")
return tuple(selected)
if component == "dc-cms" and entries is not None:
selected = []
@@ -9783,6 +9981,42 @@ def component_build_args(component, entries=None):
def component_builds(component, entries=None):
if is_device_plane_postgres_bootstrap_slice(component, entries):
return ()
if component == "device-plane" and entries is not None:
selected_services = component_services(component, entries)
builds = []
if "device-control-core" in selected_services:
builds.append((
DEVICE_PLANE_ROOT,
(
"build",
"--no-cache",
"--network=host",
"-f",
"services/device-control-core/Dockerfile",
"-t",
DEVICE_PLANE_CONTROL_CORE_IMAGE,
".",
),
))
if "device-gateway" in selected_services:
builds.append((
DEVICE_PLANE_ROOT,
(
"build",
"--no-cache",
"--network=host",
"-f",
"services/device-gateway/Dockerfile",
"-t",
DEVICE_PLANE_GATEWAY_IMAGE,
".",
),
))
return tuple(builds)
if component == "platform" and entries is not None:
touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries)
touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries)
@@ -11739,6 +11973,7 @@ def plan_artifact(artifact):
mcp_registered_execution_profiles_preflight = None
mcp_gelios_units_items_preflight = None
provider_catalog_preflight = None
device_plane_postgres_preflight = None
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
reject_terminal_engine_l2_failed_artifact(manifest, sha)
@@ -11875,6 +12110,13 @@ def plan_artifact(artifact):
)
if is_engine_provider_security_catalog_slice(manifest["component"], entries):
provider_catalog_preflight = preflight_engine_provider_security_catalog_predecessor()
if is_device_plane_postgres_bootstrap_slice(
manifest["component"],
entries,
):
device_plane_postgres_preflight = (
preflight_device_plane_postgres_bootstrap()
)
component = manifest["component"]
root = component_root(component)
@@ -13246,6 +13488,22 @@ def plan_artifact(artifact):
print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}")
print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
if component == "device-plane":
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_POSTGRES_PASSWORD_FILE}")
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE}")
print(f"runtime_secret=runner-managed:{DEVICE_PLANE_IDENTIFIER_PEPPER_FILE}")
print("device_postgres=preserved-prerequisite:not-selected")
print("device_postgres_volume=preserved:nodedc-device-plane-postgres-data")
print("device_gateway_public_ingress=disabled")
print("device_gateway_command_transport=disabled")
print("gelios=untouched")
if device_plane_postgres_preflight is not None:
print(
"device_postgres_bootstrap="
f"{device_plane_postgres_preflight}"
)
print("device_postgres_bootstrap_mode=create-if-absent")
print("device_postgres_rollback_volume=preserve")
if touches_external_data_plane:
print(f"runtime_secret=runner-managed:{EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
print(f"runtime_grants=runner-managed:{EXTERNAL_DATA_PLANE_READER_GRANTS_DIR}")
@@ -13527,6 +13785,71 @@ def rollback_platform_apply(root, backup_dir, entries, current_stamp, runtime_st
return f"source+runtime-restored:{restored_count}"
def rollback_device_plane_apply(
root,
backup_dir,
entries,
current_stamp,
runtime_started,
applied_services,
):
existing = read_backup_path_list(backup_dir / "existing-files.txt")
missing = read_backup_path_list(backup_dir / "missing-files.txt")
existing_set, _missing_set = validate_backup_partition(
entries,
existing,
missing,
"Device Plane runtime rollback",
)
baseline_entries = [rel for rel in entries if rel in existing_set]
compose_was_installed = "docker-compose.device-plane.yml" in existing_set
baseline_services = (
component_services("device-plane", baseline_entries)
if compose_was_installed
else ()
)
candidate_only_services = tuple(
service
for service in applied_services
if service not in baseline_services
)
candidate_cleanup_failed = False
if runtime_started and candidate_only_services:
# Remove only candidate services while the candidate Compose file is
# still present. PostgreSQL can appear only in the exact one-time
# bootstrap slice; volume flags are deliberately never used.
try:
stop_and_remove_compose_services(
"device-plane",
candidate_only_services,
)
except Exception:
candidate_cleanup_failed = True
restored_count = restore_platform_overlay(
root,
backup_dir,
entries,
current_stamp,
)
if candidate_cleanup_failed:
die("Device Plane candidate-only runtime cleanup failed after source restore")
if not runtime_started or not baseline_services:
return f"source-restored-runtime-unchanged:{restored_count}"
run_component_runtime(
"device-plane",
baseline_entries,
baseline_services,
)
run_healthchecks(
"device-plane",
baseline_entries,
baseline_services,
)
return f"source+runtime-restored:{restored_count}"
def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_started, applied_services):
existing = read_backup_path_list(backup_dir / "existing-files.txt")
missing = read_backup_path_list(backup_dir / "missing-files.txt")
@@ -14130,6 +14453,24 @@ def prepare_component_runtime(component, entries=None):
)
return
if component == "device-plane":
ensure_platform_runtime_secret(
DEVICE_PLANE_POSTGRES_PASSWORD_FILE,
MAP_GATEWAY_SECRET_RE,
"device plane PostgreSQL",
)
ensure_platform_runtime_secret(
DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE,
MAP_GATEWAY_SECRET_RE,
"device plane Gateway to Core",
)
ensure_platform_runtime_secret(
DEVICE_PLANE_IDENTIFIER_PEPPER_FILE,
MAP_GATEWAY_SECRET_RE,
"device plane identifier pepper",
)
return
if component == "proxy-contur":
sync_map_egress_proxy_secret()
return
@@ -14309,6 +14650,8 @@ def module_foundry_healthcheck():
def component_healthchecks(component, entries=None, services=None):
if is_device_plane_postgres_bootstrap_slice(component, entries):
return ()
if is_engine_n8n_transition(component, entries):
# The transition does not restart the Engine UI/backend generation.
# Its own acceptance below verifies n8n readiness, image, mount, logs,
@@ -14323,6 +14666,36 @@ def component_healthchecks(component, entries=None, services=None):
return ()
if component == "module-foundry":
return (module_foundry_healthcheck(),)
if component == "device-plane":
selected_services = (
tuple(services)
if services is not None
else component_services(component, entries)
)
checks = []
if "device-control-core" in selected_services:
checks.append({
"url": "http://127.0.0.1:18120/healthz",
"expected_json": {
"ok": True,
"service": "nodedc-device-control-core",
"database": "ready",
"discoveryIngest": "disabled",
"commandTransport": "disabled",
},
})
if "device-gateway" in selected_services:
checks.append({
"url": "http://127.0.0.1:18121/healthz",
"expected_json": {
"ok": True,
"service": "nodedc-device-gateway",
"tcpListener": "disabled",
"publicIngress": "disabled",
"commandTransport": "disabled",
},
})
return tuple(checks)
if (
(
is_engine_data_product_publish_grant_slice(component, entries)
@@ -14712,6 +15085,11 @@ process.stdout.write('engine-l2-closed-loop:0.7.0:cas+safe-profile+external-plan
def run_healthchecks(component, entries=None, services=None):
if is_device_plane_postgres_bootstrap_slice(component, entries):
if tuple(services or ()) != ("device-postgres",):
die("Device Plane PostgreSQL bootstrap service set mismatch")
healthcheck_compose_service("device-plane", "device-postgres")
return
if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries):
return
if is_engine_l2_closed_loop_slice(component, entries):
@@ -15504,6 +15882,11 @@ def apply_artifact(artifact):
die(f"artifact sha already applied: {sha}")
if state_has_patch_id(patch_id):
die(f"patch id already applied: {patch_id}")
if is_device_plane_postgres_bootstrap_slice(
component,
entries,
):
preflight_device_plane_postgres_bootstrap()
if not root.is_dir():
if bootstrap_root:
root.mkdir(parents=True, exist_ok=True)
@@ -16064,6 +16447,31 @@ def apply_artifact(artifact):
except Exception as rollback_exc:
rollback_status = f"failed:{type(rollback_exc).__name__}"
print("platform-automatic-rollback=failed", file=sys.stderr)
elif (
component == "device-plane"
and entries is not None
and services is not None
):
try:
restored_state = rollback_device_plane_apply(
root,
backup_dir,
entries,
current_stamp,
runtime_started,
services,
)
rollback_status = f"ok:device-plane-overlay:{restored_state}"
print(
f"device-plane-automatic-rollback={rollback_status}",
file=sys.stderr,
)
except Exception as rollback_exc:
rollback_status = f"failed:{type(rollback_exc).__name__}"
print(
"device-plane-automatic-rollback=failed",
file=sys.stderr,
)
failed_path = None
if artifact.exists() and rollback_status != "deferred:reconciliation-required":
try:
infra/deploy-runner/test_device_plane_artifact.py
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
import hashlib
import importlib.machinery
import importlib.util
import json
import os
import subprocess
import tarfile
import tempfile
import unittest
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
BUILDER = SCRIPT_DIR / "build-device-plane-artifact.mjs"
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
EXPECTED_ENTRIES = [
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway",
]
def load_runner():
loader = importlib.machinery.SourceFileLoader(
"nodedc_device_plane_artifact_runner_under_test",
str(RUNNER_PATH),
)
spec = importlib.util.spec_from_loader(loader.name, loader)
module = importlib.util.module_from_spec(spec)
loader.exec_module(module)
return module
RUNNER = load_runner()
class DevicePlaneArtifactTest(unittest.TestCase):
def build(self, artifact_dir, patch_id):
environment = os.environ.copy()
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
result = subprocess.run(
["node", str(BUILDER), patch_id],
check=True,
capture_output=True,
text=True,
env=environment,
)
return json.loads(result.stdout)
def test_artifact_is_narrow_safe_and_deterministic(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-artifact-",
) as directory:
artifact_dir = Path(directory)
first = self.build(
artifact_dir,
"device-plane-foundation-unit-001",
)
artifact = Path(first["artifact"])
first_bytes = artifact.read_bytes()
second = self.build(
artifact_dir,
"device-plane-foundation-unit-001",
)
second_bytes = Path(second["artifact"]).read_bytes()
self.assertEqual(first["component"], "device-plane")
self.assertEqual(first["entries"], EXPECTED_ENTRIES)
self.assertEqual(
first["services"],
["device-control-core", "device-gateway"],
)
self.assertEqual(
first["sha256"],
hashlib.sha256(first_bytes).hexdigest(),
)
self.assertEqual(first["sha256"], second["sha256"])
self.assertEqual(first_bytes, second_bytes)
with tarfile.open(artifact, "r:gz") as archive:
members = archive.getmembers()
names = {member.name for member in members}
files = (
archive.extractfile("files.txt")
.read()
.decode("utf-8")
.splitlines()
)
manifest = (
archive.extractfile("manifest.env")
.read()
.decode("utf-8")
)
compose = (
archive.extractfile(
"payload/docker-compose.device-plane.yml",
)
.read()
.decode("utf-8")
)
regular_payloads = [
archive.extractfile(member).read()
for member in members
if member.isfile()
]
self.assertEqual(files, EXPECTED_ENTRIES)
self.assertEqual(
manifest,
"id=device-plane-foundation-unit-001\n"
"component=device-plane\n"
"type=app-overlay\n",
)
self.assertIn(
"payload/services/device-control-core/Dockerfile",
names,
)
self.assertIn(
"payload/services/device-gateway/Dockerfile",
names,
)
self.assertFalse(any(
"/test/" in name
or "/node_modules/" in name
or Path(name).name.startswith(".env")
or name.startswith("payload/docs/")
or name.startswith("payload/runtime/")
or name.startswith("payload/secrets/")
for name in names
))
self.assertNotIn("9921:9921", compose)
self.assertIn('DEVICE_GATEWAY_LISTEN_ENABLED: "false"', compose)
self.assertIn('DEVICE_DISCOVERY_INGEST_ENABLED: "false"', compose)
self.assertNotIn(b"-----BEGIN PRIVATE KEY-----", b"\n".join(
regular_payloads,
))
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-runner-load-",
) as work_directory:
manifest_loaded, entries_loaded, payload_loaded = (
RUNNER.load_artifact(artifact, Path(work_directory))
)
self.assertEqual(manifest_loaded["component"], "device-plane")
self.assertEqual(entries_loaded, EXPECTED_ENTRIES)
self.assertEqual(payload_loaded.name, "payload")
if __name__ == "__main__":
unittest.main(verbosity=2)
infra/deploy-runner/test_device_plane_postgres_bootstrap.py
+195
View File
@@ -0,0 +1,195 @@
#!/usr/bin/env python3
import hashlib
import importlib.machinery
import importlib.util
import json
import os
import subprocess
import tarfile
import tempfile
import unittest
from pathlib import Path
from unittest import mock
SCRIPT_DIR = Path(__file__).resolve().parent
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
BUILDER = (
SCRIPT_DIR
/ "build-device-plane-postgres-bootstrap-artifact.mjs"
)
EXPECTED_ENTRIES = [
"docker-compose.device-plane.yml",
"deployment/device-postgres-bootstrap-v1.json",
]
def load_runner():
loader = importlib.machinery.SourceFileLoader(
"nodedc_device_plane_postgres_runner_under_test",
str(RUNNER_PATH),
)
spec = importlib.util.spec_from_loader(loader.name, loader)
module = importlib.util.module_from_spec(spec)
loader.exec_module(module)
return module
RUNNER = load_runner()
class DevicePlanePostgresBootstrapTest(unittest.TestCase):
def build(self, artifact_dir, patch_id):
environment = os.environ.copy()
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
result = subprocess.run(
["node", str(BUILDER), patch_id],
check=True,
capture_output=True,
text=True,
env=environment,
)
return json.loads(result.stdout)
def test_bootstrap_artifact_is_exact_deterministic_and_runner_accepted(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-postgres-artifact-",
) as directory:
artifact_dir = Path(directory)
first = self.build(
artifact_dir,
"device-plane-postgres-bootstrap-unit-001",
)
artifact = Path(first["artifact"])
first_bytes = artifact.read_bytes()
second = self.build(
artifact_dir,
"device-plane-postgres-bootstrap-unit-001",
)
second_bytes = Path(second["artifact"]).read_bytes()
self.assertEqual(first["entries"], EXPECTED_ENTRIES)
self.assertEqual(first["services"], ["device-postgres"])
self.assertEqual(first["mode"], "create-if-absent")
self.assertEqual(first["rollbackVolumePolicy"], "preserve")
self.assertEqual(
first["sha256"],
hashlib.sha256(first_bytes).hexdigest(),
)
self.assertEqual(first_bytes, second_bytes)
with tarfile.open(artifact, "r:gz") as archive:
names = {member.name for member in archive.getmembers()}
self.assertEqual(
archive.extractfile("files.txt")
.read()
.decode("utf-8")
.splitlines(),
EXPECTED_ENTRIES,
)
self.assertEqual(
names,
{
"manifest.env",
"files.txt",
"payload",
"payload/docker-compose.device-plane.yml",
"payload/deployment",
"payload/deployment/device-postgres-bootstrap-v1.json",
},
)
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-postgres-load-",
) as work_directory:
manifest, entries, _payload = RUNNER.load_artifact(
artifact,
Path(work_directory),
)
self.assertEqual(manifest["component"], "device-plane")
self.assertEqual(entries, EXPECTED_ENTRIES)
self.assertTrue(
RUNNER.is_device_plane_postgres_bootstrap_slice(
manifest["component"],
entries,
),
)
self.assertEqual(
RUNNER.component_services("device-plane", entries),
("device-postgres",),
)
self.assertEqual(
RUNNER.component_builds("device-plane", entries),
(),
)
def test_preflight_accepts_only_absent_container_and_volume(self):
absent_container = mock.Mock(returncode=0, stdout="")
absent_volume = mock.Mock(returncode=1)
with mock.patch.object(
RUNNER.subprocess,
"run",
side_effect=[absent_container, absent_volume],
):
self.assertEqual(
RUNNER.preflight_device_plane_postgres_bootstrap(),
"absent",
)
existing_container = mock.Mock(
returncode=0,
stdout="abc123def456\n",
)
with mock.patch.object(
RUNNER.subprocess,
"run",
return_value=existing_container,
):
with self.assertRaisesRegex(
RUNNER.DeployError,
"container already exists",
):
RUNNER.preflight_device_plane_postgres_bootstrap()
with mock.patch.object(
RUNNER.subprocess,
"run",
side_effect=[
absent_container,
mock.Mock(returncode=0),
],
):
with self.assertRaisesRegex(
RUNNER.DeployError,
"volume already exists",
):
RUNNER.preflight_device_plane_postgres_bootstrap()
def test_bootstrap_health_acceptance_is_database_service_only(self):
with mock.patch.object(
RUNNER,
"healthcheck_compose_service",
) as health:
RUNNER.run_healthchecks(
"device-plane",
EXPECTED_ENTRIES,
("device-postgres",),
)
health.assert_called_once_with(
"device-plane",
"device-postgres",
)
with self.assertRaisesRegex(
RUNNER.DeployError,
"service set mismatch",
):
RUNNER.run_healthchecks(
"device-plane",
EXPECTED_ENTRIES,
("device-control-core",),
)
if __name__ == "__main__":
unittest.main(verbosity=2)
infra/deploy-runner/test_device_plane_registry.py
+239
View File
@@ -0,0 +1,239 @@
#!/usr/bin/env python3
import importlib.machinery
import importlib.util
import tempfile
import unittest
from pathlib import Path
from unittest import mock
SCRIPT_DIR = Path(__file__).resolve().parent
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
def load_runner():
loader = importlib.machinery.SourceFileLoader(
"nodedc_device_plane_deploy_under_test",
str(RUNNER_PATH),
)
spec = importlib.util.spec_from_loader(loader.name, loader)
module = importlib.util.module_from_spec(spec)
loader.exec_module(module)
return module
RUNNER = load_runner()
class DevicePlaneRegistryTest(unittest.TestCase):
def test_registry_has_exact_roots_project_and_stateless_services(self):
component = RUNNER.COMPONENTS["device-plane"]
root = Path("/volume1/docker/nodedc-device-plane")
self.assertEqual(component["payload_root"], root)
self.assertEqual(component["compose_root"], root)
self.assertEqual(component["compose_project"], "nodedc-device-plane")
self.assertEqual(
component["compose_files"],
(root / "docker-compose.device-plane.yml",),
)
self.assertTrue(component["bootstrap_root"])
self.assertTrue(component["compose_no_deps"])
self.assertEqual(
component["services"],
("device-control-core", "device-gateway"),
)
self.assertNotIn("device-postgres", component["services"])
self.assertIsNone(component.get("compose_env_file"))
def test_files_select_only_the_affected_stateless_services(self):
self.assertEqual(
RUNNER.component_services(
"device-plane",
("services/device-control-core/src/server.mjs",),
),
("device-control-core",),
)
self.assertEqual(
RUNNER.component_services(
"device-plane",
("services/device-gateway/src/server.mjs",),
),
("device-gateway",),
)
self.assertEqual(
RUNNER.component_services(
"device-plane",
("packages/arusnavi-b2-adapter",),
),
("device-control-core", "device-gateway"),
)
services = RUNNER.component_services(
"device-plane",
("docker-compose.device-plane.yml",),
)
self.assertEqual(
services,
("device-control-core", "device-gateway"),
)
self.assertNotIn("device-postgres", services)
def test_build_selection_is_exact_and_database_free(self):
builds = RUNNER.component_builds(
"device-plane",
("services/device-gateway",),
)
self.assertEqual(len(builds), 1)
self.assertEqual(builds[0][0], RUNNER.DEVICE_PLANE_ROOT)
self.assertIn(
"services/device-gateway/Dockerfile",
builds[0][1],
)
self.assertIn(RUNNER.DEVICE_PLANE_GATEWAY_IMAGE, builds[0][1])
all_builds = RUNNER.component_builds(
"device-plane",
("package-lock.json",),
)
self.assertEqual(len(all_builds), 2)
self.assertNotIn("device-postgres", " ".join(
argument
for _root, arguments in all_builds
for argument in arguments
))
def test_payload_allowlist_rejects_runtime_secrets_and_broad_paths(self):
for allowed in (
".dockerignore",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract/src/index.mjs",
"services/device-control-core/Dockerfile",
"services/device-gateway/src/runtime.mjs",
):
self.assertTrue(
RUNNER.allowed_payload_path("device-plane", allowed),
)
for rejected in (
".env",
"secrets/postgres-password",
"runtime/postgres/data",
"services/unknown/server.mjs",
"docker-compose.yml",
"services/device-control-core/start.sh",
"services/device-control-core/test/app.test.mjs",
):
with self.assertRaises(RUNNER.DeployError):
RUNNER.allowed_payload_path("device-plane", rejected)
def test_healthchecks_are_service_scoped_and_fail_closed(self):
core_checks = RUNNER.component_healthchecks(
"device-plane",
("services/device-control-core",),
("device-control-core",),
)
self.assertEqual(len(core_checks), 1)
self.assertEqual(
core_checks[0]["expected_json"]["commandTransport"],
"disabled",
)
self.assertEqual(
core_checks[0]["expected_json"]["discoveryIngest"],
"disabled",
)
gateway_checks = RUNNER.component_healthchecks(
"device-plane",
("services/device-gateway",),
("device-gateway",),
)
self.assertEqual(len(gateway_checks), 1)
self.assertEqual(
gateway_checks[0]["expected_json"]["publicIngress"],
"disabled",
)
self.assertEqual(
gateway_checks[0]["expected_json"]["tcpListener"],
"disabled",
)
def test_runtime_secrets_are_runner_owned_and_not_manifest_selected(self):
with mock.patch.object(
RUNNER,
"ensure_platform_runtime_secret",
) as ensure:
RUNNER.prepare_component_runtime(
"device-plane",
("services/device-control-core",),
)
self.assertEqual(
[call.args[0] for call in ensure.call_args_list],
[
RUNNER.DEVICE_PLANE_POSTGRES_PASSWORD_FILE,
RUNNER.DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE,
RUNNER.DEVICE_PLANE_IDENTIFIER_PEPPER_FILE,
],
)
def test_compose_runtime_uses_no_deps_and_never_selects_database(self):
with mock.patch.object(RUNNER.subprocess, "run") as run:
RUNNER.run_compose(
"device-plane",
("device-control-core", "device-gateway"),
("docker-compose.device-plane.yml",),
)
command = run.call_args_list[0].args[0]
self.assertIn("--no-deps", command)
self.assertNotIn("device-postgres", command)
self.assertEqual(
command[-2:],
["device-control-core", "device-gateway"],
)
def test_initial_candidate_rollback_removes_only_stateless_services(self):
entries = (
"docker-compose.device-plane.yml",
"services/device-control-core",
"services/device-gateway",
)
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-rollback-",
) as directory:
root = Path(directory) / "live"
backup = Path(directory) / "backup"
root.mkdir()
backup.mkdir()
(backup / "existing-files.txt").write_text("", encoding="utf-8")
(backup / "missing-files.txt").write_text(
"\n".join(entries) + "\n",
encoding="utf-8",
)
with (
mock.patch.object(
RUNNER,
"stop_and_remove_compose_services",
) as stop,
mock.patch.object(
RUNNER,
"restore_platform_overlay",
return_value=3,
) as restore,
):
result = RUNNER.rollback_device_plane_apply(
root,
backup,
entries,
"test-stamp",
True,
("device-control-core", "device-gateway"),
)
stop.assert_called_once_with(
"device-plane",
("device-control-core", "device-gateway"),
)
restore.assert_called_once()
self.assertEqual(result, "source-restored-runtime-unchanged:3")
if __name__ == "__main__":
unittest.main(verbosity=2)