SILVER
/
NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
NODEDC_PLATFORM/infra/deploy-runner/nodedc-deploy

3356 lines
141 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
import argparse
import json
import os
import re
import secrets
import shutil
import stat
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
EXPECTED_SELF = Path("/usr/local/sbin/nodedc-deploy")
DEPLOY_ROOT = Path("/volume1/docker/nodedc-deploy")
INBOX = DEPLOY_ROOT / "inbox"
APPLIED_DIR = DEPLOY_ROOT / "applied"
FAILED_DIR = DEPLOY_ROOT / "failed"
BACKUPS_DIR = DEPLOY_ROOT / "backups"
STATE_DIR = DEPLOY_ROOT / "state"
TMP_DIR = DEPLOY_ROOT / ".tmp"
STATE_FILE = STATE_DIR / "applied.jsonl"
FAILED_STATE_FILE = STATE_DIR / "failed.jsonl"
LOCK_DIR = STATE_DIR / "deploy.lock"
DOCKER = Path("/usr/local/bin/docker")
MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets")
MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret"
MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token"
PROXY_CONTUR_ENV_FILE = Path("/volume1/docker/proxy-contur/.env")
DC_AMD_PROXY_RUNTIME_DIR = Path("/volume1/docker/dc-amd-proxy/runtime")
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-provisioner"
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "token"
EXTERNAL_DATA_PLANE_READER_GRANTS_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-reader-grants"
FOUNDRY_BINDING_GRANTS_DIR = MAP_GATEWAY_SECRET_DIR / "foundry-binding-grants"
N8N_PRIVATE_EXTENSION_RELEASES_ROOT = Path("/volume1/docker/nodedc-platform/n8n-private-extensions")
ENGINE_N8N_SEALED_RELEASES_ROOT = Path("/volume2/nodedc-demo/n8n-private-extensions")
ENGINE_N8N_TRANSITION_DESCRIPTOR_REL = "nodedc-source/services/n8n/private-extensions/ndc-activation.json"
ENGINE_N8N_COMPOSE_OVERRIDE_REL = "nodedc-source/services/n8n/private-extensions/docker-compose.ndc-private-extension.yml"
ENGINE_N8N_SCHEMA_ROOT_REL = "nodedc-source/server/assets/n8n/schema/v2.3.2"
ENGINE_N8N_NODES_CATALOG_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/nodes.catalog.json"
ENGINE_N8N_CREDENTIALS_CATALOG_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/credentials.catalog.json"
ENGINE_N8N_SCHEMA_META_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/meta.json"
ENGINE_N8N_ICON_REL = "nodedc-source/server/assets/n8n/icons/ndc.svg"
ENGINE_N8N_DARK_ICON_REL = "nodedc-source/server/assets/n8n/icons/ndc.dark.svg"
ENGINE_N8N_VERSION = "2.3.2"
ENGINE_N8N_BASE_IMAGE = "docker.n8n.io/n8nio/n8n:2.3.2"
ENGINE_N8N_BASE_ARCHITECTURE = "amd64"
ENGINE_N8N_RUNTIME_PACKAGE_PATH = "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc"
ENGINE_N8N_ICON_SHA256 = "1c928fc996d8b82121e8f8e327d74b1dd21556c106de99c5e8d317d926c0ba17"
ENGINE_N8N_DARK_ICON_SHA256 = "ef3b8551da4ce04527736405afa9a220a2c76d047bd18f6f7124b9adb4216b0c"
ENGINE_N8N_ACTIVATION_NODES_CATALOG_JSON_SHA256 = "230f712230f7ee8debaa4b44a4358cc19c205bc43305d8ed89d5e3daac466506"
ENGINE_N8N_ACTIVATION_CREDENTIALS_CATALOG_JSON_SHA256 = "a26824ffc0e15db857c792153de3417febc0dbba4880380d6c8cd544b3c80f4d"
ENGINE_N8N_ACTIVATION_META_JSON_SHA256 = "caf7635420c61333e65f68350f5567217aed96fb51354b38764bcac2c24e7966"
ENGINE_N8N_INACTIVE_NODES_CATALOG_JSON_SHA256 = "b70d9d8130d498c55de46a5d0758c844f1242b70803457b2291ab3a9de8056f2"
ENGINE_N8N_INACTIVE_CREDENTIALS_CATALOG_JSON_SHA256 = "680e9f52aac791efbd38e3bd99bd51ef5cded9756d867897c6c2755850e87b50"
ENGINE_N8N_INACTIVE_META_JSON_SHA256 = "0ac555d7ab31cb0ca042db4f474e83cfefbf20b5bbb2e1509fead27ed21e8acb"
MAP_GATEWAY_RUNTIME_GID = 1000
EXTERNAL_DATA_PLANE_RUNTIME_UID = 11006
EXTERNAL_DATA_PLANE_RUNTIME_GID = 11006
MAX_ARTIFACT_BYTES = 512 * 1024 * 1024
MAX_MEMBER_COUNT = 20000
MAX_PAYLOAD_BYTES = 1024 * 1024 * 1024
MAX_FILE_BYTES = 256 * 1024 * 1024
PATCH_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,96}$")
MANIFEST_KEYS = {"id", "component", "type"}
MAP_GATEWAY_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{48,256}$")
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{48,256}$")
N8N_PRIVATE_EXTENSION_RELEASE_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+-[a-f0-9]{16}$")
N8N_PRIVATE_EXTENSION_NODES = (
"dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",
"dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js",
"dist/nodes/NdcFoundryBinding/NdcFoundryBinding.node.js",
)
N8N_PRIVATE_EXTENSION_CREDENTIALS = (
"dist/credentials/NdcDataProductWriterApi.credentials.js",
"dist/credentials/NdcDataProductReaderApi.credentials.js",
"dist/credentials/NdcFoundryBindingApi.credentials.js",
)
ENGINE_N8N_NODE_TYPES = (
"n8n-nodes-ndc.ndcDataProductPublish",
"n8n-nodes-ndc.ndcDataProductRead",
"n8n-nodes-ndc.ndcFoundryBinding",
)
ENGINE_N8N_CREDENTIAL_TYPES = (
"ndcDataProductWriterApi",
"ndcDataProductReaderApi",
"ndcFoundryBindingApi",
)
ENGINE_BASE_COMPOSE_SERVICES = (
"postgresql",
"authentik-server",
"authentik-worker",
"n8n-postgres",
"ops-postgres",
"n8n",
"nodedc-backend",
"nodedc-frontend-build",
"app",
)
COMPONENTS = {
"engine": {
"payload_root": Path("/volume2/nodedc-demo"),
"compose_root": Path("/volume2/nodedc-demo"),
"services": ("nodedc-backend", "app"),
"compose_no_deps": True,
"publish_dist": True,
"healthchecks": (
"http://127.0.0.1:8080/",
"http://127.0.0.1:3001/health",
),
},
"launcher": {
"payload_root": Path("/volume1/docker/nodedc-platform/launcher/source"),
"build_root": Path("/volume1/docker/nodedc-platform/launcher/source"),
"build": ("build", "--no-cache", "-t", "nodedc/launcher:local", "."),
"compose_root": Path("/volume1/docker/nodedc-platform/platform"),
"compose_env_file": Path("/volume1/docker/nodedc-platform/platform/.env.synology"),
"compose_files": (
Path("/volume1/docker/nodedc-platform/platform/docker-compose.platform-http.yml"),
),
"compose_no_deps": True,
"services": ("launcher",),
"healthchecks": (
{
"url": "http://127.0.0.1:18080/healthz",
"headers": {"Host": "hub.nodedc.ru"},
},
),
},
"platform": {
"payload_root": Path("/volume1/docker/nodedc-platform"),
"compose_root": Path("/volume1/docker/nodedc-platform/platform"),
"compose_env_file": Path("/volume1/docker/nodedc-platform/platform/.env.synology"),
"compose_files": (
Path("/volume1/docker/nodedc-platform/platform/docker-compose.platform-http.yml"),
Path("/volume1/docker/nodedc-platform/platform/docker-compose.external-data-plane.yml"),
),
"compose_no_deps": True,
"services": ("notification-postgres", "notification-core", "ai-workspace-hub", "launcher", "reverse-proxy", "external-data-plane-postgres", "external-data-plane", "map-gateway"),
"healthchecks": (
"http://127.0.0.1:5185/healthz",
"http://127.0.0.1:18081/healthz",
{
"url": "http://127.0.0.1:18080/healthz",
"headers": {"Host": "hub.nodedc.ru"},
},
{
"url": "http://127.0.0.1:18080/",
"headers": {"Host": "id.nodedc.ru"},
},
),
},
"tasker": {
"payload_root": Path("/volume1/docker/nodedc-platform/tasker"),
"compose_root": Path("/volume1/docker/nodedc-platform/tasker/plane-app"),
"compose_project": "nodedc-tasker",
"compose_env_file": Path("/volume1/docker/nodedc-platform/tasker/plane-app/.env.synology"),
"compose_files": (
Path("/volume1/docker/nodedc-platform/tasker/plane-app/docker-compose.yaml"),
Path("/volume1/docker/nodedc-platform/tasker/plane-app/docker-compose.synology.override.yml"),
),
"services": ("api", "worker", "beat-worker", "web"),
"healthchecks": (
{
"url": "http://127.0.0.1:18090/",
"headers": {"Host": "ops.nodedc.ru"},
},
),
},
"ops-agents": {
"payload_root": Path("/volume1/docker/nodedc-platform/ops-agents"),
"compose_root": Path("/volume1/docker/nodedc-platform/ops-agents"),
"compose_env_file": Path("/volume1/docker/nodedc-platform/ops-agents/.env"),
"compose_files": (
Path("/volume1/docker/nodedc-platform/ops-agents/docker-compose.synology.yml"),
),
"compose_build": True,
"compose_no_deps": True,
"services": ("agent-gateway",),
"healthchecks": (
"http://172.22.0.222:18190/readyz",
),
},
"bim-viewer": {
"payload_root": Path("/volume1/docker/nodedc-platform/bim-viewer/source"),
"build_root": Path("/volume1/docker/nodedc-platform/bim-viewer/source/converter"),
"build": ("build", "--no-cache", "-t", "nodedc/bim-converter:local", "."),
"compose_root": Path("/volume1/docker/nodedc-platform/bim-viewer/source"),
"compose_env_file": Path("/volume1/docker/nodedc-platform/bim-viewer/source/.env"),
"compose_files": (
Path("/volume1/docker/nodedc-platform/bim-viewer/source/docker-compose.beam.yml"),
),
"compose_no_deps": True,
"services": ("ndc-beam-viewer", "nodedc-bim-converter"),
"healthchecks": (
"http://127.0.0.1:18100/api/auth/session",
),
},
"n8n-private-extension": {
# This component only stages a verified, immutable offline release in
# Platform-owned storage. It deliberately has no Compose file, service,
# container mutation or Engine activation side effect.
"payload_root": N8N_PRIVATE_EXTENSION_RELEASES_ROOT,
"bootstrap_root": True,
"artifact_only": True,
"immutable_payload": True,
"services": (),
},
"module-foundry": {
"payload_root": Path("/volume1/docker/nodedc-platform/module-foundry/source"),
"compose_root": Path("/volume1/docker/nodedc-platform/module-foundry/source/infra"),
"compose_project": "nodedc-module-foundry",
"compose_env_file": Path("/volume1/docker/nodedc-platform/module-foundry/source/.env"),
"compose_files": (
Path("/volume1/docker/nodedc-platform/module-foundry/source/infra/docker-compose.module-foundry.yml"),
),
"bootstrap_root": True,
"compose_build": True,
"compose_no_deps": True,
"services": ("nodedc-module-foundry",),
"healthchecks": (
"http://172.22.0.222:9920/healthz",
),
},
"proxy-contur": {
"payload_root": Path("/volume1/docker/proxy-contur"),
"compose_root": Path("/volume1/docker/proxy-contur"),
"compose_project": "proxy-contur",
"compose_env_file": Path("/volume1/docker/proxy-contur/.env"),
"compose_files": (
Path("/volume1/docker/proxy-contur/docker-compose.yml"),
),
"compose_build": True,
"compose_no_deps": True,
"services": ("proxy-contur",),
"health_container": "proxy-contur",
},
"dc-amd-proxy": {
"payload_root": Path("/volume1/docker/dc-amd-proxy"),
"compose_root": Path("/volume1/docker/dc-amd-proxy"),
"compose_project": "dc-amd-proxy",
"bootstrap_root": True,
"compose_build": True,
"compose_no_deps": True,
"services": ("dc-amd-proxy",),
"health_container": "dc-amd-proxy",
},
"dc-cms": {
"payload_root": Path("/volume1/docker/dc-cms/source"),
"compose_root": Path("/volume1/docker/dc-cms/source/infra"),
"compose_env_file": Path("/volume1/docker/dc-cms/source/infra/.env.synology"),
"compose_files": (
Path("/volume1/docker/dc-cms/source/infra/docker-compose.yml"),
),
"bootstrap_root": True,
"compose_build": True,
"services": ("postgresql-authentik", "authentik-server", "authentik-worker", "authentik-bootstrap", "cms-app", "reverse-proxy"),
"healthchecks": (
{
"url": "http://172.22.0.222:9918/auth/login?returnTo=%2F",
"headers": {"Host": "cms.dcserve.ru"},
},
{
"url": "http://172.22.0.222:9919/",
"headers": {"Host": "auth.dcserve.ru"},
},
),
},
"dc-cms-site-nodedc": {
"payload_root": Path("/volume1/docker/dc-cms/sites/nodedc"),
"compose_root": Path("/volume1/docker/dc-cms/source/infra"),
"compose_env_file": Path("/volume1/docker/dc-cms/source/infra/.env.synology"),
"compose_files": (
Path("/volume1/docker/dc-cms/source/infra/docker-compose.yml"),
),
"compose_no_deps": True,
"services": ("cms-app", "reverse-proxy"),
"healthchecks": (
{
"url": "http://172.22.0.222:9918/auth/login?returnTo=%2F",
"headers": {"Host": "cms.dcserve.ru"},
},
),
},
}
class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
NO_REDIRECT_OPENER = urllib.request.build_opener(NoRedirectHandler)
class DeployError(Exception):
pass
def die(message):
raise DeployError(message)
def utc_now():
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def stamp():
return datetime.now().strftime("%Y%m%d-%H%M%S")
def sha256_file(path):
import hashlib
h = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
return h.hexdigest()
def sha256_json_value(value):
import hashlib
canonical = json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(canonical).hexdigest()
def safe_name(value):
return re.sub(r"[^A-Za-z0-9._-]", "_", value)
def is_relative_to(child, parent):
try:
child.relative_to(parent)
return True
except ValueError:
return False
def ensure_layout():
for path in (INBOX, APPLIED_DIR, FAILED_DIR, BACKUPS_DIR, STATE_DIR, TMP_DIR):
path.mkdir(parents=True, exist_ok=True)
for path in (STATE_DIR, BACKUPS_DIR, TMP_DIR):
os.chown(path, 0, 0)
path.chmod(0o700)
for path in (APPLIED_DIR, FAILED_DIR):
os.chown(path, 0, 0)
path.chmod(0o755)
def ensure_platform_runtime_secret(secret_file, secret_re, label):
# This is Platform infrastructure state, not a product setting. The
# root-owned runner creates it once and only explicitly bound service mounts
# can read it. It must never be copied into an artifact or a shared .env.
secret_dir = secret_file.parent
try:
directory_stat = secret_dir.lstat()
except FileNotFoundError:
secret_dir.mkdir(parents=True, exist_ok=False)
directory_stat = secret_dir.lstat()
if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode):
die(f"{label} secret directory is unsafe: {secret_dir}")
os.chown(secret_dir, 0, MAP_GATEWAY_RUNTIME_GID)
secret_dir.chmod(0o710)
try:
secret_stat = secret_file.lstat()
except FileNotFoundError:
secret_value = secrets.token_urlsafe(48)
temporary = secret_dir / f".{secret_file.name}.{os.getpid()}.{time.time_ns()}.tmp"
descriptor = None
try:
descriptor = os.open(str(temporary), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640)
os.write(descriptor, f"{secret_value}\n".encode("ascii"))
os.fsync(descriptor)
os.fchown(descriptor, 0, MAP_GATEWAY_RUNTIME_GID)
os.fchmod(descriptor, 0o640)
os.close(descriptor)
descriptor = None
os.replace(temporary, secret_file)
fsync_directory(secret_dir)
finally:
if descriptor is not None:
os.close(descriptor)
if temporary.exists():
temporary.unlink()
return "created"
if stat.S_ISLNK(secret_stat.st_mode) or not stat.S_ISREG(secret_stat.st_mode):
die(f"{label} secret file is unsafe: {secret_file}")
if secret_stat.st_uid != 0 or secret_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
die(f"{label} secret file has unsafe ownership or permissions: {secret_file}")
if secret_stat.st_size > 512:
die(f"{label} secret file is too large: {secret_file}")
try:
secret_value = secret_file.read_text(encoding="ascii").strip()
except UnicodeDecodeError:
die(f"{label} secret file is not ascii: {secret_file}")
if not secret_re.fullmatch(secret_value):
die(f"{label} secret file has invalid format: {secret_file}")
os.chown(secret_file, 0, MAP_GATEWAY_RUNTIME_GID)
secret_file.chmod(0o640)
return "reused"
def ensure_map_gateway_admin_secret():
return ensure_platform_runtime_secret(
MAP_GATEWAY_SECRET_FILE,
MAP_GATEWAY_SECRET_RE,
"map gateway",
)
def read_proxy_contur_token():
try:
source_stat = PROXY_CONTUR_ENV_FILE.lstat()
except FileNotFoundError:
die("proxy-contur env file is required for Map egress")
if stat.S_ISLNK(source_stat.st_mode) or not stat.S_ISREG(source_stat.st_mode):
die("proxy-contur env file is unsafe")
if source_stat.st_size > 64 * 1024 or source_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
die("proxy-contur env file has unsafe permissions")
try:
lines = PROXY_CONTUR_ENV_FILE.read_text(encoding="utf-8").splitlines()
except UnicodeDecodeError:
die("proxy-contur env file is not utf-8")
value = None
for raw in lines:
line = raw.strip()
if not line or line.startswith("#"):
continue
if line.startswith("export "):
line = line[7:].lstrip()
if not line.startswith("PROXY_TOKEN="):
continue
value = line.partition("=")[2].strip()
if len(value) >= 2 and value[0] in ("'", '"') and value[-1] == value[0]:
value = value[1:-1]
break
if not value or len(value) > 4096 or any(ord(char) < 33 or ord(char) == 127 for char in value):
die("proxy-contur PROXY_TOKEN is missing or invalid")
return value
def sync_map_egress_proxy_secret():
# The Map Gateway gets a read-only file copy of the existing proxy token.
# The value is never printed, placed in an artifact, or exposed to Foundry.
token = read_proxy_contur_token()
try:
parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
except FileNotFoundError:
MAP_GATEWAY_SECRET_DIR.mkdir(parents=True, exist_ok=False)
parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode):
die(f"map egress parent secret directory is unsafe: {MAP_GATEWAY_SECRET_DIR}")
os.chown(MAP_GATEWAY_SECRET_DIR, 0, MAP_GATEWAY_RUNTIME_GID)
MAP_GATEWAY_SECRET_DIR.chmod(0o710)
if MAP_EGRESS_PROXY_SECRET_FILE.exists() or MAP_EGRESS_PROXY_SECRET_FILE.is_symlink():
target_stat = MAP_EGRESS_PROXY_SECRET_FILE.lstat()
if stat.S_ISLNK(target_stat.st_mode) or not stat.S_ISREG(target_stat.st_mode):
die("map egress proxy secret file is unsafe")
temporary = MAP_GATEWAY_SECRET_DIR / f".{MAP_EGRESS_PROXY_SECRET_FILE.name}.{os.getpid()}.{time.time_ns()}.tmp"
descriptor = None
try:
descriptor = os.open(str(temporary), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640)
os.write(descriptor, f"{token}\n".encode("utf-8"))
os.fsync(descriptor)
os.fchown(descriptor, 0, MAP_GATEWAY_RUNTIME_GID)
os.fchmod(descriptor, 0o640)
os.close(descriptor)
descriptor = None
os.replace(temporary, MAP_EGRESS_PROXY_SECRET_FILE)
fsync_directory(MAP_GATEWAY_SECRET_DIR)
finally:
if descriptor is not None:
os.close(descriptor)
if temporary.exists():
temporary.unlink()
return "synced"
def ensure_root_owned_grant_directory(path, label):
try:
parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
except FileNotFoundError:
MAP_GATEWAY_SECRET_DIR.mkdir(parents=True, exist_ok=False)
parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode):
die(f"{label} parent directory is unsafe: {MAP_GATEWAY_SECRET_DIR}")
os.chown(MAP_GATEWAY_SECRET_DIR, 0, MAP_GATEWAY_RUNTIME_GID)
MAP_GATEWAY_SECRET_DIR.chmod(0o710)
try:
directory_stat = path.lstat()
except FileNotFoundError:
path.mkdir(parents=False, exist_ok=False)
directory_stat = path.lstat()
if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode):
die(f"{label} directory is unsafe: {path}")
os.chown(path, 0, 0)
path.chmod(0o500)
for entry in path.iterdir():
entry_stat = entry.lstat()
if (not re.fullmatch(r"[a-f0-9]{64}", entry.name)
or stat.S_ISLNK(entry_stat.st_mode)
or not stat.S_ISREG(entry_stat.st_mode)
or entry_stat.st_uid != 0
or entry_stat.st_gid != 0
or stat.S_IMODE(entry_stat.st_mode) != 0o400
or entry_stat.st_size < 2
or entry_stat.st_size > 128 * 1024):
die(f"{label} record is unsafe: {entry}")
def ensure_external_data_plane_provisioner_secret():
# Unlike the Map Gateway key, this capability can mint scoped writers.
# Keep it in an isolated child directory readable only by the dedicated
# EDP/provisioner uid, never by the shared gid 1000.
try:
parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
except FileNotFoundError:
MAP_GATEWAY_SECRET_DIR.mkdir(parents=True, exist_ok=False)
parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode):
die(f"external data plane parent secret directory is unsafe: {MAP_GATEWAY_SECRET_DIR}")
os.chown(MAP_GATEWAY_SECRET_DIR, 0, MAP_GATEWAY_RUNTIME_GID)
MAP_GATEWAY_SECRET_DIR.chmod(0o710)
try:
directory_stat = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.lstat()
except FileNotFoundError:
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.mkdir(parents=False, exist_ok=False)
directory_stat = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.lstat()
if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode):
die(f"external data plane provisioner secret directory is unsafe: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR}")
os.chown(EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR, EXTERNAL_DATA_PLANE_RUNTIME_UID, EXTERNAL_DATA_PLANE_RUNTIME_GID)
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.chmod(0o500)
try:
secret_stat = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE.lstat()
except FileNotFoundError:
secret_value = secrets.token_urlsafe(48)
temporary = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / f".{EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE.name}.{os.getpid()}.{time.time_ns()}.tmp"
descriptor = None
try:
descriptor = os.open(str(temporary), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o400)
os.write(descriptor, f"{secret_value}\n".encode("ascii"))
os.fsync(descriptor)
os.fchown(descriptor, EXTERNAL_DATA_PLANE_RUNTIME_UID, EXTERNAL_DATA_PLANE_RUNTIME_GID)
os.fchmod(descriptor, 0o400)
os.close(descriptor)
descriptor = None
os.replace(temporary, EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE)
fsync_directory(EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR)
finally:
if descriptor is not None:
os.close(descriptor)
if temporary.exists():
temporary.unlink()
return "created"
if stat.S_ISLNK(secret_stat.st_mode) or not stat.S_ISREG(secret_stat.st_mode):
die(f"external data plane provisioner secret file is unsafe: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
if (secret_stat.st_uid != EXTERNAL_DATA_PLANE_RUNTIME_UID or
secret_stat.st_gid != EXTERNAL_DATA_PLANE_RUNTIME_GID or
stat.S_IMODE(secret_stat.st_mode) != 0o400):
die(f"external data plane provisioner secret file has unsafe ownership or permissions: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
if secret_stat.st_size > 512:
die(f"external data plane provisioner secret file is too large: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
try:
secret_value = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE.read_text(encoding="ascii").strip()
except UnicodeDecodeError:
die(f"external data plane provisioner secret file is not ascii: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
if not EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_RE.fullmatch(secret_value):
die(f"external data plane provisioner secret file has invalid format: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
return "reused"
def fsync_directory(path):
descriptor = os.open(str(path), os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def require_root():
if os.geteuid() != 0:
die("run with sudo")
def verify_file_mode(path, require_root_owner):
try:
st = path.lstat()
except FileNotFoundError:
die(f"missing path: {path}")
if stat.S_ISLNK(st.st_mode):
die(f"path must not be a symlink: {path}")
if require_root_owner and (st.st_uid != 0 or st.st_gid != 0):
die(f"path must be root:root: {path}")
if st.st_mode & stat.S_IWGRP:
die(f"path must not be group-writable: {path}")
if st.st_mode & stat.S_IWOTH:
die(f"path must not be world-writable: {path}")
def verify_install():
invoked = Path(sys.argv[0]).absolute()
if invoked != EXPECTED_SELF:
die(f"runner must be invoked as {EXPECTED_SELF}, current: {invoked}")
verify_file_mode(EXPECTED_SELF, require_root_owner=True)
verify_file_mode(Path("/usr/local"), require_root_owner=True)
verify_file_mode(Path("/usr/local/sbin"), require_root_owner=True)
print(f"path={EXPECTED_SELF}")
print(f"sha256={sha256_file(EXPECTED_SELF)}")
print("verify-install-ok")
def validate_artifact_location(path):
if path.suffix != ".tgz":
die("artifact must have .tgz extension")
try:
st = path.lstat()
except FileNotFoundError:
die(f"artifact not found: {path}")
if stat.S_ISLNK(st.st_mode):
die(f"artifact must not be a symlink: {path}")
if not stat.S_ISREG(st.st_mode):
die(f"artifact must be a regular file: {path}")
if st.st_size > MAX_ARTIFACT_BYTES:
die(f"artifact too large: {st.st_size} bytes")
inbox_real = INBOX.resolve()
parent_real = path.parent.resolve()
if parent_real != inbox_real:
die(f"artifact must be inside {INBOX}")
name = path.name
if "/" in name or ".." in name or name.startswith("."):
die(f"unsafe artifact filename: {name}")
def validate_posix_path(value):
if not value:
die("empty path")
if "\\" in value:
die(f"backslash is not allowed in path: {value}")
pure = PurePosixPath(value)
if pure.is_absolute():
die(f"absolute path rejected: {value}")
if any(part in ("", ".", "..") for part in pure.parts):
die(f"path escape rejected: {value}")
if any(ord(ch) < 32 for ch in value):
die(f"control character rejected in path: {value!r}")
return pure
def tar_name_to_path(work_dir, name):
pure = validate_posix_path(name)
target = work_dir.joinpath(*pure.parts)
resolved = target.resolve(strict=False)
if not is_relative_to(resolved, work_dir.resolve()):
die(f"tar path escaped work dir: {name}")
return target
def validate_tar_member(member):
name = member.name
validate_posix_path(name)
if name not in ("manifest.env", "files.txt", "payload") and not name.startswith("payload/"):
die(f"unexpected tar member: {name}")
if not (member.isfile() or member.isdir()):
die(f"unsupported tar member type: {name}")
if member.mode & stat.S_ISUID or member.mode & stat.S_ISGID:
die(f"setuid/setgid tar member rejected: {name}")
if member.isfile() and member.size > MAX_FILE_BYTES:
die(f"file too large in artifact: {name}")
def scan_tar(artifact):
member_count = 0
payload_bytes = 0
names = set()
with tarfile.open(artifact, "r:gz") as tar:
for member in tar:
member_count += 1
if member_count > MAX_MEMBER_COUNT:
die("too many files in artifact")
validate_tar_member(member)
if member.name in names:
die(f"duplicate tar member rejected: {member.name}")
names.add(member.name)
if member.isfile() and member.name.startswith("payload/"):
payload_bytes += member.size
if payload_bytes > MAX_PAYLOAD_BYTES:
die("payload is too large")
if "manifest.env" not in names:
die("manifest.env missing")
if "files.txt" not in names:
die("files.txt missing")
if not any(name == "payload" or name.startswith("payload/") for name in names):
die("payload missing")
def safe_extract(artifact, work_dir):
scan_tar(artifact)
with tarfile.open(artifact, "r:gz") as tar:
for member in tar:
validate_tar_member(member)
target = tar_name_to_path(work_dir, member.name)
if member.isdir():
target.mkdir(parents=True, exist_ok=True)
target.chmod(0o755)
continue
if target.exists() and target.is_dir():
die(f"file target already exists as directory: {member.name}")
target.parent.mkdir(parents=True, exist_ok=True)
source = tar.extractfile(member)
if source is None:
die(f"cannot read tar member: {member.name}")
tmp = target.with_name(f"{target.name}.extracting")
with source, tmp.open("wb") as out:
shutil.copyfileobj(source, out, 1024 * 1024)
tmp.chmod(0o644)
os.replace(tmp, target)
def parse_manifest(path):
data = {}
with path.open("r", encoding="utf-8") as f:
for lineno, line in enumerate(f, 1):
line = line.rstrip("\n")
if not line:
continue
if "=" not in line:
die(f"manifest line {lineno} must be key=value")
key, value = line.split("=", 1)
if key not in MANIFEST_KEYS:
die(f"unsupported manifest key: {key}")
if key in data:
die(f"duplicate manifest key: {key}")
if any(ord(ch) < 32 for ch in value):
die(f"control character in manifest value: {key}")
data[key] = value
missing = sorted(MANIFEST_KEYS - set(data))
if missing:
die(f"manifest keys missing: {', '.join(missing)}")
if not PATCH_ID_RE.match(data["id"]):
die("manifest id contains unsafe characters")
if data["type"] != "app-overlay":
die(f"unsupported artifact type: {data['type']}")
if data["component"] not in COMPONENTS:
die(f"unsupported component: {data['component']}")
return data
def parse_files_list(path):
entries = []
seen = set()
with path.open("r", encoding="utf-8") as f:
for lineno, raw in enumerate(f, 1):
rel = raw.rstrip("\n")
if not rel:
continue
if rel.strip() != rel:
die(f"files.txt line {lineno} has leading/trailing whitespace")
validate_posix_path(rel)
if rel in seen:
die(f"duplicate files.txt entry: {rel}")
seen.add(rel)
entries.append(rel)
if not entries:
die("files.txt is empty")
return entries
def denied_payload_path(component, rel):
lower = rel.lower()
parts = lower.split("/")
base = parts[-1]
if component == "engine" and rel in (
"docker-compose.yml",
"nodedc-source/services/n8n/private-extensions/n8n-nodes-ndc/Dockerfile",
):
pass
elif component == "platform" and rel in (
"platform/docker-compose.platform-http.yml",
"platform/notification-core/Dockerfile",
"platform/ai-workspace-hub/Dockerfile",
"platform/ai-workspace-assistant/Dockerfile",
"platform/ontology-core/Dockerfile",
"platform/gelios-gateway/Dockerfile",
"platform/services/map-gateway/Dockerfile",
"platform/services/external-data-plane/Dockerfile",
):
pass
elif component == "bim-viewer" and rel in (
"converter/Dockerfile",
):
pass
elif component == "dc-cms" and rel in (
"Dockerfile",
"infra/docker-compose.yml",
):
pass
elif component == "module-foundry" and rel in (
"Dockerfile",
"infra/docker-compose.module-foundry.yml",
):
pass
elif component == "proxy-contur" and rel in (
"Dockerfile",
"docker-compose.yml",
):
pass
elif component == "dc-amd-proxy" and rel in (
"Dockerfile",
"docker-compose.yml",
):
pass
elif base in (".env", "dockerfile", "docker-compose.yml", "compose.yml"):
return "sensitive or infrastructure filename"
if base.endswith(".env") or base.endswith((".pem", ".crt", ".key", ".p12", ".pfx")):
return "secret-like file extension"
if base.endswith((".sh", ".bash", ".zsh")):
return "shell files are not allowed in app-overlay artifacts"
if base.endswith(".zip"):
return "zip backup files are not deployable artifacts"
if any(part in (".git", "node_modules") for part in parts):
return "repository/build dependency directory"
# `packages/tokens` is a versioned UI design-token package, not a secret
# store. Keep this narrowly scoped exception so the general filename guard
# remains in force for every other Module Foundry payload path.
if not (component == "module-foundry" and lower.startswith("packages/tokens/")) and any(
token in lower for token in ("secret", "token", "password")
):
return "secret-like path"
runtime_prefixes = ()
if component == "engine":
runtime_prefixes = (
"nodedc-source/server/data",
"nodedc-source/server/storage",
"nodedc-source/server/logs",
"nodedc-source/public/storage",
"nodedc-source/dist/storage",
)
if rel == "nodedc-source/server/.env":
return "server env file"
elif component == "launcher":
runtime_prefixes = (
"server/data",
"server/storage",
"server/logs",
"public/storage",
"dist/storage",
)
if rel == "server/.env":
return "server env file"
elif component == "platform":
runtime_prefixes = (
"backups",
"launcher",
"ops-agents",
"tasker",
"platform/.env.synology",
"platform/authentik",
"platform/docs",
"authentik/data",
"authentik/certs",
"authentik/media",
"authentik/postgresql",
)
if rel.startswith("platform/.env"):
return "platform env file"
if rel.startswith(("platform/Caddyfile.http.bak", "platform/docker-compose.platform-http.yml.bak")):
return "platform backup file"
elif component == "tasker":
runtime_prefixes = (
"plane-app/.local-web-root",
"plane-app/archive",
"plane-app/backup",
"plane-app/plane.env",
"plane-app/plane.env.example",
"plane-app/plane.env.staging.example",
"plane-app/plane.env.synology",
"plane-app/plane.env.synology.base",
"plane-src/.cache",
"plane-src/.next",
"plane-src/.pnpm-store",
"plane-src/.react-router",
"plane-src/.turbo",
"plane-src/build",
"plane-src/coverage",
"plane-src/dist",
"plane-src/playwright-report",
"plane-src/test-results",
)
if rel.startswith(("plane-app/docker-compose.yaml.bak", "plane-app/docker-compose.synology.override.yml.bak")):
return "tasker backup file"
elif component == "ops-agents":
runtime_prefixes = (
"dist",
)
if rel.startswith(".env"):
return "ops-agents env file"
elif component == "bim-viewer":
runtime_prefixes = (
"dist",
"frontend/dist",
"server/data",
"server/logs",
"server/storage",
"server/tmp",
)
if rel == ".env" or rel.startswith(".env."):
if rel != ".env.synology.example":
return "bim-viewer env file"
if rel.startswith(("docker-compose.beam.yml.bak",)):
return "bim-viewer backup file"
elif component == "module-foundry":
runtime_prefixes = (
"runtime-data",
"node_modules",
"apps/catalog/dist",
"server/data",
"server/logs",
"server/storage",
)
if rel == ".env" or rel.startswith(".env."):
if rel != ".env.example":
return "module-foundry env file"
if rel.startswith(("Dockerfile.bak", "infra/docker-compose.module-foundry.yml.bak")):
return "module-foundry backup file"
elif component == "n8n-private-extension":
# An extension release is inert data at this boundary. Activation is
# Engine-owned and must never be smuggled into the artifact as a script,
# Compose file, environment file, pointer or mount configuration.
extension_parts = PurePosixPath(rel).parts
is_release_directory = (
len(extension_parts) == 3
and extension_parts[0] == "releases"
and extension_parts[1] == "n8n-nodes-ndc"
and N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(extension_parts[2])
)
if not is_release_directory and base not in ("package.tgz", "release.json", "rollback.json"):
return "unexpected private extension release member"
elif component == "proxy-contur":
runtime_prefixes = (
"node_modules",
)
if rel == ".env" or rel.startswith(".env."):
return "proxy-contur env file"
if rel.startswith(("Dockerfile.bak", "docker-compose.yml.bak")):
return "proxy-contur backup file"
elif component == "dc-amd-proxy":
runtime_prefixes = (
"node_modules",
"runtime",
)
if rel == ".env" or rel.startswith(".env."):
return "dc-amd-proxy env file"
if rel.startswith(("Dockerfile.bak", "docker-compose.yml.bak")):
return "dc-amd-proxy backup file"
elif component == "dc-cms":
runtime_prefixes = (
"admin/uploads",
"infra/authentik/certs",
"infra/authentik/data",
"infra/authentik/media",
"infra/authentik/postgresql",
"server/data",
"server/logs",
"server/storage",
"sites",
)
if rel == ".env" or rel.startswith(".env."):
return "dc-cms root env file"
if rel in ("infra/.env", "infra/.env.synology"):
return "dc-cms env file"
if rel.startswith("infra/.env.") and rel not in (
"infra/.env.example",
"infra/.env.synology.example",
):
return "dc-cms env file"
if rel.startswith(("Dockerfile.bak", "infra/docker-compose.yml.bak")):
return "dc-cms backup file"
elif component == "dc-cms-site-nodedc":
runtime_prefixes = (
"assets/.trash",
"assets/uploads/.trash",
"node_modules",
)
if any(rel == prefix or rel.startswith(prefix + "/") for prefix in runtime_prefixes):
return "runtime data path"
return None
def allowed_payload_path(component, rel):
validate_posix_path(rel)
reason = denied_payload_path(component, rel)
if reason:
die(f"path rejected: {rel}: {reason}")
if component == "engine":
if rel in ("nodedc-source/package.json", "nodedc-source/package-lock.json"):
return True
if rel == "docker-compose.yml":
return True
if rel.startswith((
"nodedc-source/server/",
"nodedc-source/src/",
"nodedc-source/dist/",
"nodedc-source/public/",
"nodedc-source/workers/ai-workspace-bridge/",
)):
return True
if rel == "nodedc-source/services/n8n/private-extensions" or rel.startswith(
"nodedc-source/services/n8n/private-extensions/"
):
return True
if rel == "nodedc-source/dist":
return True
if component == "launcher":
if rel in (
"index.html",
"package.json",
"package-lock.json",
"tsconfig.json",
"tsconfig.app.json",
"tsconfig.node.json",
"vite.config.ts",
):
return True
if rel.startswith(("server/", "src/", "public/", "scripts/", "dc-ui-guideline/")):
return True
if component == "platform":
if rel in (
"platform/Caddyfile.http",
"platform/docker-compose.platform-http.yml",
"platform/docker-compose.external-data-plane.yml",
):
return True
if rel == "platform/notification-core" or rel.startswith("platform/notification-core/"):
return True
if rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/"):
return True
if rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/"):
return True
if rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/"):
return True
if rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/"):
return True
if rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/"):
return True
if rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/"):
return True
if rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/"):
return True
if rel == "authentik/custom-templates" or rel.startswith("authentik/custom-templates/"):
return True
if component == "tasker":
if rel in (
"plane-app/docker-compose.yaml",
"plane-app/docker-compose.synology.override.yml",
):
return True
if rel.startswith((
"plane-src/apps/api/",
"plane-src/apps/web/",
"plane-src/packages/",
)):
return True
if component == "ops-agents":
if rel in (
".dockerignore",
".env.synology.example",
".gitignore",
"Dockerfile",
"README.md",
"docker-compose.local.yml",
"docker-compose.synology.yml",
"package.json",
"package-lock.json",
"tsconfig.json",
):
return True
if rel.startswith((
"docs/",
"migrations/",
"src/",
)):
return True
if component == "bim-viewer":
if rel in (
"converter",
"embeddedDemos",
"frontend",
"images",
"locales",
"processor",
"report",
"src",
):
return True
if rel in (
".env.synology.example",
".esdoc.json",
".gitattributes",
".gitignore",
".release-please-manifest.json",
"CHANGELOG.md",
"CODE_OF_CONDUCT.md",
"LICENSE",
"README.md",
"SECURITY.md",
"_config.yml",
"changelog-template.hbs",
"docker-compose.beam.yml",
"embeddedViewer.html",
"index.js",
"package.json",
"package-lock.json",
"release-please-config.json",
"rollup.config.js",
"rollup.dev.config.js",
"webComponentExample.html",
"xeokit-bim-viewer.css",
):
return True
if rel.startswith((
"converter/",
"embeddedDemos/",
"frontend/",
"images/",
"locales/",
"processor/",
"report/",
"server/",
"src/",
)):
return True
if component == "module-foundry":
if rel in (
".dockerignore",
".env.example",
".gitignore",
"Dockerfile",
"package.json",
"package-lock.json",
"tsconfig.base.json",
"infra/docker-compose.module-foundry.yml",
):
return True
if rel in ("apps", "packages", "registry", "runtime-seed", "scripts", "server"):
return True
if rel.startswith((
"apps/",
"packages/",
"registry/",
"runtime-seed/",
"scripts/",
"server/",
)):
return True
if component == "n8n-private-extension":
parts = PurePosixPath(rel).parts
if (
len(parts) == 3
and parts[0] == "releases"
and parts[1] == "n8n-nodes-ndc"
and N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(parts[2])
):
return True
if (
len(parts) == 4
and parts[0] == "releases"
and parts[1] == "n8n-nodes-ndc"
and N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(parts[2])
and parts[3] in ("package.tgz", "release.json", "rollback.json")
):
return True
if component == "proxy-contur":
if rel in (
"Dockerfile",
"README.md",
"docker-compose.yml",
"package.json",
"package-lock.json",
"server.js",
):
return True
if component == "dc-amd-proxy":
if rel in (
"Dockerfile",
"README.md",
"docker-compose.yml",
"package.json",
"server.mjs",
):
return True
if component == "dc-cms":
if rel in (
".dockerignore",
".gitignore",
"Dockerfile",
"README.md",
"package.json",
"package-lock.json",
"infra/.env.example",
"infra/.env.synology.example",
"infra/docker-compose.yml",
"infra/authentik/bootstrap-cms.py",
):
return True
if rel.startswith((
"admin/",
"infra/authentik/custom-templates/",
"infra/reverse-proxy/",
"projects/",
"server/",
)):
return True
if component == "dc-cms-site-nodedc":
if rel in (
".gitignore",
"README.md",
"asset-manifest.json",
"index.html",
"package.json",
"package-lock.json",
"robots.txt",
"sitemap.xml",
):
return True
if rel.startswith((
"assets/",
"content/",
"knowledge/",
"templates/",
"tools/",
)):
return True
die(f"path allowlist rejected: {rel}")
def path_is_covered_by_files_list(rel, entries):
return any(rel == entry or rel.startswith(entry.rstrip("/") + "/") for entry in entries)
def validate_payload_tree(component, payload_dir, entries):
root = payload_dir.resolve()
for path in payload_dir.rglob("*"):
resolved = path.resolve(strict=False)
if not is_relative_to(resolved, root):
die(f"payload path escaped payload dir: {path}")
rel = path.relative_to(payload_dir).as_posix()
if path.is_symlink():
die(f"payload symlink rejected: {rel}")
if not path.is_file() and not path.is_dir():
die(f"payload special file rejected: {rel}")
if path.is_dir():
continue
allowed_payload_path(component, rel)
if not path_is_covered_by_files_list(rel, entries):
die(f"payload file is not covered by files.txt: {rel}")
def read_strict_json(path, label, max_bytes=64 * 1024):
try:
file_stat = path.lstat()
except FileNotFoundError:
die(f"{label} missing: {path}")
if stat.S_ISLNK(file_stat.st_mode) or not stat.S_ISREG(file_stat.st_mode):
die(f"{label} must be a regular file")
if file_stat.st_size < 2 or file_stat.st_size > max_bytes:
die(f"{label} has invalid size")
def reject_duplicate_keys(pairs):
result = {}
for key, value in pairs:
if key in result:
die(f"{label} has duplicate key: {key}")
result[key] = value
return result
try:
return json.loads(
path.read_text(encoding="utf-8"),
object_pairs_hook=reject_duplicate_keys,
)
except UnicodeDecodeError:
die(f"{label} is not utf-8")
except json.JSONDecodeError as exc:
die(f"{label} is invalid json: {exc.msg}")
def require_exact_json_keys(value, expected, label):
if not isinstance(value, dict):
die(f"{label} must be an object")
actual = set(value)
if actual != set(expected):
die(f"{label} keys mismatch: expected={sorted(expected)} actual={sorted(actual)}")
def validate_n8n_package_tarball(package_path, release):
package_stat = package_path.lstat()
if stat.S_ISLNK(package_stat.st_mode) or not stat.S_ISREG(package_stat.st_mode):
die("private extension package must be a regular file")
if package_stat.st_size < 1024 or package_stat.st_size > 32 * 1024 * 1024:
die("private extension package has invalid size")
if sha256_file(package_path) != release["package"]["sha256"]:
die("private extension package sha256 mismatch")
if package_stat.st_size != release["package"]["bytes"]:
die("private extension package byte count mismatch")
names = set()
package_bytes = 0
package_json_bytes = None
packed_node_sources = {}
with tarfile.open(package_path, "r:gz") as archive:
for member_count, member in enumerate(archive, 1):
if member_count > 512:
die("private extension package has too many members")
validate_posix_path(member.name)
if member.name in names:
die(f"duplicate private extension package member: {member.name}")
names.add(member.name)
if member.name != "package" and not member.name.startswith("package/"):
die(f"private extension package member escaped package root: {member.name}")
if not (member.isfile() or member.isdir()):
die(f"unsupported private extension package member: {member.name}")
if member.name == "package":
if not member.isdir():
die("private extension package root must be a directory")
else:
package_rel = PurePosixPath(member.name).relative_to("package")
if any(part.startswith(".") or part.startswith("._") for part in package_rel.parts):
die(f"private extension package hidden member rejected: {member.name}")
if not (
package_rel.as_posix() in ("README.md", "package.json")
or package_rel.as_posix().startswith("dist/")
):
die(f"private extension package member is not allowlisted: {member.name}")
expected_mode = 0o644 if member.isfile() else 0o755
if stat.S_IMODE(member.mode) != expected_mode:
die(f"unsafe private extension package mode: {member.name}")
if member.isfile():
package_bytes += member.size
if member.size > 4 * 1024 * 1024 or package_bytes > 64 * 1024 * 1024:
die("private extension package payload is too large")
if member.name == "package/package.json":
source = archive.extractfile(member)
if source is None:
die("private extension package.json cannot be read")
package_json_bytes = source.read(64 * 1024 + 1)
if member.name.startswith("package/dist/nodes/") and member.name.endswith(".node.js"):
source = archive.extractfile(member)
if source is None:
die(f"private extension packed node cannot be read: {member.name}")
packed_node_sources[member.name[len("package/"):]] = source.read(4 * 1024 * 1024 + 1)
if package_json_bytes is None or len(package_json_bytes) > 64 * 1024:
die("private extension package.json missing or too large")
try:
package_json = json.loads(package_json_bytes.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
die("private extension package.json is invalid")
if package_json.get("name") != "n8n-nodes-ndc":
die("private extension package name mismatch")
if package_json.get("version") != release["package"]["version"]:
die("private extension package version mismatch")
if package_json.get("private") is not True:
die("private extension package must remain private")
if package_json.get("dependencies") not in (None, {}):
die("private extension runtime dependencies are forbidden")
scripts = package_json.get("scripts") or {}
if not isinstance(scripts, dict):
die("private extension package scripts must be an object")
for lifecycle in ("preinstall", "install", "postinstall", "prepack", "prepare", "postpack"):
if lifecycle in scripts:
die(f"private extension lifecycle script is forbidden: {lifecycle}")
n8n = package_json.get("n8n")
if not isinstance(n8n, dict):
die("private extension n8n registration missing")
registered = []
expected_registrations = {
"nodes": N8N_PRIVATE_EXTENSION_NODES,
"credentials": N8N_PRIVATE_EXTENSION_CREDENTIALS,
}
for field, expected in expected_registrations.items():
values = n8n.get(field)
if not isinstance(values, list) or not values:
die(f"private extension n8n.{field} registration missing")
if values != list(expected):
die(f"private extension n8n.{field} registration mismatch")
for value in values:
if not isinstance(value, str) or not value.startswith("dist/") or value not in names and f"package/{value}" not in names:
die(f"private extension n8n.{field} member missing: {value}")
registered.append(value)
if len(registered) != len(set(registered)):
die("private extension n8n registration contains duplicates")
packed_nodes = sorted(
name[len("package/"):]
for name in names
if name.startswith("package/dist/nodes/") and name.endswith(".node.js")
)
packed_credentials = sorted(
name[len("package/"):]
for name in names
if name.startswith("package/dist/credentials/") and name.endswith(".credentials.js")
)
if packed_nodes != sorted(N8N_PRIVATE_EXTENSION_NODES):
die("private extension packed node set mismatch")
if packed_credentials != sorted(N8N_PRIVATE_EXTENSION_CREDENTIALS):
die("private extension packed credential set mismatch")
for node_path in N8N_PRIVATE_EXTENSION_NODES:
source = packed_node_sources.get(node_path)
if source is None or len(source) > 4 * 1024 * 1024:
die(f"private extension packed node source missing or too large: {node_path}")
if b"usableAsTool" in source:
die(f"private extension packed node would generate a tool variant: {node_path}")
def validate_n8n_private_extension_release(payload_dir, entries):
if len(entries) != 1:
die("private extension artifact must contain exactly one immutable release entry")
release_rel = entries[0]
parts = PurePosixPath(release_rel).parts
if (
len(parts) != 3
or parts[0] != "releases"
or parts[1] != "n8n-nodes-ndc"
or not N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(parts[2])
):
die("private extension files.txt entry must be a digest-bound release directory")
release_dir = payload_dir / release_rel
expected_files = {"package.tgz", "release.json", "rollback.json"}
actual_files = {path.name for path in release_dir.iterdir() if path.is_file()}
actual_dirs = [path.name for path in release_dir.iterdir() if path.is_dir()]
if actual_dirs or actual_files != expected_files:
die("private extension release must contain only package.tgz, release.json and rollback.json")
release = read_strict_json(release_dir / "release.json", "private extension release manifest")
require_exact_json_keys(release, ("schemaVersion", "releaseId", "package", "storage", "activation"), "private extension release manifest")
require_exact_json_keys(release["package"], ("name", "version", "sha256", "bytes", "runtimeTypePrefix"), "private extension package manifest")
require_exact_json_keys(release["storage"], ("relativePath", "immutable"), "private extension storage manifest")
require_exact_json_keys(
release["activation"],
(
"owner",
"status",
"requiredCommunityPackagePath",
"requiresAtomicReleaseSwitch",
"requiresAllN8nProcessesRestart",
"requiresMcpSchemaAcceptance",
"rollbackBaselinePolicy",
),
"private extension activation manifest",
)
require_exact_json_keys(
release["activation"]["rollbackBaselinePolicy"],
("allowed", "firstActivation", "requiresPreActivationVerification"),
"private extension rollback baseline policy",
)
release_id = parts[2]
package = release["package"]
activation = release["activation"]
if release["schemaVersion"] != "nodedc.n8n-private-extension-release/v2":
die("private extension release schema mismatch")
if release["releaseId"] != release_id:
die("private extension release id mismatch")
if package["name"] != "n8n-nodes-ndc" or package["runtimeTypePrefix"] != "n8n-nodes-ndc.":
die("private extension package identity mismatch")
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", str(package["version"])):
die("private extension package version is invalid")
if not re.fullmatch(r"[a-f0-9]{64}", str(package["sha256"])):
die("private extension package sha256 is invalid")
if not isinstance(package["bytes"], int) or isinstance(package["bytes"], bool):
die("private extension package byte count is invalid")
if release_id != f"{package['version']}-{package['sha256'][:16]}":
die("private extension release id is not digest-bound")
if release["storage"] != {"relativePath": release_rel, "immutable": True}:
die("private extension storage manifest mismatch")
rollback_baseline_policy = {
"allowed": [
"previous_verified_immutable_release",
"verified_inactive",
],
"firstActivation": "verified_inactive",
"requiresPreActivationVerification": True,
}
if activation != {
"owner": "engine",
"status": "blocked_pending_engine_owned_mount",
"requiredCommunityPackagePath": "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc",
"requiresAtomicReleaseSwitch": True,
"requiresAllN8nProcessesRestart": True,
"requiresMcpSchemaAcceptance": True,
"rollbackBaselinePolicy": rollback_baseline_policy,
}:
die("private extension activation boundary mismatch")
rollback = read_strict_json(release_dir / "rollback.json", "private extension rollback manifest")
require_exact_json_keys(
rollback,
("schemaVersion", "releaseId", "packageSha256", "mode", "baselinePolicy", "steps", "forbidden"),
"private extension rollback manifest",
)
require_exact_json_keys(
rollback["baselinePolicy"],
("allowed", "firstActivation", "requiresPreActivationVerification"),
"private extension rollback manifest baseline policy",
)
if rollback != {
"schemaVersion": "nodedc.n8n-private-extension-rollback/v2",
"releaseId": release_id,
"packageSha256": package["sha256"],
"mode": "engine-owned-atomic-release-switch",
"baselinePolicy": rollback_baseline_policy,
"steps": [
"select_verified_previous_release_or_preverified_inactive_baseline",
"switch_engine_owned_mount_atomically",
"restart_all_n8n_processes",
"verify_mcp_schema_state_matches_selected_baseline",
],
"forbidden": [
"delete_active_release",
"mutate_engine_core",
"live_npm_install",
],
}:
die("private extension rollback manifest mismatch")
validate_n8n_package_tarball(release_dir / "package.tgz", release)
def is_engine_n8n_transition(component, entries):
return component == "engine" and entries is not None and ENGINE_N8N_TRANSITION_DESCRIPTOR_REL in entries
def read_engine_n8n_transition_descriptor(path, label="Engine n8n transition descriptor"):
descriptor = read_strict_json(path, label)
require_exact_json_keys(
descriptor,
(
"schemaVersion",
"action",
"releaseId",
"packageVersion",
"packageSha256",
"n8nVersion",
"baseImage",
"baseImageArchitecture",
"baseImageIdentityPolicy",
"sealedReleaseRelativePath",
"composeOverride",
"runtimePackagePath",
"topologyServices",
"expectedCurrent",
"expectedNodeTypes",
"expectedCredentialTypes",
"rollbackBaseline",
),
label,
)
action = descriptor.get("action")
if descriptor.get("schemaVersion") != "nodedc.engine-n8n-private-extension-transition/v1":
die("Engine n8n transition schema mismatch")
if action not in ("activate", "rollback-inactive"):
die("Engine n8n transition action mismatch")
release_id = descriptor.get("releaseId")
package_version = descriptor.get("packageVersion")
package_sha256 = descriptor.get("packageSha256")
if not isinstance(release_id, str) or not N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(release_id):
die("Engine n8n transition release id is invalid")
if not isinstance(package_version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", package_version):
die("Engine n8n transition package version is invalid")
if not isinstance(package_sha256, str) or not re.fullmatch(r"[a-f0-9]{64}", package_sha256):
die("Engine n8n transition package sha256 is invalid")
if release_id != f"{package_version}-{package_sha256[:16]}":
die("Engine n8n transition release identity mismatch")
expected_sealed_path = f"n8n-private-extensions/releases/n8n-nodes-ndc/{release_id}/package"
exact_common = {
"n8nVersion": ENGINE_N8N_VERSION,
"baseImage": ENGINE_N8N_BASE_IMAGE,
"baseImageArchitecture": ENGINE_N8N_BASE_ARCHITECTURE,
"baseImageIdentityPolicy": "running-container-and-local-tag-must-match",
"sealedReleaseRelativePath": expected_sealed_path,
"composeOverride": ENGINE_N8N_COMPOSE_OVERRIDE_REL,
"runtimePackagePath": ENGINE_N8N_RUNTIME_PACKAGE_PATH,
"topologyServices": ["n8n"],
"rollbackBaseline": "verified_inactive",
}
for key, expected in exact_common.items():
if descriptor.get(key) != expected:
die(f"Engine n8n transition {key} mismatch")
if action == "activate":
if descriptor.get("expectedCurrent") != "verified_inactive":
die("Engine n8n activation must start from the verified inactive baseline")
if descriptor.get("expectedNodeTypes") != list(ENGINE_N8N_NODE_TYPES):
die("Engine n8n activation node type set mismatch")
if descriptor.get("expectedCredentialTypes") != list(ENGINE_N8N_CREDENTIAL_TYPES):
die("Engine n8n activation credential type set mismatch")
else:
if descriptor.get("expectedCurrent") != release_id:
die("Engine n8n rollback expected-current release mismatch")
if descriptor.get("expectedNodeTypes") != [] or descriptor.get("expectedCredentialTypes") != []:
die("Engine n8n rollback must select the inactive catalog")
return descriptor
def expected_engine_n8n_compose_override(descriptor):
health_script = (
"const http=require('http');const req=http.get('http://127.0.0.1:5678/healthz/readiness',"
"r=>{r.resume();process.exit(r.statusCode===200?0:1)});req.on('error',()=>process.exit(1));"
"req.setTimeout(4000,()=>{req.destroy();process.exit(1)});"
)
health_test = json.dumps(["CMD", "node", "-e", health_script], separators=(",", ":"))
mount_source = f"/volume2/nodedc-demo/{descriptor['sealedReleaseRelativePath']}"
return "\n".join((
"services:",
" n8n:",
f" image: {ENGINE_N8N_BASE_IMAGE}",
" platform: linux/amd64",
" pull_policy: never",
" environment:",
" N8N_USER_FOLDER: /home/node",
' N8N_COMMUNITY_PACKAGES_ENABLED: "true"',
' N8N_COMMUNITY_PACKAGES_PREVENT_LOADING: "false"',
' N8N_REINSTALL_MISSING_PACKAGES: "false"',
" volumes:",
f" - {mount_source}:{ENGINE_N8N_RUNTIME_PACKAGE_PATH}:ro",
" healthcheck:",
f" test: {health_test}",
" interval: 10s",
" timeout: 5s",
" retries: 30",
" start_period: 30s",
" labels:",
f" nodedc.n8n-private-extension.release: {descriptor['releaseId']}",
f" nodedc.n8n-private-extension.package-sha256: {descriptor['packageSha256']}",
"",
))
def validate_engine_n8n_catalog_payload(payload_dir, descriptor):
action = descriptor["action"]
nodes_path = payload_dir / ENGINE_N8N_NODES_CATALOG_REL
credentials_path = payload_dir / ENGINE_N8N_CREDENTIALS_CATALOG_REL
meta_path = payload_dir / ENGINE_N8N_SCHEMA_META_REL
nodes = read_strict_json(
nodes_path,
"Engine n8n node catalog",
max_bytes=32 * 1024 * 1024,
)
credentials = read_strict_json(
credentials_path,
"Engine n8n credential catalog",
max_bytes=4 * 1024 * 1024,
)
meta = read_strict_json(meta_path, "Engine n8n schema metadata")
if not isinstance(nodes, list) or not isinstance(credentials, list):
die("Engine n8n schema catalogs must be arrays")
private_nodes = [
item for item in nodes
if isinstance(item, dict) and str(item.get("name") or "").startswith("n8n-nodes-ndc.")
]
private_credentials = [
item for item in credentials
if isinstance(item, dict) and str(item.get("name") or "") in ENGINE_N8N_CREDENTIAL_TYPES
]
expected_node_types = list(ENGINE_N8N_NODE_TYPES) if action == "activate" else []
expected_credential_types = list(ENGINE_N8N_CREDENTIAL_TYPES) if action == "activate" else []
if [item.get("name") for item in private_nodes] != expected_node_types:
die("Engine n8n pinned node catalog exact set mismatch")
if [item.get("name") for item in private_credentials] != expected_credential_types:
die("Engine n8n pinned credential catalog exact set mismatch")
if any("usableAsTool" in item for item in private_nodes):
die("Engine n8n pinned node catalog would generate tool variants")
if action == "activate":
if len(nodes) != 437 or len(credentials) != 388:
die("Engine n8n activation catalog count mismatch")
if any(not str(item.get("displayName") or "").startswith("NDC ") for item in private_nodes):
die("Engine n8n private node visible name mismatch")
if any(not str(item.get("displayName") or "").startswith("NDC ") for item in private_credentials):
die("Engine n8n private credential visible name mismatch")
private_catalog_text = json.dumps(
{"nodes": private_nodes, "credentials": private_credentials},
ensure_ascii=False,
).lower()
if "gelios" in private_catalog_text or "robot2b" in private_catalog_text:
die("provider business identity is forbidden in the Engine n8n extension catalog")
else:
if len(nodes) != 434 or len(credentials) != 385:
die("Engine n8n inactive baseline catalog count mismatch")
if not isinstance(meta, dict):
die("Engine n8n schema metadata must be an object")
if meta.get("n8nVersion") != ENGINE_N8N_VERSION:
die("Engine n8n schema metadata version mismatch")
if meta.get("nodeCount") != len(nodes) or meta.get("credentialCount") != len(credentials):
die("Engine n8n schema metadata count mismatch")
if action == "activate":
if meta.get("source") != f"n8n-core+n8n-nodes-ndc@{descriptor['packageVersion']}":
die("Engine n8n schema metadata source mismatch")
if sha256_file(payload_dir / ENGINE_N8N_ICON_REL) != ENGINE_N8N_ICON_SHA256:
die("Engine n8n light icon sha256 mismatch")
if sha256_file(payload_dir / ENGINE_N8N_DARK_ICON_REL) != ENGINE_N8N_DARK_ICON_SHA256:
die("Engine n8n dark icon sha256 mismatch")
expected_catalog_hashes = (
(nodes, ENGINE_N8N_ACTIVATION_NODES_CATALOG_JSON_SHA256, "node"),
(credentials, ENGINE_N8N_ACTIVATION_CREDENTIALS_CATALOG_JSON_SHA256, "credential"),
(meta, ENGINE_N8N_ACTIVATION_META_JSON_SHA256, "metadata"),
)
else:
expected_catalog_hashes = (
(nodes, ENGINE_N8N_INACTIVE_NODES_CATALOG_JSON_SHA256, "node"),
(credentials, ENGINE_N8N_INACTIVE_CREDENTIALS_CATALOG_JSON_SHA256, "credential"),
(meta, ENGINE_N8N_INACTIVE_META_JSON_SHA256, "metadata"),
)
for catalog, expected_sha256, label in expected_catalog_hashes:
if sha256_json_value(catalog) != expected_sha256:
die(f"Engine n8n {action} {label} catalog sha256 mismatch")
def validate_engine_n8n_transition(payload_dir, entries):
descriptor = read_engine_n8n_transition_descriptor(payload_dir / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL)
activation_entries = [
ENGINE_N8N_TRANSITION_DESCRIPTOR_REL,
ENGINE_N8N_COMPOSE_OVERRIDE_REL,
ENGINE_N8N_NODES_CATALOG_REL,
ENGINE_N8N_CREDENTIALS_CATALOG_REL,
ENGINE_N8N_SCHEMA_META_REL,
ENGINE_N8N_ICON_REL,
ENGINE_N8N_DARK_ICON_REL,
]
rollback_entries = [
ENGINE_N8N_TRANSITION_DESCRIPTOR_REL,
ENGINE_N8N_NODES_CATALOG_REL,
ENGINE_N8N_CREDENTIALS_CATALOG_REL,
ENGINE_N8N_SCHEMA_META_REL,
]
expected_entries = activation_entries if descriptor["action"] == "activate" else rollback_entries
if entries != expected_entries:
die("Engine n8n transition files.txt exact set/order mismatch")
if descriptor["action"] == "activate":
override = payload_dir / ENGINE_N8N_COMPOSE_OVERRIDE_REL
try:
override_text = override.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
die("Engine n8n Compose override cannot be read")
if override_text != expected_engine_n8n_compose_override(descriptor):
die("Engine n8n Compose override mismatch")
if "N8N_CUSTOM_EXTENSIONS" in override_text or "CUSTOM." in override_text:
die("Engine n8n custom-extension loader is forbidden")
validate_engine_n8n_catalog_payload(payload_dir, descriptor)
return descriptor
def current_engine_n8n_transition_descriptor():
path = component_root("engine") / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL
if not path.exists():
return None
if path.is_symlink() or not path.is_file():
die("installed Engine n8n transition descriptor is unsafe")
return read_engine_n8n_transition_descriptor(path, "installed Engine n8n transition descriptor")
def validate_engine_n8n_staged_release(descriptor):
release_rel = f"releases/n8n-nodes-ndc/{descriptor['releaseId']}"
release_dir = N8N_PRIVATE_EXTENSION_RELEASES_ROOT / release_rel
current = N8N_PRIVATE_EXTENSION_RELEASES_ROOT
for part in PurePosixPath(release_rel).parts:
current = current / part
try:
current_stat = current.lstat()
except FileNotFoundError:
die(f"staged private extension release is missing: {release_rel}")
if stat.S_ISLNK(current_stat.st_mode):
die(f"staged private extension release symlink rejected: {current}")
validate_n8n_private_extension_release(N8N_PRIVATE_EXTENSION_RELEASES_ROOT, [release_rel])
release = read_strict_json(release_dir / "release.json", "staged private extension release manifest")
if release.get("releaseId") != descriptor["releaseId"]:
die("staged private extension release id mismatch")
if release.get("package", {}).get("version") != descriptor["packageVersion"]:
die("staged private extension package version mismatch")
if release.get("package", {}).get("sha256") != descriptor["packageSha256"]:
die("staged private extension package sha256 mismatch")
for path in [release_dir, *release_dir.rglob("*")]:
path_stat = path.lstat()
if path_stat.st_uid != 0 or path_stat.st_gid != 0:
die(f"staged private extension release ownership mismatch: {path}")
expected_mode = 0o555 if stat.S_ISDIR(path_stat.st_mode) else 0o444
if stat.S_IMODE(path_stat.st_mode) != expected_mode:
die(f"staged private extension release mode mismatch: {path}")
return release_dir
def load_artifact(artifact, work_dir):
safe_extract(artifact, work_dir)
manifest_path = work_dir / "manifest.env"
files_path = work_dir / "files.txt"
payload_dir = work_dir / "payload"
if not manifest_path.is_file():
die("manifest.env missing after extract")
if not files_path.is_file():
die("files.txt missing after extract")
if not payload_dir.is_dir():
die("payload directory missing after extract")
manifest = parse_manifest(manifest_path)
entries = parse_files_list(files_path)
for rel in entries:
allowed_payload_path(manifest["component"], rel)
if not (payload_dir / rel).exists():
die(f"files.txt entry missing in payload: {rel}")
validate_payload_tree(manifest["component"], payload_dir, entries)
if manifest["component"] == "n8n-private-extension":
validate_n8n_private_extension_release(payload_dir, entries)
if manifest["component"] == "engine":
touches_private_extension = any(
rel == "nodedc-source/services/n8n/private-extensions"
or rel.startswith("nodedc-source/services/n8n/private-extensions/")
for rel in entries
)
if touches_private_extension and not is_engine_n8n_transition(manifest["component"], entries):
die("Engine private-extension payload requires the canonical transition descriptor")
if is_engine_n8n_transition(manifest["component"], entries):
validate_engine_n8n_transition(payload_dir, entries)
return manifest, entries, payload_dir
def load_state(path):
if not path.exists():
return []
rows = []
with path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
rows.append(json.loads(line))
return rows
def append_jsonl(path, row):
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a", encoding="utf-8") as f:
f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
def state_has_sha(sha):
return any(row.get("sha256") == sha for row in load_state(STATE_FILE))
def state_has_patch_id(patch_id):
return any(row.get("id") == patch_id for row in load_state(STATE_FILE))
class DeployLock:
def __enter__(self):
try:
LOCK_DIR.mkdir()
except FileExistsError:
die(f"deploy lock is busy: {LOCK_DIR}")
(LOCK_DIR / "pid").write_text(str(os.getpid()), encoding="utf-8")
return self
def __exit__(self, exc_type, exc, tb):
shutil.rmtree(LOCK_DIR, ignore_errors=True)
def component_root(component):
return COMPONENTS[component]["payload_root"]
def component_compose_root(component):
return COMPONENTS[component].get("compose_root", component_root(component))
def component_compose_project(component):
return COMPONENTS[component].get("compose_project")
def component_artifact_only(component):
return bool(COMPONENTS[component].get("artifact_only"))
def component_services(component, entries=None):
if is_engine_n8n_transition(component, entries):
# The actual Engine topology has one n8n process and no separately
# deployed worker/webhook services. The release barrier therefore
# force-recreates exactly this service and never touches its database.
return ("n8n",)
if component == "dc-cms" and entries is not None:
selected = []
def add(*services):
for service in services:
if service not in selected:
selected.append(service)
touches_compose = any(rel == "infra/docker-compose.yml" for rel in entries)
touches_proxy = any(rel == "infra/reverse-proxy" or rel.startswith("infra/reverse-proxy/") for rel in entries)
touches_authentik = any(rel == "infra/authentik" or rel.startswith("infra/authentik/") for rel in entries)
touches_app = any(
rel in ("Dockerfile", "package.json", "package-lock.json", "README.md")
or rel.startswith(("admin/", "projects/", "server/"))
for rel in entries
)
if touches_compose:
return COMPONENTS[component]["services"]
if touches_authentik:
add("postgresql-authentik", "authentik-server", "authentik-worker", "authentik-bootstrap", "cms-app", "reverse-proxy")
if touches_app:
add("cms-app")
if touches_proxy:
add("reverse-proxy")
if selected:
return tuple(selected)
if component == "tasker" and entries is not None:
selected = ["api", "worker", "beat-worker", "web"]
touches_compose = any(
rel in ("plane-app/docker-compose.yaml", "plane-app/docker-compose.synology.override.yml")
for rel in entries
)
touches_proxy = any(rel == "plane-src/apps/proxy/Caddyfile.ce" for rel in entries)
if touches_compose or touches_proxy:
selected.append("proxy")
return tuple(selected)
if component == "platform" and entries is not None:
selected = []
def add(*services):
for service in services:
if service not in selected:
selected.append(service)
touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries)
touches_caddy = any(rel == "platform/Caddyfile.http" for rel in entries)
touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries)
touches_ai_workspace = any(rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/") for rel in entries)
touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries)
touches_ontology = any(rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/") for rel in entries)
touches_gelios = any(rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/") for rel in entries)
touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries)
touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries)
touches_authentik = any(rel == "authentik/custom-templates" or rel.startswith("authentik/custom-templates/") for rel in entries)
map_gateway_only_compose = touches_compose and touches_map_gateway and not any((touches_notification, touches_ai_workspace, touches_ai_workspace_assistant, touches_ontology, touches_gelios, touches_external_data_plane, touches_authentik))
restart_all_for_compose = touches_compose and not map_gateway_only_compose
if touches_notification or restart_all_for_compose:
add("notification-postgres", "notification-core", "launcher", "reverse-proxy")
if touches_ai_workspace or restart_all_for_compose:
add("ai-workspace-hub", "reverse-proxy")
if touches_ai_workspace_assistant or restart_all_for_compose:
add("ai-workspace-postgres", "ai-workspace-assistant")
if touches_ontology or restart_all_for_compose:
add("ontology-core", "ai-workspace-hub")
if touches_gelios or restart_all_for_compose:
add("gelios-postgres", "gelios-gateway")
if touches_map_gateway:
add("map-gateway")
if touches_external_data_plane:
add("external-data-plane-postgres", "external-data-plane")
if touches_authentik:
add("authentik-server", "authentik-worker", "reverse-proxy")
if touches_caddy:
add("reverse-proxy")
if selected:
return tuple(selected)
return COMPONENTS[component]["services"]
def component_compose_no_deps(component, entries=None):
if component == "dc-cms" and entries is not None:
touches_compose = any(rel == "infra/docker-compose.yml" for rel in entries)
touches_proxy = any(rel == "infra/reverse-proxy" or rel.startswith("infra/reverse-proxy/") for rel in entries)
touches_authentik = any(rel == "infra/authentik" or rel.startswith("infra/authentik/") for rel in entries)
return not (touches_compose or touches_proxy or touches_authentik)
return bool(COMPONENTS[component].get("compose_no_deps"))
def component_compose_files(component):
if component == "engine":
root = component_root(component)
files = [root / "docker-compose.yml"]
descriptor = current_engine_n8n_transition_descriptor()
if descriptor and descriptor["action"] == "activate":
override = root / ENGINE_N8N_COMPOSE_OVERRIDE_REL
if override.is_symlink() or not override.is_file():
die("active Engine n8n Compose override is missing or unsafe")
try:
override_text = override.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
die("active Engine n8n Compose override cannot be read")
if override_text != expected_engine_n8n_compose_override(descriptor):
die("installed Engine n8n Compose override drift detected")
files.append(override)
return tuple(files)
return COMPONENTS[component].get("compose_files", ())
def component_compose_env_file(component):
return COMPONENTS[component].get("compose_env_file")
def component_build_root(component):
return COMPONENTS[component].get("build_root")
def component_build_args(component, entries=None):
if component == "platform" and entries is not None:
touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries)
return COMPONENTS[component].get("build") if touches_notification else None
if component == "bim-viewer" and entries is not None:
touches_converter = any(rel == "converter" or rel.startswith("converter/") for rel in entries)
return COMPONENTS[component].get("build") if touches_converter else None
return COMPONENTS[component].get("build")
def component_builds(component, entries=None):
if component == "platform" and entries is not None:
touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries)
touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries)
touches_ai_workspace = any(rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/") for rel in entries)
touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries)
touches_ontology = any(rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/") for rel in entries)
touches_gelios = any(rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/") for rel in entries)
touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries)
touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries)
map_gateway_only_compose = touches_compose and touches_map_gateway and not any((touches_notification, touches_ai_workspace, touches_ai_workspace_assistant, touches_ontology, touches_gelios, touches_external_data_plane))
rebuild_all_for_compose = touches_compose and not map_gateway_only_compose
builds = []
if touches_notification or rebuild_all_for_compose:
builds.append((
Path("/volume1/docker/nodedc-platform/platform/notification-core"),
("build", "--no-cache", "-t", "nodedc/notification-core:local", "."),
))
if touches_ai_workspace or rebuild_all_for_compose:
builds.append((
Path("/volume1/docker/nodedc-platform/platform/ai-workspace-hub"),
("build", "--no-cache", "-t", "nodedc/ai-workspace-hub:local", "."),
))
if touches_ai_workspace_assistant or rebuild_all_for_compose:
builds.append((
Path("/volume1/docker/nodedc-platform/platform/ai-workspace-assistant"),
("build", "--no-cache", "-t", "nodedc/ai-workspace-assistant:local", "."),
))
if touches_ontology or rebuild_all_for_compose:
builds.append((
Path("/volume1/docker/nodedc-platform/platform/ontology-core"),
("build", "--no-cache", "-t", "nodedc/ontology-core:local", "."),
))
if touches_gelios or rebuild_all_for_compose:
builds.append((
Path("/volume1/docker/nodedc-platform/platform/gelios-gateway"),
("build", "--no-cache", "-t", "nodedc/gelios-gateway:local", "."),
))
if touches_map_gateway:
builds.append((
Path("/volume1/docker/nodedc-platform/platform/services/map-gateway"),
("build", "--no-cache", "-t", "nodedc/map-gateway:local", "."),
))
if touches_external_data_plane:
builds.append((
Path("/volume1/docker/nodedc-platform/platform"),
("build", "--no-cache", "-f", "services/external-data-plane/Dockerfile", "-t", "nodedc/external-data-plane:local", "."),
))
return tuple(builds)
if component == "tasker":
plane_src = Path("/volume1/docker/nodedc-platform/tasker/plane-src")
return (
(
plane_src / "apps/api",
(
"build",
"--network=host",
"-t",
"nodedc/plane-backend:local",
"-f",
"Dockerfile.api",
".",
),
),
(
plane_src,
(
"build",
"--network=host",
"--build-arg",
"VITE_NODEDC_LAUNCHER_URL=https://hub.nodedc.ru",
"--build-arg",
"VITE_NODEDC_OIDC_LOGIN_ENABLED=1",
"-t",
"nodedc/plane-frontend:ru",
"-f",
"apps/web/Dockerfile.web.nas-legacy",
".",
),
),
)
build_root = component_build_root(component)
build_args = component_build_args(component, entries)
if build_root and build_args:
return ((build_root, build_args),)
return ()
def docker_json(args, label):
result = subprocess.run(
[str(DOCKER), *args],
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
die(f"{label} failed: exit={result.returncode}")
try:
return json.loads(result.stdout)
except json.JSONDecodeError:
die(f"{label} returned invalid JSON")
def engine_n8n_container_ids():
result = subprocess.run(
[*compose_base_cmd("engine"), "ps", "-q", "n8n"],
cwd=str(component_compose_root("engine")),
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
die("Engine n8n container lookup failed")
container_ids = [line.strip() for line in result.stdout.splitlines() if line.strip()]
if len(container_ids) != 1:
die(f"Engine n8n topology mismatch: expected=1 actual={len(container_ids)}")
return container_ids
def inspect_engine_n8n_base_image():
images = docker_json(["image", "inspect", ENGINE_N8N_BASE_IMAGE], "Engine n8n base image inspect")
if not isinstance(images, list) or len(images) != 1 or not isinstance(images[0], dict):
die("Engine n8n base image inspect shape mismatch")
image = images[0]
image_id = image.get("Id")
if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", image_id):
die("Engine n8n base image id is invalid")
if image.get("Architecture") != ENGINE_N8N_BASE_ARCHITECTURE or image.get("Os") != "linux":
die("Engine n8n base image platform mismatch")
return {
"id": image_id,
"architecture": image.get("Architecture"),
"repo_digests": tuple(image.get("RepoDigests") or ()),
}
def inspect_container(container_id):
containers = docker_json(["inspect", container_id], "Engine n8n container inspect")
if not isinstance(containers, list) or len(containers) != 1 or not isinstance(containers[0], dict):
die("Engine n8n container inspect shape mismatch")
return containers[0]
def engine_n8n_version(container_id):
result = subprocess.run(
[str(DOCKER), "exec", container_id, "n8n", "--version"],
check=False,
capture_output=True,
text=True,
timeout=60,
)
version = result.stdout.strip()
if result.returncode != 0 or version != ENGINE_N8N_VERSION:
die("Engine n8n runtime version mismatch")
return version
def engine_n8n_private_loader_catalog(container_id):
container = inspect_container(container_id)
package_mounts = [
mount for mount in (container.get("Mounts") or ())
if mount.get("Destination") == ENGINE_N8N_RUNTIME_PACKAGE_PATH
]
if not package_mounts:
return {"node_types": [], "credential_types": []}
if len(package_mounts) != 1 or package_mounts[0].get("RW") is not False:
die("Engine n8n package-loader probe found an unsafe package mount")
script = (
"const {PackageDirectoryLoader}=require('/usr/local/lib/node_modules/n8n/node_modules/n8n-core');"
"(async()=>{const loader=new PackageDirectoryLoader('"
+ ENGINE_N8N_RUNTIME_PACKAGE_PATH
+ "');await loader.loadAll();process.stdout.write(JSON.stringify({packageName:loader.packageName,"
"nodes:loader.types.nodes.map(x=>x.name),credentials:loader.types.credentials.map(x=>x.name)}))})()"
".catch(()=>process.exit(1));"
)
result = subprocess.run(
[str(DOCKER), "exec", container_id, "node", "-e", script],
check=False,
capture_output=True,
text=True,
timeout=120,
)
if result.returncode != 0:
die("Engine n8n package-loader probe failed")
try:
catalog = json.loads(result.stdout)
except json.JSONDecodeError:
die("Engine n8n package-loader probe returned invalid JSON")
require_exact_json_keys(catalog, ("packageName", "nodes", "credentials"), "Engine n8n package-loader probe")
if catalog.get("packageName") != "n8n-nodes-ndc":
die("Engine n8n package-loader probe package identity mismatch")
nodes = catalog.get("nodes")
credentials = catalog.get("credentials")
if not isinstance(nodes, list) or not all(isinstance(value, str) for value in nodes):
die("Engine n8n package-loader probe node set is invalid")
if not isinstance(credentials, list) or not all(isinstance(value, str) for value in credentials):
die("Engine n8n package-loader probe credential set is invalid")
return {
"node_types": [f"n8n-nodes-ndc.{value}" for value in nodes],
"credential_types": credentials,
}
def engine_n8n_current_state():
descriptor = current_engine_n8n_transition_descriptor()
if descriptor and descriptor["action"] == "activate":
return descriptor["releaseId"], descriptor
return "verified_inactive", descriptor
def validate_engine_n8n_base_compose_source():
compose_path = component_root("engine") / "docker-compose.yml"
try:
compose = compose_path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
die("Engine base Compose cannot be read")
services = re.findall(r"(?m)^ ([A-Za-z0-9_-]+):\s*$", compose)
if tuple(services) != ENGINE_BASE_COMPOSE_SERVICES:
die("Engine base Compose exact service topology mismatch")
match = re.search(r"(?ms)^ n8n:\s*\n(?P<body>.*?)(?=^ [A-Za-z0-9_-]+:\s*$)", compose)
if not match:
die("Engine base Compose n8n service cannot be isolated")
n8n_service = match.group("body")
expected_image = "image: docker.n8n.io/n8nio/n8n:${N8N_IMAGE_TAG:-2.3.2}"
if expected_image not in n8n_service:
die("Engine base Compose n8n version mismatch")
required_fragments = (
"./n8n-data:/home/node/.n8n",
"./nodedc-source/services/n8n/runtime-plugin:/opt/nodedc/runtime-plugin:ro",
"n8n-postgres:",
)
if any(fragment not in n8n_service for fragment in required_fragments):
die("Engine base Compose n8n persistence/runtime dependency mismatch")
forbidden_fragments = (
"build:",
"N8N_CUSTOM_EXTENSIONS",
"CUSTOM.",
ENGINE_N8N_RUNTIME_PACKAGE_PATH,
"N8N_USER_FOLDER",
"N8N_COMMUNITY_PACKAGES_ENABLED",
"N8N_COMMUNITY_PACKAGES_PREVENT_LOADING",
"N8N_REINSTALL_MISSING_PACKAGES",
)
if any(fragment in n8n_service for fragment in forbidden_fragments):
die("Engine base Compose contains a non-baseline n8n activation setting")
def preflight_engine_n8n_transition(descriptor, enforce_expected_current):
validate_engine_n8n_base_compose_source()
if descriptor["action"] == "activate":
validate_engine_n8n_staged_release(descriptor)
base_image = inspect_engine_n8n_base_image()
container_id = engine_n8n_container_ids()[0]
container = inspect_container(container_id)
state = container.get("State") or {}
if state.get("Status") != "running":
die("Engine n8n baseline container is not running")
if container.get("Image") != base_image["id"]:
die("Engine n8n running image does not match the local 2.3.2 base image")
version = engine_n8n_version(container_id)
current_state, current_descriptor = engine_n8n_current_state()
state_matches = current_state == descriptor["expectedCurrent"]
current_catalog = engine_n8n_private_loader_catalog(container_id)
current_types = current_catalog["node_types"]
current_credentials = current_catalog["credential_types"]
expected_current_types = list(ENGINE_N8N_NODE_TYPES) if current_state != "verified_inactive" else []
expected_current_credentials = list(ENGINE_N8N_CREDENTIAL_TYPES) if current_state != "verified_inactive" else []
if current_types != expected_current_types:
die("Engine n8n current live private-node set does not match its transition state")
if current_credentials != expected_current_credentials:
die("Engine n8n current live private-credential set does not match its transition state")
catalog_descriptor = (
current_descriptor
if current_state != "verified_inactive" and current_descriptor is not None
else inactive_engine_n8n_descriptor(descriptor)
)
validate_engine_n8n_catalog_payload(component_root("engine"), catalog_descriptor)
if enforce_expected_current and not state_matches:
die(
"Engine n8n transition stale current state: "
f"expected={descriptor['expectedCurrent']} actual={current_state}"
)
if current_descriptor and current_state != "verified_inactive":
if current_descriptor.get("packageSha256") != descriptor["packageSha256"]:
if enforce_expected_current:
die("Engine n8n current release package sha256 mismatch")
return {
"base_image_id": base_image["id"],
"base_image_architecture": base_image["architecture"],
"base_image_repo_digests": base_image["repo_digests"],
"container_id": container_id,
"current_state": current_state,
"current_types": current_types,
"current_credentials": current_credentials,
"n8n_version": version,
"state_matches": state_matches,
}
def add_engine_n8n_sealed_member_paths(expected_paths, member_name):
parts = PurePosixPath(member_name).parts
if not parts or parts[0] != "package":
die("Engine sealed n8n package member root mismatch")
for depth in range(1, len(parts) + 1):
expected_paths.add(PurePosixPath(*parts[:depth]).as_posix())
def validate_engine_n8n_sealed_release(release_dir, staged_release_dir, descriptor):
if release_dir.is_symlink() or not release_dir.is_dir():
die("Engine sealed n8n release is missing or unsafe")
for filename in ("package.tgz", "release.json", "rollback.json"):
installed = release_dir / filename
staged = staged_release_dir / filename
if installed.is_symlink() or not installed.is_file():
die(f"Engine sealed n8n release member is unsafe: {filename}")
if sha256_file(installed) != sha256_file(staged):
die(f"Engine sealed n8n release member mismatch: {filename}")
package_root = release_dir / "package"
if package_root.is_symlink() or not package_root.is_dir():
die("Engine sealed n8n package tree is missing or unsafe")
expected_paths = {"package", "package.tgz", "release.json", "rollback.json"}
with tarfile.open(staged_release_dir / "package.tgz", "r:gz") as archive:
for member in archive:
# npm package tarballs do not have to carry explicit directory
# members. Extraction still creates those parent directories, so
# they are part of the exact sealed tree rather than unexpected
# content.
add_engine_n8n_sealed_member_paths(expected_paths, member.name)
target = release_dir.joinpath(*PurePosixPath(member.name).parts)
target_stat = target.lstat()
if stat.S_ISLNK(target_stat.st_mode):
die(f"Engine sealed n8n package symlink rejected: {member.name}")
if member.isdir():
if not stat.S_ISDIR(target_stat.st_mode):
die(f"Engine sealed n8n package directory mismatch: {member.name}")
else:
if not stat.S_ISREG(target_stat.st_mode) or target_stat.st_size != member.size:
die(f"Engine sealed n8n package file mismatch: {member.name}")
source = archive.extractfile(member)
if source is None:
die(f"Engine sealed n8n package file unreadable: {member.name}")
import hashlib
expected_sha = hashlib.sha256(source.read()).hexdigest()
if sha256_file(target) != expected_sha:
die(f"Engine sealed n8n package content mismatch: {member.name}")
actual_paths = {
path.relative_to(release_dir).as_posix()
for path in release_dir.rglob("*")
}
if actual_paths != expected_paths:
die("Engine sealed n8n release tree exact member set mismatch")
for path in [release_dir, *release_dir.rglob("*")]:
path_stat = path.lstat()
if path_stat.st_uid != 0 or path_stat.st_gid != 0:
die(f"Engine sealed n8n release ownership mismatch: {path}")
expected_mode = 0o555 if stat.S_ISDIR(path_stat.st_mode) else 0o444
if stat.S_IMODE(path_stat.st_mode) != expected_mode:
die(f"Engine sealed n8n release mode mismatch: {path}")
package_json = read_strict_json(package_root / "package.json", "Engine sealed n8n package.json")
if package_json.get("name") != "n8n-nodes-ndc" or package_json.get("version") != descriptor["packageVersion"]:
die("Engine sealed n8n package identity mismatch")
def ensure_engine_n8n_sealed_release(descriptor, current_stamp):
staged_release_dir = validate_engine_n8n_staged_release(descriptor)
release_parent = ENGINE_N8N_SEALED_RELEASES_ROOT / "releases" / "n8n-nodes-ndc"
release_parent.mkdir(parents=True, exist_ok=True)
for path in (
ENGINE_N8N_SEALED_RELEASES_ROOT,
ENGINE_N8N_SEALED_RELEASES_ROOT / "releases",
release_parent,
):
path_stat = path.lstat()
if stat.S_ISLNK(path_stat.st_mode) or not stat.S_ISDIR(path_stat.st_mode):
die(f"Engine sealed n8n release parent is unsafe: {path}")
os.chown(path, 0, 0)
path.chmod(0o755)
release_dir = release_parent / descriptor["releaseId"]
if release_dir.exists() or release_dir.is_symlink():
validate_engine_n8n_sealed_release(release_dir, staged_release_dir, descriptor)
return release_dir
next_dir = release_parent / f".{descriptor['releaseId']}.next-{current_stamp}"
if next_dir.exists() or next_dir.is_symlink():
die(f"Engine sealed n8n release staging path already exists: {next_dir}")
next_dir.mkdir(mode=0o700)
for filename in ("package.tgz", "release.json", "rollback.json"):
shutil.copy2(staged_release_dir / filename, next_dir / filename)
with tarfile.open(staged_release_dir / "package.tgz", "r:gz") as archive:
for member in archive:
relative = PurePosixPath(member.name)
target = next_dir.joinpath(*relative.parts)
if member.isdir():
target.mkdir(parents=True, exist_ok=True)
continue
target.parent.mkdir(parents=True, exist_ok=True)
source = archive.extractfile(member)
if source is None:
die(f"Engine sealed n8n package member unreadable: {member.name}")
with target.open("xb") as output:
shutil.copyfileobj(source, output)
for path in [next_dir, *next_dir.rglob("*")]:
path_stat = path.lstat()
if stat.S_ISLNK(path_stat.st_mode):
die(f"Engine sealed n8n release symlink rejected: {path}")
os.chown(path, 0, 0)
if stat.S_ISDIR(path_stat.st_mode):
path.chmod(0o555)
elif stat.S_ISREG(path_stat.st_mode):
path.chmod(0o444)
else:
die(f"Engine sealed n8n release special member rejected: {path}")
if release_dir.exists() or release_dir.is_symlink():
die("Engine sealed n8n release appeared concurrently")
next_dir.rename(release_dir)
validate_engine_n8n_sealed_release(release_dir, staged_release_dir, descriptor)
return release_dir
def wait_engine_n8n_readiness(container_id):
script = (
"const http=require('http');const req=http.get('http://127.0.0.1:5678/healthz/readiness',"
"r=>{r.resume();process.exit(r.statusCode===200?0:1)});req.on('error',()=>process.exit(1));"
"req.setTimeout(4000,()=>{req.destroy();process.exit(1)});"
)
for _attempt in range(1, 61):
result = subprocess.run(
[str(DOCKER), "exec", container_id, "node", "-e", script],
check=False,
capture_output=True,
text=True,
timeout=10,
)
if result.returncode == 0:
return
time.sleep(5)
die("Engine n8n readiness check failed")
def accept_engine_n8n_runtime(descriptor):
base_image = inspect_engine_n8n_base_image()
container_id = engine_n8n_container_ids()[0]
container = inspect_container(container_id)
state = container.get("State") or {}
if state.get("Status") != "running" or container.get("Image") != base_image["id"]:
die("Engine n8n runtime image/state acceptance failed")
wait_engine_n8n_readiness(container_id)
engine_n8n_version(container_id)
config = container.get("Config") or {}
env = set(config.get("Env") or ())
mounts = container.get("Mounts") or ()
package_mounts = [mount for mount in mounts if mount.get("Destination") == ENGINE_N8N_RUNTIME_PACKAGE_PATH]
if descriptor["action"] == "activate":
expected_source = f"/volume2/nodedc-demo/{descriptor['sealedReleaseRelativePath']}"
if len(package_mounts) != 1:
die("Engine n8n package mount count mismatch")
package_mount = package_mounts[0]
if package_mount.get("Source") != expected_source or package_mount.get("RW") is not False:
die("Engine n8n sealed package mount mismatch")
required_env = {
"N8N_USER_FOLDER=/home/node",
"N8N_COMMUNITY_PACKAGES_ENABLED=true",
"N8N_COMMUNITY_PACKAGES_PREVENT_LOADING=false",
"N8N_REINSTALL_MISSING_PACKAGES=false",
}
if not required_env.issubset(env):
die("Engine n8n package loader environment mismatch")
labels = config.get("Labels") or {}
if labels.get("nodedc.n8n-private-extension.release") != descriptor["releaseId"]:
die("Engine n8n release label mismatch")
if labels.get("nodedc.n8n-private-extension.package-sha256") != descriptor["packageSha256"]:
die("Engine n8n package label mismatch")
else:
if package_mounts:
die("Engine n8n inactive baseline still has the private package mount")
inactive_forbidden = (
"N8N_USER_FOLDER=",
"N8N_COMMUNITY_PACKAGES_ENABLED=",
"N8N_COMMUNITY_PACKAGES_PREVENT_LOADING=",
"N8N_REINSTALL_MISSING_PACKAGES=",
)
if any(item.startswith(inactive_forbidden) for item in env):
die("Engine n8n inactive baseline still has private package loader flags")
if any(item.startswith("N8N_CUSTOM_EXTENSIONS=") or item.startswith("CUSTOM.") for item in env):
die("Engine n8n custom extension environment is forbidden")
started_at = str(state.get("StartedAt") or "")
logs = subprocess.run(
[str(DOCKER), "logs", "--since", started_at, container_id],
check=False,
capture_output=True,
text=True,
timeout=60,
)
if logs.returncode != 0:
die("Engine n8n loader log inspection failed")
scoped_logs = f"{logs.stdout}\n{logs.stderr}".lower()
error_patterns = (
r"n8n-nodes-ndc.{0,400}(?:cannot find|error|failed|duplicate|eacces|syntaxerror)",
r"(?:cannot find|error loading|failed to load|duplicate|eacces|syntaxerror).{0,400}n8n-nodes-ndc",
)
if any(re.search(pattern, scoped_logs, re.DOTALL) for pattern in error_patterns):
die("Engine n8n private extension loader log acceptance failed")
expected_types = list(ENGINE_N8N_NODE_TYPES) if descriptor["action"] == "activate" else []
expected_credentials = list(ENGINE_N8N_CREDENTIAL_TYPES) if descriptor["action"] == "activate" else []
loader_catalog = engine_n8n_private_loader_catalog(container_id)
if loader_catalog["node_types"] != expected_types:
die("Engine n8n live package-loader node catalog acceptance failed")
if loader_catalog["credential_types"] != expected_credentials:
die("Engine n8n live package-loader credential catalog acceptance failed")
validate_engine_n8n_catalog_payload(component_root("engine"), descriptor)
first_restart_count = int(container.get("RestartCount") or 0)
time.sleep(2)
stable_container = inspect_container(container_id)
if stable_container.get("State", {}).get("Status") != "running":
die("Engine n8n container did not remain running")
if int(stable_container.get("RestartCount") or 0) != first_restart_count:
die("Engine n8n container restarted during acceptance")
return {
"container_id": container_id,
"image_id": base_image["id"],
"node_types": expected_types,
"credential_types": list(descriptor["expectedCredentialTypes"]),
}
def plan_artifact(artifact):
validate_artifact_location(artifact)
ensure_layout()
sha = sha256_file(artifact)
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
transition_descriptor = None
transition_preflight = None
if is_engine_n8n_transition(manifest["component"], entries):
transition_descriptor = read_engine_n8n_transition_descriptor(
payload_dir / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL
)
transition_preflight = preflight_engine_n8n_transition(
transition_descriptor,
enforce_expected_current=False,
)
component = manifest["component"]
root = component_root(component)
compose_root = component_compose_root(component)
services = component_services(component, entries)
builds = component_builds(component, entries)
print("== plan ==")
print(f"artifact={artifact.name}")
print(f"sha256={sha}")
print(f"id={manifest['id']}")
print(f"component={component}")
print(f"type={manifest['type']}")
print(f"payload_root={root}")
print(f"compose_root={compose_root}")
compose_project = component_compose_project(component)
if compose_project:
print(f"compose_project={compose_project}")
for build_root, build_args in builds:
print(f"build_root={build_root}")
print(f"build={' '.join((str(DOCKER),) + tuple(build_args))}")
print(f"services={' '.join(services)}")
if transition_descriptor:
print(f"n8n_transition={transition_descriptor['action']}")
print(f"n8n_version={transition_descriptor['n8nVersion']}")
print(f"n8n_release={transition_descriptor['releaseId']}")
print(f"n8n_package_sha256={transition_descriptor['packageSha256']}")
print(f"n8n_base_image={transition_descriptor['baseImage']}")
print(f"n8n_base_image_id={transition_preflight['base_image_id']}")
print(f"n8n_base_image_architecture={transition_preflight['base_image_architecture']}")
for repo_digest in transition_preflight["base_image_repo_digests"]:
print(f"n8n_base_image_repo_digest={repo_digest}")
print(f"n8n_current_state={transition_preflight['current_state']}")
print(f"n8n_expected_current={transition_descriptor['expectedCurrent']}")
print(f"n8n_state_matches={'yes' if transition_preflight['state_matches'] else 'not-yet'}")
print(f"n8n_sealed_package=/volume2/nodedc-demo/{transition_descriptor['sealedReleaseRelativePath']}")
print("n8n_build=none")
print("n8n_pull=never")
print("n8n_force_recreate=yes")
print("n8n_persistent_data=preserved:/volume2/nodedc-demo/n8n-data")
print("n8n_database=preserved:n8n-postgres")
print(f"n8n_expected_node_types={','.join(transition_descriptor['expectedNodeTypes'])}")
print(f"n8n_expected_credential_types={','.join(transition_descriptor['expectedCredentialTypes'])}")
print("n8n_rollback_baseline=verified_inactive")
touches_map_gateway = component == "platform" and any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries)
touches_external_data_plane = component == "platform" and any(
rel == "platform/services/external-data-plane"
or rel.startswith("platform/services/external-data-plane/")
or rel == "platform/packages/external-provider-contract"
or rel.startswith("platform/packages/external-provider-contract/")
or rel == "platform/docker-compose.external-data-plane.yml"
for rel in entries
)
if component == "module-foundry" or touches_map_gateway:
print(f"runtime_secret=runner-managed:{MAP_GATEWAY_SECRET_FILE}")
if component == "module-foundry":
print(f"runtime_grants=runner-managed:{EXTERNAL_DATA_PLANE_READER_GRANTS_DIR}")
print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
if touches_external_data_plane:
print(f"runtime_secret=runner-managed:{EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
print(f"runtime_grants=runner-managed:{EXTERNAL_DATA_PLANE_READER_GRANTS_DIR}")
if component in ("proxy-contur", "dc-amd-proxy") or touches_map_gateway:
print(f"runtime_secret=runner-synced:{MAP_EGRESS_PROXY_SECRET_FILE}")
if component == "dc-amd-proxy":
print(f"runtime_state=runner-prepared:{DC_AMD_PROXY_RUNTIME_DIR}")
if state_has_sha(sha):
print("state=sha-already-applied")
elif state_has_patch_id(manifest["id"]):
print("state=patch-id-already-applied")
else:
print("state=new")
print("== files ==")
for rel in entries:
print(f" {rel}")
def add_backup_path(tar, root, rel):
src = root / rel
if src.exists():
tar.add(src, arcname=rel, recursive=True)
return True
return False
def create_backup(root, backup_dir, entries, include_nginx_html):
existing = []
missing = []
with tarfile.open(backup_dir / "source-before.tgz", "w:gz") as tar:
for rel in entries:
if add_backup_path(tar, root, rel):
existing.append(rel)
else:
missing.append(rel)
if include_nginx_html and (root / "nginx-html").exists():
tar.add(root / "nginx-html", arcname="nginx-html", recursive=True)
existing.append("nginx-html")
(backup_dir / "existing-files.txt").write_text("\n".join(existing) + ("\n" if existing else ""), encoding="utf-8")
(backup_dir / "missing-files.txt").write_text("\n".join(missing) + ("\n" if missing else ""), encoding="utf-8")
def read_backup_path_list(path):
if not path.is_file():
die(f"deploy backup path list missing: {path}")
return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
def inactive_engine_n8n_descriptor(descriptor):
inactive = dict(descriptor)
inactive["action"] = "rollback-inactive"
inactive["expectedCurrent"] = descriptor["releaseId"]
inactive["expectedNodeTypes"] = []
inactive["expectedCredentialTypes"] = []
return inactive
def restore_engine_n8n_transition(root, backup_dir, entries, descriptor, current_stamp):
existing = set(read_backup_path_list(backup_dir / "existing-files.txt"))
missing = set(read_backup_path_list(backup_dir / "missing-files.txt"))
if existing | missing != set(entries):
die("Engine n8n rollback backup entry set mismatch")
required_baseline = {
ENGINE_N8N_NODES_CATALOG_REL,
ENGINE_N8N_CREDENTIALS_CATALOG_REL,
ENGINE_N8N_SCHEMA_META_REL,
}
if not required_baseline.issubset(existing):
die("Engine n8n rollback baseline catalogs were not present before apply")
with tempfile.TemporaryDirectory(prefix="engine-n8n-rollback-", dir=TMP_DIR) as tmp:
restore_root = Path(tmp)
with tarfile.open(backup_dir / "source-before.tgz", "r:gz") as archive:
for rel in entries:
if rel not in existing:
continue
try:
member = archive.getmember(rel)
except KeyError:
die(f"Engine n8n rollback backup member missing: {rel}")
if not member.isfile():
die(f"Engine n8n rollback backup member is not a file: {rel}")
source = archive.extractfile(member)
if source is None:
die(f"Engine n8n rollback backup member unreadable: {rel}")
staged = restore_root / rel
staged.parent.mkdir(parents=True, exist_ok=True)
with staged.open("xb") as output:
shutil.copyfileobj(source, output)
replace_file(staged, root / rel, f"{current_stamp}-rollback")
if ENGINE_N8N_TRANSITION_DESCRIPTOR_REL not in existing:
inactive = inactive_engine_n8n_descriptor(descriptor)
descriptor_path = root / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL
descriptor_path.parent.mkdir(parents=True, exist_ok=True)
next_path = descriptor_path.with_name(f"{descriptor_path.name}.next-{current_stamp}-rollback")
if next_path.exists() or next_path.is_symlink():
die(f"Engine n8n rollback descriptor staging path exists: {next_path}")
next_path.write_text(json.dumps(inactive, indent=2) + "\n", encoding="utf-8")
os.replace(next_path, descriptor_path)
restored_descriptor = current_engine_n8n_transition_descriptor()
if restored_descriptor is None:
die("Engine n8n rollback did not restore an explicit baseline descriptor")
run_compose("engine", ("n8n",), entries)
accept_engine_n8n_runtime(restored_descriptor)
return restored_descriptor["action"]
def replace_file(src, dst, current_stamp):
dst.parent.mkdir(parents=True, exist_ok=True)
tmp = dst.with_name(f"{dst.name}.next-{current_stamp}")
if tmp.exists():
die(f"staging path already exists: {tmp}")
shutil.copy2(src, tmp)
os.replace(tmp, dst)
def replace_directory(src, dst, current_stamp):
dst.parent.mkdir(parents=True, exist_ok=True)
next_path = dst.with_name(f"{dst.name}.next-{current_stamp}")
prev_path = dst.with_name(f"{dst.name}.prev-{current_stamp}")
if next_path.exists() or prev_path.exists():
die(f"staging path already exists near: {dst}")
shutil.copytree(src, next_path)
if dst.exists():
dst.rename(prev_path)
next_path.rename(dst)
def copy_payload_path(payload_dir, root, rel, current_stamp):
src = payload_dir / rel
dst = root / rel
root_real = root.resolve()
dst_real = dst.resolve(strict=False)
if not is_relative_to(dst_real, root_real):
die(f"target escaped component root: {rel}")
if src.is_dir():
replace_directory(src, dst, current_stamp)
elif src.is_file():
replace_file(src, dst, current_stamp)
else:
die(f"payload path is neither file nor directory: {rel}")
def seal_n8n_private_extension_release(root, rel):
release_dir = root / rel
if not release_dir.is_dir() or release_dir.is_symlink():
die("private extension release was not installed as a directory")
for path in [release_dir, *release_dir.rglob("*")]:
path_stat = path.lstat()
if stat.S_ISLNK(path_stat.st_mode):
die(f"private extension installed symlink rejected: {path}")
os.chown(path, 0, 0)
if stat.S_ISDIR(path_stat.st_mode):
path.chmod(0o555)
elif stat.S_ISREG(path_stat.st_mode):
path.chmod(0o444)
else:
die(f"private extension installed special file rejected: {path}")
def publish_engine_dist(root, current_stamp):
dist = root / "nodedc-source" / "dist"
if not dist.is_dir():
return
dst = root / "nginx-html"
next_path = root / f"nginx-html.next-{current_stamp}"
prev_path = root / f"nginx-html.prev-{current_stamp}"
if next_path.exists() or prev_path.exists():
die("nginx-html staging path already exists")
shutil.copytree(dist, next_path)
if dst.exists():
dst.rename(prev_path)
next_path.rename(dst)
def component_publish_dist(component, entries=None):
if not COMPONENTS[component].get("publish_dist"):
return False
if component == "engine":
return entries is not None and any(
rel == "nodedc-source/dist" or rel.startswith("nodedc-source/dist/")
for rel in entries
)
return True
def run_build(component, entries=None):
for build_root, build_args in component_builds(component, entries):
dockerfile = "Dockerfile"
for idx, arg in enumerate(build_args):
if arg == "-f" and idx + 1 < len(build_args):
dockerfile = build_args[idx + 1]
break
if not (build_root / dockerfile).is_file():
die(f"Dockerfile not found in build root: {build_root / dockerfile}")
cmd = [str(DOCKER), *build_args]
env = os.environ.copy()
if component == "tasker":
env["DOCKER_BUILDKIT"] = "0"
subprocess.run(cmd, cwd=str(build_root), env=env, check=True)
def compose_base_cmd(component):
cmd = [str(DOCKER), "compose"]
compose_project = component_compose_project(component)
if compose_project:
cmd.extend(["-p", compose_project])
env_file = component_compose_env_file(component)
if env_file:
cmd.extend(["--env-file", str(env_file)])
for compose_file in component_compose_files(component):
cmd.extend(["-f", str(compose_file)])
return cmd
def run_compose(component, services, entries=None):
compose_root = component_compose_root(component)
cmd = [*compose_base_cmd(component), "up", "-d", "--force-recreate"]
if is_engine_n8n_transition(component, entries):
cmd.extend(["--pull", "never"])
if COMPONENTS[component].get("compose_build"):
cmd.append("--build")
if component_compose_no_deps(component, entries):
cmd.append("--no-deps")
cmd.extend(services)
try:
subprocess.run(cmd, cwd=str(compose_root), check=True)
except subprocess.CalledProcessError:
subprocess.run(
[*compose_base_cmd(component), "logs", "--no-color", "--tail=180", *services],
cwd=str(compose_root),
check=False,
)
raise
subprocess.run([*compose_base_cmd(component), "ps"], cwd=str(compose_root), check=True)
def prepare_component_runtime(component, entries=None):
if is_engine_n8n_transition(component, entries):
descriptor = current_engine_n8n_transition_descriptor()
if descriptor is None:
die("installed Engine n8n transition descriptor is missing")
if descriptor["action"] == "activate":
ensure_engine_n8n_sealed_release(descriptor, stamp())
return
if component == "module-foundry":
ensure_map_gateway_admin_secret()
ensure_root_owned_grant_directory(
EXTERNAL_DATA_PLANE_READER_GRANTS_DIR,
"external data plane reader grants",
)
ensure_root_owned_grant_directory(
FOUNDRY_BINDING_GRANTS_DIR,
"foundry binding grants",
)
return
if component == "proxy-contur":
sync_map_egress_proxy_secret()
return
if component == "dc-amd-proxy":
# The connector access value arrives only through the one-time pairing
# endpoint. The artifact must never carry it, and the service user is
# the only non-root identity that can write this directory.
sync_map_egress_proxy_secret()
try:
runtime_stat = DC_AMD_PROXY_RUNTIME_DIR.lstat()
except FileNotFoundError:
DC_AMD_PROXY_RUNTIME_DIR.mkdir(parents=True, exist_ok=False)
runtime_stat = DC_AMD_PROXY_RUNTIME_DIR.lstat()
if stat.S_ISLNK(runtime_stat.st_mode) or not stat.S_ISDIR(runtime_stat.st_mode):
die(f"dc-amd-proxy runtime directory is unsafe: {DC_AMD_PROXY_RUNTIME_DIR}")
os.chown(DC_AMD_PROXY_RUNTIME_DIR, MAP_GATEWAY_RUNTIME_GID, MAP_GATEWAY_RUNTIME_GID)
DC_AMD_PROXY_RUNTIME_DIR.chmod(0o700)
connector_state = DC_AMD_PROXY_RUNTIME_DIR / "connector-access"
if connector_state.exists() or connector_state.is_symlink():
connector_stat = connector_state.lstat()
if (
stat.S_ISLNK(connector_stat.st_mode)
or not stat.S_ISREG(connector_stat.st_mode)
or connector_stat.st_uid != MAP_GATEWAY_RUNTIME_GID
or connector_stat.st_mode & (stat.S_IRWXG | stat.S_IRWXO)
or connector_stat.st_size > 512
):
die("dc-amd-proxy connector pair state is unsafe")
return
if component == "platform" and entries is not None:
touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries)
touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries)
if touches_map_gateway:
# These NAS directories are the only canonical persistent stores
# for all Map Page instances. Creation lives in the root-owned
# runner, never in a user-supplied artifact or a Compose side effect.
cache_root = Path("/volume1/docker/nodedc-platform/map-gateway")
for cache_dir in (cache_root / "live-tile-cache", cache_root / "offline-snapshot"):
cache_dir.mkdir(parents=True, exist_ok=True)
os.chown(cache_dir, 1000, 1000)
os.chmod(cache_dir, 0o750)
ensure_map_gateway_admin_secret()
sync_map_egress_proxy_secret()
if touches_external_data_plane:
ensure_external_data_plane_provisioner_secret()
ensure_root_owned_grant_directory(
EXTERNAL_DATA_PLANE_READER_GRANTS_DIR,
"external data plane reader grants",
)
return
if component in ("dc-cms", "dc-cms-site-nodedc"):
(Path("/volume1/docker/dc-cms/sites") / "nodedc").mkdir(parents=True, exist_ok=True)
return
if component != "bim-viewer":
return
data_root = component_root(component) / "server" / "data"
for rel in ("uploads", "projects", "shares", "models", "comments"):
(data_root / rel).mkdir(parents=True, exist_ok=True)
def run_component_runtime(component, entries, services):
if component_artifact_only(component):
return
run_build(component, entries)
prepare_component_runtime(component, entries)
run_compose(component, services, entries)
def healthcheck_url(check):
headers = {}
if isinstance(check, dict):
url = check.get("url")
headers = check.get("headers") or {}
else:
url = check
last_error = None
for _attempt in range(1, 61):
try:
request = urllib.request.Request(url, headers=headers)
with NO_REDIRECT_OPENER.open(request, timeout=10) as response:
if 200 <= response.status < 400:
return
last_error = f"HTTP {response.status}"
except urllib.error.HTTPError as exc:
if 200 <= exc.code < 400:
return
last_error = f"HTTP {exc.code}"
except Exception as exc:
last_error = str(exc)
time.sleep(5)
die(f"healthcheck failed for {url}: {last_error}")
def component_healthchecks(component, entries=None):
if is_engine_n8n_transition(component, entries):
# The transition does not restart the Engine UI/backend generation.
# Its own acceptance below verifies n8n readiness, image, mount, logs,
# live node export and the pinned Engine MCP catalogs.
return ()
checks = list(COMPONENTS[component].get("healthchecks", ()))
if component == "platform" and entries is not None:
touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries)
touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries)
touches_ontology = any(rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/") for rel in entries)
touches_gelios = any(rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/") for rel in entries)
touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries)
touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries)
map_gateway_only_compose = touches_compose and touches_map_gateway and not any((touches_ai_workspace_assistant, touches_ontology, touches_gelios, touches_external_data_plane))
healthcheck_all_for_compose = touches_compose and not map_gateway_only_compose
if healthcheck_all_for_compose or touches_ai_workspace_assistant:
checks.append("http://127.0.0.1:18082/healthz")
if healthcheck_all_for_compose or touches_ontology:
checks.append("http://127.0.0.1:18104/healthz")
if healthcheck_all_for_compose or touches_gelios:
checks.append("http://127.0.0.1:18105/healthz")
if touches_map_gateway:
checks.append({"url": "http://127.0.0.1:18103/healthz", "headers": {"x-nodedc-user-id": "deploy-healthcheck"}})
if touches_external_data_plane:
checks.append("http://127.0.0.1:18106/healthz")
return tuple(checks)
def healthcheck_container(container_name):
last_status = "unknown"
for _attempt in range(1, 61):
result = subprocess.run(
[str(DOCKER), "inspect", "--format", "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", container_name],
check=False,
capture_output=True,
text=True,
)
status_text = result.stdout.strip().lower()
last_status = status_text or result.stderr.strip() or f"inspect-exit-{result.returncode}"
if status_text in ("healthy", "running"):
return
if status_text in ("unhealthy", "exited", "dead"):
break
time.sleep(5)
die(f"container healthcheck failed for {container_name}: {last_status}")
def run_healthchecks(component, entries=None):
if is_engine_n8n_transition(component, entries):
descriptor = current_engine_n8n_transition_descriptor()
if descriptor is None:
die("installed Engine n8n transition descriptor is missing during acceptance")
accept_engine_n8n_runtime(descriptor)
return
for url in component_healthchecks(component, entries):
healthcheck_url(url)
container_name = COMPONENTS[component].get("health_container")
if container_name:
healthcheck_container(container_name)
def move_artifact(src, target_dir, suffix=""):
target_dir.mkdir(parents=True, exist_ok=True)
name = src.name + suffix
dst = target_dir / name
if dst.exists():
dst = target_dir / f"{src.name}.{stamp()}"
src.rename(dst)
return dst
def apply_artifact(artifact):
require_root()
validate_artifact_location(artifact)
ensure_layout()
sha = sha256_file(artifact)
backup_id = None
backup_dir = None
manifest = None
entries = None
component = None
root = None
transition_descriptor = None
apply_started = False
current_stamp = stamp()
with DeployLock():
try:
with tempfile.TemporaryDirectory(prefix=f"apply-{current_stamp}-", dir=TMP_DIR) as tmp:
work = Path(tmp)
manifest, entries, payload_dir = load_artifact(artifact, work)
patch_id = manifest["id"]
component = manifest["component"]
root = component_root(component)
compose_root = component_compose_root(component)
bootstrap_root = bool(COMPONENTS[component].get("bootstrap_root"))
services = component_services(component, entries)
artifact_only = component_artifact_only(component)
if state_has_sha(sha):
die(f"artifact sha already applied: {sha}")
if state_has_patch_id(patch_id):
die(f"patch id already applied: {patch_id}")
if not root.is_dir():
if bootstrap_root:
root.mkdir(parents=True, exist_ok=True)
else:
die(f"component payload root not found: {root}")
if not artifact_only and not compose_root.is_dir():
if bootstrap_root and is_relative_to(compose_root.resolve(strict=False), root.resolve(strict=False)):
compose_root.mkdir(parents=True, exist_ok=True)
else:
die(f"component compose root not found: {compose_root}")
compose_files = component_compose_files(component)
if not artifact_only and compose_files:
for compose_file in compose_files:
if not compose_file.is_file():
compose_entry = None
if is_relative_to(compose_file.resolve(strict=False), root.resolve(strict=False)):
compose_entry = compose_file.resolve(strict=False).relative_to(root.resolve(strict=False)).as_posix()
if compose_entry not in entries:
die(f"compose file not found: {compose_file}")
elif not artifact_only and not (compose_root / "docker-compose.yml").is_file():
# A bootstrap component begins with no payload root. Its
# compose file may therefore be supplied by the same
# reviewed overlay that is about to create that root.
compose_entry = None
if is_relative_to(compose_root.resolve(strict=False), root.resolve(strict=False)):
compose_entry = compose_root.resolve(strict=False).relative_to(root.resolve(strict=False)).joinpath("docker-compose.yml").as_posix()
if not (bootstrap_root and compose_entry in entries):
die(f"docker-compose.yml not found in component compose root: {compose_root}")
compose_env_file = component_compose_env_file(component)
if not artifact_only and compose_env_file and not compose_env_file.is_file():
die(f"compose env file not found: {compose_env_file}")
if not artifact_only and not DOCKER.is_file():
die(f"docker not found: {DOCKER}")
if is_engine_n8n_transition(component, entries):
transition_descriptor = read_engine_n8n_transition_descriptor(
payload_dir / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL
)
preflight_engine_n8n_transition(
transition_descriptor,
enforce_expected_current=True,
)
if COMPONENTS[component].get("immutable_payload"):
for rel in entries:
destination = root / rel
if destination.exists() or destination.is_symlink():
die(f"immutable release already exists: {rel}")
backup_id = safe_name(f"{component}-{patch_id}-{current_stamp}")
backup_dir = BACKUPS_DIR / backup_id
backup_dir.mkdir(parents=True, exist_ok=False)
(backup_dir / "manifest.env").write_text((work / "manifest.env").read_text(encoding="utf-8"), encoding="utf-8")
(backup_dir / "files.txt").write_text((work / "files.txt").read_text(encoding="utf-8"), encoding="utf-8")
include_nginx_html = component == "engine" and component_publish_dist(component, entries)
create_backup(root, backup_dir, entries, include_nginx_html)
apply_started = True
for rel in entries:
copy_payload_path(payload_dir, root, rel, current_stamp)
if component == "n8n-private-extension":
for rel in entries:
seal_n8n_private_extension_release(root, rel)
if component_publish_dist(component, entries):
publish_engine_dist(root, current_stamp)
run_component_runtime(component, entries, services)
run_healthchecks(component, entries)
applied_path = move_artifact(artifact, APPLIED_DIR)
append_jsonl(STATE_FILE, {
"applied_at": utc_now(),
"artifact": applied_path.name,
"backup_id": backup_id,
"component": component,
"id": patch_id,
"sha256": sha,
"status": "ok",
})
print(f"deploy-ok patch={patch_id} component={component} backup={backup_id}")
except Exception as exc:
rollback_status = "not-required"
if (
apply_started
and backup_dir is not None
and root is not None
and transition_descriptor is not None
and is_engine_n8n_transition(component, entries)
):
try:
restored_action = restore_engine_n8n_transition(
root,
backup_dir,
entries,
transition_descriptor,
current_stamp,
)
rollback_status = f"ok:{restored_action}"
print(f"engine-n8n-automatic-rollback={rollback_status}", file=sys.stderr)
except Exception as rollback_exc:
rollback_status = f"failed:{type(rollback_exc).__name__}"
print("engine-n8n-automatic-rollback=failed", file=sys.stderr)
failed_path = None
if artifact.exists():
try:
failed_path = move_artifact(artifact, FAILED_DIR, suffix=f".{current_stamp}")
except Exception:
failed_path = None
append_jsonl(FAILED_STATE_FILE, {
"artifact": str(failed_path.name if failed_path else artifact.name),
"backup_id": backup_id,
"component": manifest.get("component") if manifest else None,
"failed_at": utc_now(),
"id": manifest.get("id") if manifest else None,
"message": str(exc),
"rollback_status": rollback_status,
"sha256": sha,
"started_apply": apply_started,
"status": "failed",
})
if backup_id:
print(f"backup={backup_id}", file=sys.stderr)
raise
def main(argv):
parser = argparse.ArgumentParser(prog="nodedc-deploy")
sub = parser.add_subparsers(dest="cmd", required=True)
plan = sub.add_parser("plan")
plan.add_argument("artifact")
apply = sub.add_parser("apply")
apply.add_argument("artifact")
sub.add_parser("verify-install")
args = parser.parse_args(argv)
if args.cmd == "verify-install":
verify_install()
return 0
artifact = Path(args.artifact).absolute()
if args.cmd == "plan":
plan_artifact(artifact)
return 0
if args.cmd == "apply":
apply_artifact(artifact)
return 0
parser.print_help()
return 2
if __name__ == "__main__":
try:
raise SystemExit(main(sys.argv[1:]))
except DeployError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
raise SystemExit(1)
Reference in New Issue View Git Blame Copy Permalink