From 567f1550abc2117db0636b0166ff9a25ed0c1088 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 02:25:31 +0300 Subject: [PATCH] feat(deploy): register managed platform runtime components --- .gitignore | 3 + README.md | 2 + infra/.env.example | 62 + infra/README.md | 13 +- infra/authentik/bootstrap-dev.py | 13 +- infra/deploy-runner/README.md | 158 ++ .../build-dc-amd-proxy-artifact.mjs | 44 + ...-engine-n8n-private-extension-artifact.mjs | 344 +++ .../build-external-data-plane-artifact.mjs | 68 + .../build-gelios-data-plane-artifact.mjs | 109 + ...build-launcher-module-foundry-artifact.mjs | 85 + .../build-map-gateway-artifact.mjs | 49 + .../build-module-foundry-artifact.mjs | 85 + .../build-n8n-private-extension-artifact.mjs | 286 +++ infra/deploy-runner/nodedc-deploy | 1946 ++++++++++++++++- .../test_engine_n8n_private_extension.py | 285 +++ .../test_n8n_private_extension.py | 260 +++ infra/docker-compose.dev.yml | 95 + infra/scripts/check-local-test-system.sh | 3 + infra/synology/.env.synology.example | 78 + infra/synology/README.md | 20 + infra/synology/apply-current-runtime.sh | 18 +- infra/synology/backup-current.sh | 2 + infra/synology/deploy-current.sh | 35 +- .../docker-compose.external-data-plane.yml | 96 + .../synology/docker-compose.platform-http.yml | 166 ++ .../prepare-external-data-plane-env.sh | 39 + .../synology/prepare-gelios-all-units-env.sh | 81 + infra/synology/verify-current-runtime.sh | 8 + 29 files changed, 4413 insertions(+), 40 deletions(-) create mode 100644 infra/deploy-runner/build-dc-amd-proxy-artifact.mjs create mode 100644 infra/deploy-runner/build-engine-n8n-private-extension-artifact.mjs create mode 100644 infra/deploy-runner/build-external-data-plane-artifact.mjs create mode 100644 infra/deploy-runner/build-gelios-data-plane-artifact.mjs create mode 100644 infra/deploy-runner/build-launcher-module-foundry-artifact.mjs create mode 100644 infra/deploy-runner/build-map-gateway-artifact.mjs create mode 100644 infra/deploy-runner/build-module-foundry-artifact.mjs create mode 100644 infra/deploy-runner/build-n8n-private-extension-artifact.mjs create mode 100644 infra/deploy-runner/test_engine_n8n_private_extension.py create mode 100644 infra/deploy-runner/test_n8n_private_extension.py create mode 100644 infra/synology/docker-compose.external-data-plane.yml create mode 100755 infra/synology/prepare-external-data-plane-env.sh create mode 100755 infra/synology/prepare-gelios-all-units-env.sh diff --git a/.gitignore b/.gitignore index 6d140e2..f710a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ build/ .next/ coverage/ +# canonical deploy packages are transferred through the NAS inbox, not Git +infra/deploy-artifacts/ + # logs *.log logs/ diff --git a/README.md b/README.md index d1af247..5e5f4e9 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,5 @@ AI Workspace Assistant живёт в `services/ai-workspace-assistant` как о Ontology Core живёт в `services/ontology-core` как docs-first семантический слой для canonical entities, relations, aliases, guardrails, evidence, первого resolver MVP между OPS и ENGINE и policy MVP для NDC Core Assistant access. Он не владеет доменными БД HUB/OPS/ENGINE и не заменяет Launcher/HUB roles или OPS Gateway enforcement. Map Gateway живёт в `services/map-gateway` как общий platform boundary для provider credentials, безопасного asset endpoint exchange, tile/3D Tiles proxy и persistent offline TileCache. Runtime cache не является source artifact и хранится в named volume/object storage, а не в Git. + +Внешние бизнес-поставщики подключаются по `packages/external-provider-contract`: adapter конкретного API принадлежит изолированному NDC Agent L2 workflow, а Platform даёт один provider-neutral External Data Plane для raw retention, canonical facts, current/history projections и scoped read products. Connection instance несёт tenant scope, credential reference и collection profile, но не secret value. Provider/domain mapping, entity filtering и renderer logic не попадают в Platform Data Plane; writer capability привязана к immutable EDP binding и выдаётся только отдельному Engine provisioner, не shared internal bearer. Подробный канон — `docs/ADR_L2_OWNED_EXTERNAL_CONNECTORS.md`. diff --git a/infra/.env.example b/infra/.env.example index e57614f..1fc4efd 100644 --- a/infra/.env.example +++ b/infra/.env.example @@ -51,6 +51,35 @@ NODEDC_INTERNAL_ACCESS_TOKEN=change-me-generate-with-infra-scripts-init-dev-env COOKIE_DOMAIN=.local.nodedc COOKIE_SECURE=false +# External Data Plane — provider-neutral storage owned by the Platform. This +# password is a database credential only; never reuse NODEDC_INTERNAL_ACCESS_TOKEN. +EXTERNAL_DATA_PLANE_TIMESCALE_IMAGE=timescale/timescaledb-ha:pg16.14-ts2.28.2-all +EXTERNAL_DATA_PLANE_PG_DB=nodedc_data_plane +EXTERNAL_DATA_PLANE_PG_USER=nodedc_data_plane +EXTERNAL_DATA_PLANE_PG_PASS=change-me-generate-with-infra-scripts-init-dev-env +EXTERNAL_DATA_PLANE_HOST_BIND=127.0.0.1:18106 +EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE=10 +EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS=14 +EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES=5242880 +EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH=5000 +EXTERNAL_DATA_PLANE_MAX_ATTRIBUTES_BYTES_PER_FACT=65536 +EXTERNAL_DATA_PLANE_MAX_PATCH_OPERATIONS=500 +EXTERNAL_DATA_PLANE_MAX_PATCH_BYTES=262144 +EXTERNAL_DATA_PLANE_PATCH_RETENTION_MS=3600000 +EXTERNAL_DATA_PLANE_RECEIPT_RETENTION_MS=604800000 +EXTERNAL_DATA_PLANE_RETENTION_DELETE_LIMIT=10000 +EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS=20000 +EXTERNAL_DATA_PLANE_STREAM_POLL_MS=1000 +EXTERNAL_DATA_PLANE_MAX_READER_STREAMS=10 +EXTERNAL_DATA_PLANE_WRITER_BINDING_MAX_TTL_DAYS=90 +EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS=300 +EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS=3600000 +EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED=false +# The writer-provisioner secret is not an env value. The root-owned deploy +# runner keeps it in /volume1/docker/nodedc-platform/secrets/external-data-plane-provisioner/ +# and mounts it +# only into External Data Plane and the future dedicated Engine provisioner. + # notification core NOTIFICATION_PG_DB=nodedc_notifications NOTIFICATION_PG_USER=nodedc_notifications @@ -66,8 +95,15 @@ NODEDC_AI_WORKSPACE_ASSISTANT_URL=http://ai-workspace-assistant:18082 AI_WORKSPACE_OPS_ENTITLEMENT_URL=http://host.docker.internal:4100/api/internal/v1/ai-workspace/entitlements AI_WORKSPACE_OPS_ENTITLEMENT_TOKEN=replace-with-ops-agent-gateway-internal-token AI_WORKSPACE_OPS_ENTITLEMENT_REQUIRED=false +# Add Module Foundry only after its domain, Launcher handoff and Authentik group +# are verified. Preserve existing adapters when adding this JSON member: +# {"module-foundry":{"url":"https:///api/ai-workspace/entitlements","required":false}} +# The generic adapter reuses the existing NODE.DC internal server credential; +# never define a separate Foundry token for a browser or worker. +AI_WORKSPACE_ENTITLEMENT_ADAPTERS_JSON= AI_WORKSPACE_ASSISTANT_ACTION_RELAY_ENABLED=true AI_WORKSPACE_ASSISTANT_ACTION_RELAY_ID=local-dev +AI_WORKSPACE_ONTOLOGY_MCP_ENABLED=true # AI Workspace Hub for downloaded Codex workers. # Default local development may use the deployed AI Hub only as a relay for remote Codex workers. @@ -78,6 +114,32 @@ AI_WORKSPACE_HUB_HOST_BIND=127.0.0.1:18081 AI_WORKSPACE_HUB_PUBLIC_URL=wss://ai-hub.nodedc.ru/api/ai-workspace/hub AI_WORKSPACE_HUB_INTERNAL_URL=https://ai-hub.nodedc.ru AI_WORKSPACE_HUB_FALLBACK_URLS= +# The worker receives a per-run, pairing-bound read-only Ontology MCP URL through AI Hub. +# Do not put Ontology Core tokens or catalog paths in worker configuration. +AI_WORKSPACE_ONTOLOGY_MCP_PUBLIC_URL= +ONTOLOGY_CORE_HOST_BIND=127.0.0.1:18104 + +# Gelios Gateway — storage/read service. Provider credentials belong to the +# protected Engine Collector and are never configured in this service. +GELIOS_TIMESCALE_IMAGE=timescale/timescaledb-ha:pg16.14-ts2.28.2-all +GELIOS_PG_DB=nodedc_gelios +GELIOS_PG_USER=nodedc_gelios +GELIOS_PG_PASS=change-me-generate-with-infra-scripts-init-dev-env +# URL-encode reserved characters in GELIOS_PG_PASS when forming this URL. +GELIOS_DATABASE_URL=postgresql://nodedc_gelios:change-me-generate-with-infra-scripts-init-dev-env@gelios-postgres:5432/nodedc_gelios +GELIOS_GATEWAY_HOST_BIND=127.0.0.1:18105 +# Explicit tenant and connection identifiers are deployment configuration; +# do not encode a customer or pilot name in source defaults. +GELIOS_TENANT_ID=replace-with-tenant-id +GELIOS_CONNECTION_ID=gelios-connection-id +# `allowlist` accepts only GELIOS_ALLOWED_UNIT_IDS. `all` accepts every unit +# returned by this already-approved tenant + connection, including future units. +GELIOS_UNIT_SCOPE=allowlist +GELIOS_ALLOWED_UNIT_IDS= +GELIOS_INTAKE_ENABLED=false +GELIOS_RAW_RETENTION_DAYS=14 +# Presentation state only; it does not change provider collection cadence. +GELIOS_POSITION_STALE_AFTER_MS=300000 # map gateway — keep the actual ion token only in local/staging environment files or Docker secrets. # Never copy it into workflow metadata, Git, frontend code, or a runtime cache key. diff --git a/infra/README.md b/infra/README.md index fe16f66..2185fd0 100644 --- a/infra/README.md +++ b/infra/README.md @@ -78,18 +78,11 @@ Generated Authentik bootstrap credentials are stored only in `infra/.env`. ## Map Gateway и offline TileCache -`map-gateway` добавлен как общий платформенный сервис на `127.0.0.1:18103`. Его mutable live-cache монтируется как внешний named Docker volume `nodedc-platform_map-live-tile-cache`; offline snapshot — как `nodedc-platform_map-offline-snapshot`. Они не пишутся в `platform/`, не попадают в Git и не включаются в Docker image. +`map-gateway` добавлен как общий платформенный сервис на `127.0.0.1:18103`. На NAS его mutable live-cache лежит в `/volume1/docker/nodedc-platform/map-gateway/live-tile-cache`; read-only offline snapshot — рядом в `offline-snapshot`. Это host bind mounts, поэтому папки видны через SMB как `nodedc-platform/map-gateway/`, но не попадают в Git, Docker image или deployment artifact. -Оба тома намеренно объявлены `external`: `docker compose down -v` не должен уничтожать уже записанные тайлы. На чистой машине они создаются один раз до первого запуска: +Обе папки создаёт root-owned `nodedc-deploy` при первом Map Gateway artifact. Agent не создаёт их напрямую через SMB и не кладёт в artifact. `docker compose down -v` их не удаляет. Очистка допустима только отдельной явно согласованной root-операцией при остановленном Gateway. -```bash -docker volume create nodedc-platform_map-live-tile-cache -docker volume create nodedc-platform_map-offline-snapshot -``` - -Очистка кэша — только осознанной командой `docker volume rm nodedc-platform_map-live-tile-cache` при остановленном Gateway. - -Для реального ion terrain/buildings положите `CESIUM_ION_TOKEN` только в неотслеживаемый `infra/.env` или deployment secret. Studio получает asset-scoped endpoint token, а master token остаётся внутри Gateway. Детали API, cache modes и production access boundary описаны в `services/map-gateway/README.md`. +Для реального ion terrain/buildings положите `CESIUM_ION_TOKEN` только в неотслеживаемый `infra/.env` или deployment secret. Browser получает лишь публичный provider URL через same-origin Foundry proxy; Gateway добавляет asset credential только в исходящем private request. Детали API, cache modes и production access boundary описаны в `services/map-gateway/README.md`. 5. Bootstrap local Authentik groups and OIDC applications: diff --git a/infra/authentik/bootstrap-dev.py b/infra/authentik/bootstrap-dev.py index 2264354..97ee359 100644 --- a/infra/authentik/bootstrap-dev.py +++ b/infra/authentik/bootstrap-dev.py @@ -27,6 +27,13 @@ GROUP_SPECS = [ ("nodedc:taskmanager:admin", False), ("nodedc:taskmanager:user", False), ("nodedc:bim:access", False), + # Module Foundry roles travel through the existing Authentik `groups` + # claim and Launcher handoff. `access` remains a backward-compatible + # member role until every existing assignment is migrated. + ("nodedc:module-foundry:admin", False), + ("nodedc:module-foundry:user", False), + ("nodedc:module-foundry:blocked", False), + ("nodedc:module-foundry:access", False), ] APP_SPECS = [ @@ -121,7 +128,11 @@ def ensure_user_groups(groups): user.groups.add(authentik_admins) for name in groups: - user.groups.add(groups[name]) + # The bootstrap owner must remain capable of entering Foundry after + # the first provisioning run. Blocking is an explicit operator action, + # never a default membership of the bootstrap principal. + if name != "nodedc:module-foundry:blocked": + user.groups.add(groups[name]) return user diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md index 60e94b8..3a89d11 100644 --- a/infra/deploy-runner/README.md +++ b/infra/deploy-runner/README.md @@ -28,6 +28,164 @@ Supported components in this source: - `tasker` - `ops-agents` - `bim-viewer` +- `n8n-private-extension` +- `module-foundry` +- `proxy-contur` +- `dc-amd-proxy` + +`n8n-private-extension` is a staging-only trust boundary for reviewed offline +n8n private-node releases. Its artifact may contain exactly one digest-bound +`n8n-nodes-ndc` release with `package.tgz`, `release.json` and +`rollback.json`. The runner validates the inner npm tarball, rejects lifecycle +scripts and runtime dependencies, refuses to overwrite an existing release, +and seals the installed release root-owned/read-only under: + +```text +/volume1/docker/nodedc-platform/n8n-private-extensions/releases/n8n-nodes-ndc/- +``` + +This component has no Compose file, service, container mutation or activation +side effect. In particular, staging does **not** make the node visible to n8n. +Activation remains an Engine-owned change: mount the reviewed immutable release +at `/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc`, atomically switch between +verified releases, restart every n8n process, and accept only after MCP exposes +the package-qualified `n8n-nodes-ndc.*` schemas. The Platform runner cannot +cross that boundary and never runs `npm install` in a live container. + +Build a verified offline release artifact: + +```bash +node infra/deploy-runner/build-n8n-private-extension-artifact.mjs \ + n8n-nodes-ndc-release-YYYYMMDD-NNN +``` + +The builder is byte-reproducible and accepts exactly the three reviewed NDC +runtime types: + +- `n8n-nodes-ndc.ndcDataProductPublish` +- `n8n-nodes-ndc.ndcDataProductRead` +- `n8n-nodes-ndc.ndcFoundryBinding` + +Their three opaque capability credential schemas are +`ndcDataProductWriterApi`, `ndcDataProductReaderApi` and +`ndcFoundryBindingApi`. A node description containing `usableAsTool` is +rejected because n8n 2.3.2 would synthesize an additional `*Tool` runtime type +and violate the exact-three activation contract. Run the positive and negative +release-policy suite before publishing an artifact: + +```bash +PYTHONDONTWRITEBYTECODE=1 \ + python3 infra/deploy-runner/test_n8n_private_extension.py +``` + +Release/rollback manifests use schema v2. Before a first activation, the +Engine-owned activator must verify and record the current inactive state. That +`verified_inactive` state is an allowed rollback baseline when no previous +verified immutable release exists; later upgrades prefer the previous verified +release. Rollback never deletes or mutates a staged release. + +The historical `0.1.0` release remains immutable and must not be overwritten. +Release `0.1.1-994756958861518e` is retained as rejected/inactive: its three +node descriptions used `usableAsTool`, so n8n 2.3.2 exposed six NDC runtime +types instead of the required three. It must not be activated, overwritten or +deleted. + +The corrected candidate is package version `0.1.2`, built with patch id +`n8n-nodes-ndc-release-20260716-003`. It receives a new digest-bound release +directory and remains inert after staging; only a separately reviewed +Engine-owned activator may select it after exact MCP schema acceptance. + +The paired Engine activation is built by +`build-engine-n8n-private-extension-artifact.mjs`. It deliberately does not +copy the dirty Engine `docker-compose.yml` and does not build an image. Instead +it installs a narrowly scoped Compose override plus a strict transition +descriptor. On apply, the runner validates the staged release again, verifies +that the running n8n container and the NAS-local `2.3.2` tag resolve to the +same immutable image ID, and extracts the package into the root-owned, +read-only Engine release tree: + +```text +/volume2/nodedc-demo/n8n-private-extensions/releases/n8n-nodes-ndc/0.1.2-05e4b38b14b4a019/package +``` + +The override sets `N8N_USER_FOLDER=/home/node`, which is required because the +actual Engine service runs as root while the canonical community package path +is below `/home/node/.n8n`. It enables loading but disables reinstall, mounts +only the exact release read-only, and uses both Compose `pull_policy: never` +and `docker compose up --pull never`. No registry access, lifecycle script, +database `installed_packages` row or custom-extension loader is involved. + +The runner pins the exact Engine service topology observed in source and +rejects an added worker/webhook generation. Only the single actual `n8n` +service is force-recreated with `--no-deps`; the +Postgres service, `.n8n` data, encryption key and credentials remain intact. +The apply gate verifies readiness, the running image/version, sealed mount, +loader environment, package-loader node/credential sets, scoped loader logs, +restart stability and content-exact pinned Engine MCP catalogs. The runner pins +both the complete 434/385 inactive baseline and the reviewed 437/388 +activation catalogs, so a same-count substitution of any built-in schema is +rejected. Any gate failure after +mutation automatically restores the pre-apply catalogs/descriptor and +force-recreates the previous verified runtime. Staged, sealed and failed +releases are retained. The separate rollback artifact returns the first +activation to the verified inactive 434-node/385-credential catalog baseline. + +Run both policy suites before publishing the Engine pair: + +```bash +PYTHONDONTWRITEBYTECODE=1 \ + python3 infra/deploy-runner/test_n8n_private_extension.py +PYTHONDONTWRITEBYTECODE=1 \ + python3 infra/deploy-runner/test_engine_n8n_private_extension.py +``` + +For `platform` artifacts, the allowlist includes the versioned Ontology Core, +the legacy Gelios experiment and the provider-neutral External Data Plane +sources. An External Data Plane artifact builds only its image and starts +`external-data-plane-postgres` plus `external-data-plane`; it never contains a +provider credential, provider endpoint, collection schedule or command +transport. Database credentials remain root-owned live `.env.synology` +configuration and must not reuse `NODEDC_INTERNAL_ACCESS_TOKEN`. + +The External Data Plane writer-provisioner credential is different: on the +first relevant Platform apply, the root-owned runner creates +`/volume1/docker/nodedc-platform/secrets/external-data-plane-provisioner/token` +atomically in a dedicated UID/GID `11006` directory (directory `0500`, token +`0400`). It is never an `.env` value or an artifact member; Compose mounts it +read-only only into External Data Plane and, when implemented, its dedicated +Engine provisioner running under the same restricted identity. + +`module-foundry` is an independent, authenticated application component. Its +artifact contains source and compose infrastructure only; its live +`/volume1/docker/nodedc-platform/module-foundry/source/.env` is root-owned and +never enters an artifact. The component reuses the existing internal platform +credential for Launcher handoff validation and requires that runtime +configuration before its first `apply`. + +The Foundry ↔ Map Gateway signing key is not an application or `.env` setting. +On the first relevant `platform` or `module-foundry` apply, the root-owned +runner creates `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret` +atomically (root:gid 1000, mode `0640`). Both containers receive that file only +as a read-only mount. The value is never printed, backed up with source, +included in an artifact, or administered through Foundry. + +`proxy-contur` is the canonical VPN egress for selected Map Gateway provider +hosts. Its existing root-owned `PROXY_TOKEN` is copied by the runner into +`/volume1/docker/nodedc-platform/secrets/map-egress-proxy-token` with +`root:gid 1000`, mode `0640`, then mounted read-only only into Map Gateway. +The value is neither printed nor contained in an artifact, Foundry setting, or +browser response. Apply the `proxy-contur` artifact before the Platform Map +Gateway artifact: it creates the private `nodedc-map-egress` Docker network. + +`dc-amd-proxy` is the separate, staged connector for the neighbouring AMD VPN +machine. Its active artifact attaches only to the private `nodedc-map-egress` +network, exposes a narrow NAS-LAN pairing port, and has no direct provider +egress. The runner preserves a `0700`, service-user-owned runtime directory +for the one-time paired connector credential and synchronizes the existing +private Map Gateway egress credential as a read-only file. Neither value is +ever placed in an artifact, `.env`, browser response, or runner output. The +separate Platform switch is applied only after the connector and pairing are +verified; it does not alter NAS routes, VPN, DNS, or Tailscale. Install or update the root-owned live runner on Synology: diff --git a/infra/deploy-runner/build-dc-amd-proxy-artifact.mjs b/infra/deploy-runner/build-dc-amd-proxy-artifact.mjs new file mode 100644 index 0000000..6879dfd --- /dev/null +++ b/infra/deploy-runner/build-dc-amd-proxy-artifact.mjs @@ -0,0 +1,44 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, lstat, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const artifactDir = resolve(scriptDir, "../deploy-artifacts"); +const [patchId = "dc-amd-proxy-bootstrap-20260715-001", ...extra] = process.argv.slice(2); +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) throw new Error("usage: build-dc-amd-proxy-artifact.mjs [patch-id]"); + +const files = [ + ["services/dc-amd-proxy/Dockerfile", "Dockerfile"], + ["services/dc-amd-proxy/README.md", "README.md"], + ["services/dc-amd-proxy/docker-compose.yml", "docker-compose.yml"], + ["services/dc-amd-proxy/package.json", "package.json"], + ["services/dc-amd-proxy/server.mjs", "server.mjs"], +]; +const stage = await mkdtemp(join(tmpdir(), "nodedc-dc-amd-proxy-artifact-")); +const payload = join(stage, "payload"); +const target = join(artifactDir, `nodedc-${patchId}.tgz`); + +try { + await mkdir(payload, { recursive: true }); + for (const [source, destination] of files) await copySafe(resolve(platformRoot, source), join(payload, destination)); + await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=dc-amd-proxy\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${files.map(([, destination]) => destination).join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + const tar = spawnSync("python3", ["-c", "import sys,tarfile\nwith tarfile.open(sys.argv[1],'w:gz',format=tarfile.PAX_FORMAT) as a:\n [a.add(n,arcname=n,recursive=True) for n in ('manifest.env','files.txt','payload')]", target], { cwd: stage, encoding: "utf8" }); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + console.log(JSON.stringify({ ok: true, patchId, artifact: target, sha256: createHash("sha256").update(await readFile(target)).digest("hex") }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function copySafe(source, destination) { + const info = await lstat(source); + if (info.isSymbolicLink() || !info.isFile()) throw new Error(`source_file_rejected:${source}`); + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true }); +} diff --git a/infra/deploy-runner/build-engine-n8n-private-extension-artifact.mjs b/infra/deploy-runner/build-engine-n8n-private-extension-artifact.mjs new file mode 100644 index 0000000..be74dfa --- /dev/null +++ b/infra/deploy-runner/build-engine-n8n-private-extension-artifact.mjs @@ -0,0 +1,344 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createRequire, Module } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(here, "../.."); +const engineRoot = resolve(platformRoot, "../NODEDC_ENGINE_INFRA"); +const artifactRoot = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || join(here, "../deploy-artifacts")); +const stageArtifact = resolve( + process.env.NODEDC_N8N_EXTENSION_STAGE_ARTIFACT + || join(artifactRoot, "nodedc-n8n-private-extension-n8n-nodes-ndc-release-20260716-003.tgz"), +); + +const stageArtifactSha256 = "4601c16c57e5182996adf18d0163837511b27ab3f7cfdf97eac679418ee2d078"; +const releaseId = "0.1.2-05e4b38b14b4a019"; +const packageVersion = "0.1.2"; +const packageSha256 = "05e4b38b14b4a019ce1f6eee27b9e320094cb3560903bd68074966b3a1267af5"; +const n8nVersion = "2.3.2"; +const baseImage = "docker.n8n.io/n8nio/n8n:2.3.2"; +const architecture = "amd64"; +const generatedAt = "2026-07-15T21:51:41.000Z"; +const activationId = "engine-n8n-private-extension-20260716-003"; +const rollbackId = "engine-n8n-private-extension-rollback-20260716-003"; + +const transitionRoot = "nodedc-source/services/n8n/private-extensions"; +const descriptorRel = `${transitionRoot}/ndc-activation.json`; +const overrideRel = `${transitionRoot}/docker-compose.ndc-private-extension.yml`; +const schemaRoot = "nodedc-source/server/assets/n8n/schema/v2.3.2"; +const nodesCatalogRel = `${schemaRoot}/nodes.catalog.json`; +const credentialsCatalogRel = `${schemaRoot}/credentials.catalog.json`; +const metaRel = `${schemaRoot}/meta.json`; +const iconRoot = "nodedc-source/server/assets/n8n/icons"; +const iconRel = `${iconRoot}/ndc.svg`; +const darkIconRel = `${iconRoot}/ndc.dark.svg`; +const sealedReleaseRelativePath = `n8n-private-extensions/releases/n8n-nodes-ndc/${releaseId}/package`; +const runtimePackagePath = "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc"; + +const expectedNodeTypes = [ + "n8n-nodes-ndc.ndcDataProductPublish", + "n8n-nodes-ndc.ndcDataProductRead", + "n8n-nodes-ndc.ndcFoundryBinding", +]; +const expectedCredentialTypes = [ + "ndcDataProductWriterApi", + "ndcDataProductReaderApi", + "ndcFoundryBindingApi", +]; +const nodeModules = [ + ["dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", "NdcDataProductPublish"], + ["dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js", "NdcDataProductRead"], + ["dist/nodes/NdcFoundryBinding/NdcFoundryBinding.node.js", "NdcFoundryBinding"], +]; +const credentialModules = [ + ["dist/credentials/NdcDataProductWriterApi.credentials.js", "NdcDataProductWriterApi"], + ["dist/credentials/NdcDataProductReaderApi.credentials.js", "NdcDataProductReaderApi"], + ["dist/credentials/NdcFoundryBindingApi.credentials.js", "NdcFoundryBindingApi"], +]; + +await mkdir(artifactRoot, { recursive: true }); +assertSha(await readFile(stageArtifact), stageArtifactSha256, "staging artifact"); +assertEngineBaseline(await readFile(join(engineRoot, "docker-compose.yml"), "utf8")); + +const work = await mkdtemp(join(tmpdir(), "nodedc-engine-n8n-sealed-")); +try { + extractArchive(stageArtifact, work); + const stagedRelease = join(work, "payload", "releases", "n8n-nodes-ndc", releaseId); + const release = JSON.parse(await readFile(join(stagedRelease, "release.json"), "utf8")); + assertRelease(release); + assertSha(await readFile(join(stagedRelease, "package.tgz")), packageSha256, "private package"); + + const unpacked = join(work, "unpacked"); + await mkdir(unpacked); + extractArchive(join(stagedRelease, "package.tgz"), unpacked); + const packageRoot = join(unpacked, "package"); + const packageJson = JSON.parse(await readFile(join(packageRoot, "package.json"), "utf8")); + assertPackage(packageJson); + + const devNodeModules = join(platformRoot, "packages", "n8n-nodes-ndc", "node_modules"); + const nodePath = String(process.env.NODE_PATH || "").split(":").filter(Boolean); + if (!nodePath.includes(devNodeModules)) nodePath.unshift(devNodeModules); + process.env.NODE_PATH = nodePath.join(":"); + Module._initPaths(); + const packageRequire = createRequire(join(packageRoot, "package.json")); + const privateNodes = nodeModules.map(([path, className], index) => { + const NodeClass = packageRequire(join(packageRoot, path))[className]; + if (typeof NodeClass !== "function") throw new Error(`node_class_missing:${className}`); + const description = structuredClone(new NodeClass().description); + description.name = expectedNodeTypes[index]; + description.icon = { light: "file:ndc.svg", dark: "file:ndc.dark.svg" }; + if (Object.prototype.hasOwnProperty.call(description, "usableAsTool")) { + throw new Error(`tool_variant_forbidden:${description.name}`); + } + return description; + }); + const privateCredentials = credentialModules.map(([path, className]) => { + const CredentialClass = packageRequire(join(packageRoot, path))[className]; + if (typeof CredentialClass !== "function") throw new Error(`credential_class_missing:${className}`); + const description = structuredClone(new CredentialClass()); + description.icon = { light: "file:ndc.svg", dark: "file:ndc.dark.svg" }; + return description; + }); + assertExact(privateNodes.map((item) => item.name), expectedNodeTypes, "node types"); + assertExact(privateCredentials.map((item) => item.name), expectedCredentialTypes, "credential types"); + + const baselineNodes = JSON.parse(gitFile(nodesCatalogRel)); + const baselineCredentials = JSON.parse(gitFile(credentialsCatalogRel)); + const baselineMeta = JSON.parse(gitFile(metaRel)); + assertBaselineCatalogs(baselineNodes, baselineCredentials, baselineMeta); + const activeNodes = [...baselineNodes, ...privateNodes]; + const activeCredentials = [...baselineCredentials, ...privateCredentials]; + const activeMeta = { + n8nVersion, + generatedAt, + source: `n8n-core+n8n-nodes-ndc@${packageVersion}`, + nodeCount: activeNodes.length, + credentialCount: activeCredentials.length, + }; + + const activationDescriptor = descriptor("activate", expectedNodeTypes, expectedCredentialTypes, "verified_inactive"); + const rollbackDescriptor = descriptor("rollback-inactive", [], [], releaseId); + const override = composeOverride(); + + await writeJson(join(engineRoot, nodesCatalogRel), activeNodes); + await writeJson(join(engineRoot, credentialsCatalogRel), activeCredentials); + await writeJson(join(engineRoot, metaRel), activeMeta); + await writeJson(join(engineRoot, descriptorRel), activationDescriptor); + await writeFile(join(engineRoot, overrideRel), override, "utf8"); + await cp(join(packageRoot, "dist/icons/ndc.svg"), join(engineRoot, iconRel), { force: true }); + await cp(join(packageRoot, "dist/icons/ndc.dark.svg"), join(engineRoot, darkIconRel), { force: true }); + + const activationEntries = [ + descriptorRel, + overrideRel, + nodesCatalogRel, + credentialsCatalogRel, + metaRel, + iconRel, + darkIconRel, + ]; + const activationArtifact = await buildArtifact(work, activationId, activationEntries, async (payload) => { + for (const rel of activationEntries) { + await cp(join(engineRoot, rel), join(payload, rel), { recursive: true, force: false }); + } + }); + + const rollbackEntries = [descriptorRel, nodesCatalogRel, credentialsCatalogRel, metaRel]; + const rollbackArtifact = await buildArtifact(work, rollbackId, rollbackEntries, async (payload) => { + await writeJson(join(payload, descriptorRel), rollbackDescriptor); + await mkdir(join(payload, schemaRoot), { recursive: true }); + await writeFile(join(payload, nodesCatalogRel), `${JSON.stringify(baselineNodes, null, 2)}\n`, "utf8"); + await writeFile(join(payload, credentialsCatalogRel), `${JSON.stringify(baselineCredentials, null, 2)}\n`, "utf8"); + await writeFile(join(payload, metaRel), `${JSON.stringify(baselineMeta, null, 2)}\n`, "utf8"); + }); + + console.log(JSON.stringify({ + ok: true, + releaseId, + packageSha256, + nodeTypes: expectedNodeTypes, + credentialTypes: expectedCredentialTypes, + activation: activationArtifact, + rollback: rollbackArtifact, + }, null, 2)); +} finally { + await rm(work, { recursive: true, force: true }); +} + +function descriptor(action, nodeTypes, credentialTypes, expectedCurrent) { + return { + schemaVersion: "nodedc.engine-n8n-private-extension-transition/v1", + action, + releaseId, + packageVersion, + packageSha256, + n8nVersion, + baseImage, + baseImageArchitecture: architecture, + baseImageIdentityPolicy: "running-container-and-local-tag-must-match", + sealedReleaseRelativePath, + composeOverride: overrideRel, + runtimePackagePath, + topologyServices: ["n8n"], + expectedCurrent, + expectedNodeTypes: nodeTypes, + expectedCredentialTypes: credentialTypes, + rollbackBaseline: "verified_inactive", + }; +} + +function composeOverride() { + const health = "const http=require('http');const req=http.get('http://127.0.0.1:5678/healthz/readiness',r=>{r.resume();process.exit(r.statusCode===200?0:1)});req.on('error',()=>process.exit(1));req.setTimeout(4000,()=>{req.destroy();process.exit(1)});"; + return [ + "services:", + " n8n:", + ` image: ${baseImage}`, + " platform: linux/amd64", + " pull_policy: never", + " environment:", + " N8N_USER_FOLDER: /home/node", + " N8N_COMMUNITY_PACKAGES_ENABLED: \"true\"", + " N8N_COMMUNITY_PACKAGES_PREVENT_LOADING: \"false\"", + " N8N_REINSTALL_MISSING_PACKAGES: \"false\"", + " volumes:", + ` - /volume2/nodedc-demo/${sealedReleaseRelativePath}:${runtimePackagePath}:ro`, + " healthcheck:", + ` test: ${JSON.stringify(["CMD", "node", "-e", health])}`, + " interval: 10s", + " timeout: 5s", + " retries: 30", + " start_period: 30s", + " labels:", + ` nodedc.n8n-private-extension.release: ${releaseId}`, + ` nodedc.n8n-private-extension.package-sha256: ${packageSha256}`, + "", + ].join("\n"); +} + +async function buildArtifact(workRoot, id, entries, populate) { + const stage = join(workRoot, id); + const payload = join(stage, "payload"); + await mkdir(payload, { recursive: true }); + await populate(payload); + await writeFile(join(stage, "manifest.env"), `id=${id}\ncomponent=engine\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8"); + const artifact = join(artifactRoot, `nodedc-${id}.tgz`); + run("python3", ["-c", canonicalTarScript(), artifact, stage]); + return { id, artifact, sha256: sha(await readFile(artifact)), entries }; +} + +function assertRelease(value) { + if (value?.releaseId !== releaseId + || value?.package?.name !== "n8n-nodes-ndc" + || value?.package?.version !== packageVersion + || value?.package?.sha256 !== packageSha256 + || value?.storage?.relativePath !== `releases/n8n-nodes-ndc/${releaseId}`) { + throw new Error("staged_release_identity_mismatch"); + } +} + +function assertPackage(value) { + if (value.name !== "n8n-nodes-ndc" || value.version !== packageVersion || value.private !== true) { + throw new Error("package_identity_mismatch"); + } + if (value.dependencies !== undefined) throw new Error("runtime_dependencies_forbidden"); + for (const name of ["preinstall", "install", "postinstall", "prepare", "prepack", "postpack"]) { + if (value.scripts?.[name] !== undefined) throw new Error(`lifecycle_forbidden:${name}`); + } + assertExact(value.n8n?.nodes, nodeModules.map(([path]) => path), "package nodes"); + assertExact(value.n8n?.credentials, credentialModules.map(([path]) => path), "package credentials"); +} + +function assertBaselineCatalogs(nodes, credentials, meta) { + if (!Array.isArray(nodes) || nodes.length !== 434 || nodes.some((item) => String(item?.name || "").startsWith("n8n-nodes-ndc."))) { + throw new Error("baseline_node_catalog_mismatch"); + } + if (!Array.isArray(credentials) || credentials.length !== 385 + || credentials.some((item) => expectedCredentialTypes.includes(String(item?.name || "")))) { + throw new Error("baseline_credential_catalog_mismatch"); + } + if (meta?.n8nVersion !== n8nVersion || meta?.nodeCount !== 434 || meta?.credentialCount !== 385) { + throw new Error("baseline_meta_mismatch"); + } +} + +function assertEngineBaseline(compose) { + const exactImage = `image: docker.n8n.io/n8nio/n8n:\${N8N_IMAGE_TAG:-${n8nVersion}}`; + if (!compose.includes(exactImage)) throw new Error("engine_n8n_version_mismatch"); + if ((compose.match(/^ n8n:\s*$/gm) || []).length !== 1) throw new Error("engine_n8n_topology_mismatch"); + if (/^ n8n-(?:worker|webhook)|^ (?:worker|webhook):/gm.test(compose)) throw new Error("unexpected_n8n_process_service"); + if (compose.includes("N8N_CUSTOM_EXTENSIONS") || compose.includes("CUSTOM.")) throw new Error("custom_extension_loader_forbidden"); +} + +function assertExact(actual, expected, label) { + if (!Array.isArray(actual) || JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`${label.replaceAll(" ", "_")}_mismatch`); + } +} + +function assertSha(bytes, expected, label) { + const actual = sha(bytes); + if (actual !== expected) throw new Error(`${label.replaceAll(" ", "_")}_sha256_mismatch:${actual}`); +} + +function gitFile(rel) { + return run("git", ["show", `HEAD:${rel}`], engineRoot).stdout; +} + +async function writeJson(path, value) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function extractArchive(archive, destination) { + const script = [ + "import pathlib, sys, tarfile", + "src=pathlib.Path(sys.argv[1]); dst=pathlib.Path(sys.argv[2]).resolve()", + "with tarfile.open(src, 'r:gz') as tf:", + " for m in tf:", + " p=pathlib.PurePosixPath(m.name)", + " if p.is_absolute() or '..' in p.parts or any(x.startswith('._') for x in p.parts) or not (m.isfile() or m.isdir()): raise SystemExit('unsafe archive member')", + " target=dst.joinpath(*p.parts)", + " target.mkdir(parents=True, exist_ok=True) if m.isdir() else target.parent.mkdir(parents=True, exist_ok=True)", + " if m.isfile():", + " source=tf.extractfile(m)", + " with open(target, 'xb') as out: out.write(source.read())", + ].join("\n"); + run("python3", ["-c", script, archive, destination]); +} + +function canonicalTarScript() { + return [ + "import gzip, io, pathlib, sys, tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1], 'wb') as out:", + " with gzip.GzipFile(filename='', mode='wb', fileobj=out, compresslevel=9, mtime=0) as gz:", + " with tarfile.open(fileobj=gz, mode='w', format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p] + (sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x), arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info, src if info.isfile() else None)", + ].join("\n"); +} + +function sha(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function run(command, args, cwd) { + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) throw new Error(`${command}_failed:${result.stderr || result.stdout}`); + return result; +} diff --git a/infra/deploy-runner/build-external-data-plane-artifact.mjs b/infra/deploy-runner/build-external-data-plane-artifact.mjs new file mode 100644 index 0000000..55abd1a --- /dev/null +++ b/infra/deploy-runner/build-external-data-plane-artifact.mjs @@ -0,0 +1,68 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const artifactDir = resolve(scriptDir, "../deploy-artifacts"); +const [patchId = "external-data-plane-20260714-001", ...extra] = process.argv.slice(2); + +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error("usage: build-external-data-plane-artifact.mjs [patch-id]"); +} + +const files = [ + ["infra/synology/docker-compose.external-data-plane.yml", "platform/docker-compose.external-data-plane.yml"], + ["services/external-data-plane", "platform/services/external-data-plane"], + ["packages/external-provider-contract", "platform/packages/external-provider-contract"], +]; +const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); +const stage = await mkdtemp(join(tmpdir(), "nodedc-external-data-plane-artifact-")); +const payload = join(stage, "payload"); +const target = join(artifactDir, `nodedc-platform-${patchId}.tgz`); + +try { + await mkdir(payload, { recursive: true }); + for (const [sourceRelative, destinationRelative] of files) { + await copySafe(resolve(platformRoot, sourceRelative), join(payload, destinationRelative)); + } + await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=platform\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${files.map(([, destination]) => destination).join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + + const tar = spawnSync("python3", ["-c", [ + "import sys, tarfile", + "with tarfile.open(sys.argv[1], 'w:gz', format=tarfile.PAX_FORMAT) as archive:", + " [archive.add(name, arcname=name, recursive=True) for name in ('manifest.env', 'files.txt', 'payload')]", + ].join("\n"), target], { cwd: stage, encoding: "utf8" }); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + + const digest = createHash("sha256").update(await (await import("node:fs/promises")).readFile(target)).digest("hex"); + console.log(JSON.stringify({ ok: true, patchId, artifact: target, sha256: digest }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) throw new Error(`source_symlink_rejected:${source}`); + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, verbatimSymlinks: true }); + return; + } + if (!sourceStat.isDirectory()) throw new Error(`source_type_rejected:${source}`); + + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if (ignoredBasenames.has(entry.name) || entry.name.startsWith(".env")) continue; + const childSource = join(source, entry.name); + const childDestination = join(destination, entry.name); + if (entry.isSymbolicLink()) throw new Error(`source_symlink_rejected:${relative(platformRoot, childSource)}`); + await copySafe(childSource, childDestination); + } +} diff --git a/infra/deploy-runner/build-gelios-data-plane-artifact.mjs b/infra/deploy-runner/build-gelios-data-plane-artifact.mjs new file mode 100644 index 0000000..0566f6e --- /dev/null +++ b/infra/deploy-runner/build-gelios-data-plane-artifact.mjs @@ -0,0 +1,109 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const artifactDir = resolve(scriptDir, "../deploy-artifacts"); +const args = process.argv.slice(2); +const gatewayOnly = args.includes("--gateway-only"); +const positionalArgs = args.filter((argument) => argument !== "--gateway-only"); +if (positionalArgs.length > 1) { + throw new Error("usage: build-gelios-data-plane-artifact.mjs [patch-id] [--gateway-only]"); +} +const patchId = positionalArgs[0] || "gelios-data-plane-20260713-001"; + +if (!/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error("patch_id_must_contain_only_letters_digits_dot_underscore_hyphen"); +} + +const fullDataPlaneFiles = [ + ["infra/synology/docker-compose.platform-http.yml", "platform/docker-compose.platform-http.yml"], + ["services/ontology-core", "platform/ontology-core"], + ["services/ai-workspace-hub", "platform/ai-workspace-hub"], + ["services/ai-workspace-assistant", "platform/ai-workspace-assistant"], + ["services/gelios-gateway", "platform/gelios-gateway"], +]; +// A policy/code update to an already deployed Gelios data plane must not +// carry the common Platform compose file. The deploy runner treats that file +// as a whole-Platform change. The existing Gelios service already uses the +// live Platform env_file, so this overlay can safely recreate only Gelios. +const files = gatewayOnly + ? [["services/gelios-gateway", "platform/gelios-gateway"]] + : fullDataPlaneFiles; + +const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); +const stage = await mkdtemp(join(tmpdir(), "nodedc-gelios-artifact-")); +const payload = join(stage, "payload"); +const target = join(artifactDir, `nodedc-platform-${patchId}.tgz`); + +try { + await mkdir(payload, { recursive: true }); + + for (const [sourceRelative, destinationRelative] of files) { + const source = resolve(platformRoot, sourceRelative); + const destination = join(payload, destinationRelative); + await copySafe(source, destination); + } + + if (!gatewayOnly) { + // The Assistant imports the deterministic catalog as a local sibling at + // runtime. Its production Dockerfile therefore expects this directory + // inside the Assistant build context. Keep the deploy artifact equivalent + // to the canonical Synology staging layout without making a second source + // copy in the repository. + await copySafe( + resolve(platformRoot, "services/ontology-core"), + join(payload, "platform/ai-workspace-assistant/ontology-core"), + ); + // This nested copy is source-only build input for the Assistant. The nested + // service Dockerfile is neither used nor allowed by the production runner. + await rm(join(payload, "platform/ai-workspace-assistant/ontology-core/Dockerfile"), { force: true }); + } + + await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=platform\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${files.map(([, destination]) => destination).join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + + // macOS bsdtar includes AppleDouble sidecar files for extended attributes. + // The root runner rejects those as unexpected members, so use stdlib tarfile + // to generate a portable, data-only archive instead. + const tar = spawnSync("python3", ["-c", [ + "import sys, tarfile", + "with tarfile.open(sys.argv[1], 'w:gz', format=tarfile.PAX_FORMAT) as archive:", + " [archive.add(name, arcname=name, recursive=True) for name in ('manifest.env', 'files.txt', 'payload')]", + ].join("\n"), target], { + cwd: stage, + encoding: "utf8", + }); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + + const digest = createHash("sha256").update(await (await import("node:fs/promises")).readFile(target)).digest("hex"); + console.log(JSON.stringify({ ok: true, patchId, gatewayOnly, artifact: target, sha256: digest }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) throw new Error(`source_symlink_rejected:${source}`); + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, verbatimSymlinks: true }); + return; + } + if (!sourceStat.isDirectory()) throw new Error(`source_type_rejected:${source}`); + + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if (ignoredBasenames.has(entry.name) || entry.name.startsWith(".env")) continue; + const childSource = join(source, entry.name); + const childDestination = join(destination, entry.name); + if (entry.isSymbolicLink()) throw new Error(`source_symlink_rejected:${relative(platformRoot, childSource)}`); + await copySafe(childSource, childDestination); + } +} diff --git a/infra/deploy-runner/build-launcher-module-foundry-artifact.mjs b/infra/deploy-runner/build-launcher-module-foundry-artifact.mjs new file mode 100644 index 0000000..55ec2fd --- /dev/null +++ b/infra/deploy-runner/build-launcher-module-foundry-artifact.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, lstat, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const launcherRoot = resolve(platformRoot, "../../data/nodedc_launcher"); +const artifactDir = resolve(scriptDir, "../deploy-artifacts"); +const patchId = process.argv[2] || "module-foundry-hub-registration-20260714-001"; +const profile = process.argv[3] || "registration"; + +if (!/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error("patch_id_must_contain_only_letters_digits_dot_underscore_hyphen"); +} + +// DCPLATFORM-21: the production runner overlays this payload onto the existing +// Launcher source and recreates only the `launcher` service. Keep corrective +// patches to their exact reviewed file set instead of re-sending unrelated +// registration sources. +const registrationFiles = [ + "server/authentik-sync.mjs", + "server/control-plane-store.mjs", + "server/dev-server.mjs", + "src/entities/service/types.ts", +]; + +const filesByProfile = { + registration: registrationFiles, + "handoff-fix": ["server/dev-server.mjs"], +}; + +const files = filesByProfile[profile]; +if (!files) { + throw new Error(`unknown_profile:${profile}`); +} + +const stage = await mkdtemp(join(tmpdir(), "nodedc-launcher-foundry-artifact-")); +const payloadRoot = join(stage, "payload"); +const artifact = join(artifactDir, `launcher-${patchId}.tgz`); +const checksum = `${artifact}.sha256`; + +try { + await mkdir(payloadRoot, { recursive: true }); + + for (const relativePath of files) { + const source = resolve(launcherRoot, relativePath); + const destination = join(payloadRoot, relativePath); + const stat = await lstat(source); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`source_file_rejected:${relativePath}`); + } + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, verbatimSymlinks: true }); + } + + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=launcher\ntype=app-overlay\n`, + "utf8", + ); + await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + + // macOS bsdtar may emit AppleDouble `._*` sidecars. DCPLATFORM-21 rejects + // them, therefore create a data-only POSIX archive through stdlib tarfile. + const tar = spawnSync("python3", ["-c", [ + "import sys, tarfile", + "with tarfile.open(sys.argv[1], 'w:gz', format=tarfile.PAX_FORMAT) as archive:", + " [archive.add(name, arcname=name, recursive=True) for name in ('manifest.env', 'files.txt', 'payload')]", + ].join("\n"), artifact], { + cwd: stage, + encoding: "utf8", + }); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + + const digest = createHash("sha256").update(await readFile(artifact)).digest("hex"); + await writeFile(checksum, `${digest} ${artifact.split("/").at(-1)}\n`, "utf8"); + console.log(JSON.stringify({ ok: true, patchId, profile, artifact, checksum, sha256: digest, files }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} diff --git a/infra/deploy-runner/build-map-gateway-artifact.mjs b/infra/deploy-runner/build-map-gateway-artifact.mjs new file mode 100644 index 0000000..cbd31e2 --- /dev/null +++ b/infra/deploy-runner/build-map-gateway-artifact.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, lstat, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const artifactDir = resolve(scriptDir, "../deploy-artifacts"); +const [patchId = "platform-map-gateway-20260714-001", ...extra] = process.argv.slice(2); +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) throw new Error("usage: build-map-gateway-artifact.mjs [patch-id]"); + +const files = [ + ["infra/synology/docker-compose.platform-http.yml", "platform/docker-compose.platform-http.yml"], + ["services/map-gateway", "platform/services/map-gateway"], +]; +const ignored = new Set([".DS_Store", ".git", "node_modules"]); +const stage = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-artifact-")); +const payload = join(stage, "payload"); +const target = join(artifactDir, `nodedc-platform-${patchId}.tgz`); + +try { + await mkdir(payload, { recursive: true }); + for (const [source, destination] of files) await copySafe(resolve(platformRoot, source), join(payload, destination)); + await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=platform\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${files.map(([, destination]) => destination).join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + const tar = spawnSync("python3", ["-c", "import sys,tarfile\nwith tarfile.open(sys.argv[1],'w:gz',format=tarfile.PAX_FORMAT) as a:\n [a.add(n,arcname=n,recursive=True) for n in ('manifest.env','files.txt','payload')]", target], { cwd: stage, encoding: "utf8" }); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + console.log(JSON.stringify({ ok: true, patchId, artifact: target, sha256: createHash("sha256").update(await readFile(target)).digest("hex") }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function copySafe(source, destination) { + const info = await lstat(source); + if (info.isSymbolicLink()) throw new Error(`source_symlink_rejected:${source}`); + if (info.isFile()) { await mkdir(dirname(destination), { recursive: true }); await cp(source, destination, { force: true }); return; } + if (!info.isDirectory()) throw new Error(`source_type_rejected:${source}`); + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if (ignored.has(entry.name) || entry.name.startsWith(".env")) continue; + const child = join(source, entry.name); + if (entry.isSymbolicLink()) throw new Error(`source_symlink_rejected:${relative(platformRoot, child)}`); + await copySafe(child, join(destination, entry.name)); + } +} diff --git a/infra/deploy-runner/build-module-foundry-artifact.mjs b/infra/deploy-runner/build-module-foundry-artifact.mjs new file mode 100644 index 0000000..cf95dc1 --- /dev/null +++ b/infra/deploy-runner/build-module-foundry-artifact.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { cp, lstat, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const workspaceRoot = resolve(platformRoot, ".."); +const foundryRoot = resolve(workspaceRoot, "NODEDC_DESIGN_GUIDELINE"); +const artifactDir = resolve(scriptDir, "../deploy-artifacts"); +const patchId = process.argv[2] || "module-foundry-bootstrap-20260714-001"; + +if (!/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error("patch_id_must_contain_only_letters_digits_dot_underscore_hyphen"); +} + +const files = [ + ".dockerignore", + ".env.example", + ".gitignore", + "Dockerfile", + "package.json", + "package-lock.json", + "tsconfig.base.json", + "infra/docker-compose.module-foundry.yml", + "apps", + "packages", + "registry", + "runtime-seed", + "scripts", + "server", +]; +const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules", "runtime-data", "dist"]); +const stage = await mkdtemp(join(tmpdir(), "nodedc-module-foundry-artifact-")); +const payload = join(stage, "payload"); +const target = join(artifactDir, `nodedc-module-foundry-${patchId}.tgz`); + +try { + await mkdir(payload, { recursive: true }); + for (const sourceRelative of files) { + await copySafe(resolve(foundryRoot, sourceRelative), join(payload, sourceRelative)); + } + + await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=module-foundry\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${files.join("\n")}\n`, "utf8"); + await mkdir(artifactDir, { recursive: true }); + + const tar = spawnSync("python3", ["-c", [ + "import sys, tarfile", + "with tarfile.open(sys.argv[1], 'w:gz', format=tarfile.PAX_FORMAT) as archive:", + " [archive.add(name, arcname=name, recursive=True) for name in ('manifest.env', 'files.txt', 'payload')]", + ].join("\n"), target], { + cwd: stage, + encoding: "utf8", + }); + if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + + const digest = createHash("sha256").update(await readFile(target)).digest("hex"); + console.log(JSON.stringify({ ok: true, patchId, artifact: target, sha256: digest }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) throw new Error(`source_symlink_rejected:${relative(foundryRoot, source)}`); + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { force: true, verbatimSymlinks: true }); + return; + } + if (!sourceStat.isDirectory()) throw new Error(`source_type_rejected:${source}`); + + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if (ignoredBasenames.has(entry.name) || entry.name.startsWith(".env") || entry.name.endsWith(".tsbuildinfo")) continue; + const childSource = join(source, entry.name); + const childDestination = join(destination, entry.name); + if (entry.isSymbolicLink()) throw new Error(`source_symlink_rejected:${relative(foundryRoot, childSource)}`); + await copySafe(childSource, childDestination); + } +} diff --git a/infra/deploy-runner/build-n8n-private-extension-artifact.mjs b/infra/deploy-runner/build-n8n-private-extension-artifact.mjs new file mode 100644 index 0000000..48f2ee2 --- /dev/null +++ b/infra/deploy-runner/build-n8n-private-extension-artifact.mjs @@ -0,0 +1,286 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; +import { spawnSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const packageRoot = resolve(platformRoot, "packages/n8n-nodes-ndc"); +const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts")); +const requireModule = createRequire(import.meta.url); +const expectedPackageVersion = "0.1.2"; +const [patchId = "n8n-nodes-ndc-release-20260716-003", ...extra] = process.argv.slice(2); +const expectedRuntimeNodes = [ + { + file: "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", + exportName: "NdcDataProductPublish", + name: "ndcDataProductPublish", + }, + { + file: "dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js", + exportName: "NdcDataProductRead", + name: "ndcDataProductRead", + }, + { + file: "dist/nodes/NdcFoundryBinding/NdcFoundryBinding.node.js", + exportName: "NdcFoundryBinding", + name: "ndcFoundryBinding", + }, +]; +const expectedN8nNodes = expectedRuntimeNodes.map((node) => node.file); +const expectedRuntimeNodeTypes = expectedRuntimeNodes.map((node) => `n8n-nodes-ndc.${node.name}`); +const expectedN8nCredentials = [ + "dist/credentials/NdcDataProductWriterApi.credentials.js", + "dist/credentials/NdcDataProductReaderApi.credentials.js", + "dist/credentials/NdcFoundryBindingApi.credentials.js", +]; +const expectedCredentialTypes = [ + "ndcDataProductWriterApi", + "ndcDataProductReaderApi", + "ndcFoundryBindingApi", +]; + +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error("usage: build-n8n-private-extension-artifact.mjs [patch-id]"); +} + +const packageJson = JSON.parse(await readFile(join(packageRoot, "package.json"), "utf8")); +assertPackageSourcePolicy(packageJson); +run("npm", ["test"], packageRoot); +run("npm", ["run", "lint"], packageRoot); +assertRuntimeNodePolicy(packageJson); + +const stage = await mkdtemp(join(tmpdir(), "nodedc-n8n-private-extension-")); +const packDir = join(stage, "pack"); +const payload = join(stage, "payload"); + +try { + await mkdir(packDir, { recursive: true }); + const pack = run("npm", ["pack", "--ignore-scripts", "--json", "--pack-destination", packDir], packageRoot); + const packResult = JSON.parse(pack.stdout); + if (!Array.isArray(packResult) || packResult.length !== 1) throw new Error("npm_pack_result_invalid"); + const metadata = packResult[0]; + assertPackedFilePolicy(metadata, packageJson); + + const packedPath = join(packDir, metadata.filename); + const canonicalPackedPath = join(packDir, "n8n-nodes-ndc.canonical.tgz"); + const canonicalPackageScript = [ + "import gzip, io, sys, tarfile", + "members = []", + "with tarfile.open(sys.argv[1], 'r:gz') as source:", + " for original in source:", + " if not (original.isfile() or original.isdir()):", + " raise SystemExit('unsupported npm package member')", + " content = b''", + " if original.isfile():", + " extracted = source.extractfile(original)", + " if extracted is None:", + " raise SystemExit('unreadable npm package member')", + " content = extracted.read()", + " members.append((original.name, original.isdir(), content))", + "with open(sys.argv[2], 'wb') as output:", + " with gzip.GzipFile(filename='', mode='wb', fileobj=output, compresslevel=9, mtime=0) as compressed:", + " with tarfile.open(fileobj=compressed, mode='w', format=tarfile.PAX_FORMAT) as target:", + " for name, is_dir, content in sorted(members, key=lambda item: item[0]):", + " member = tarfile.TarInfo(name)", + " member.uid = member.gid = 0", + " member.uname = member.gname = 'root'", + " member.mtime = 0", + " member.mode = 0o755 if is_dir else 0o644", + " member.type = tarfile.DIRTYPE if is_dir else tarfile.REGTYPE", + " member.size = 0 if is_dir else len(content)", + " target.addfile(member, None if is_dir else io.BytesIO(content))", + ].join("\n"); + run("python3", ["-c", canonicalPackageScript, packedPath, canonicalPackedPath], stage); + const packageBytes = await readFile(canonicalPackedPath); + const packageSha256 = createHash("sha256").update(packageBytes).digest("hex"); + const releaseId = `${packageJson.version}-${packageSha256.slice(0, 16)}`; + const relativeReleasePath = `releases/n8n-nodes-ndc/${releaseId}`; + const releaseDir = join(payload, relativeReleasePath); + await mkdir(releaseDir, { recursive: true }); + await cp(canonicalPackedPath, join(releaseDir, "package.tgz"), { force: false }); + + const rollbackBaselinePolicy = { + allowed: [ + "previous_verified_immutable_release", + "verified_inactive", + ], + firstActivation: "verified_inactive", + requiresPreActivationVerification: true, + }; + const release = { + schemaVersion: "nodedc.n8n-private-extension-release/v2", + releaseId, + package: { + name: "n8n-nodes-ndc", + version: packageJson.version, + sha256: packageSha256, + bytes: packageBytes.byteLength, + runtimeTypePrefix: "n8n-nodes-ndc.", + }, + storage: { + relativePath: relativeReleasePath, + immutable: true, + }, + activation: { + owner: "engine", + status: "blocked_pending_engine_owned_mount", + requiredCommunityPackagePath: "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc", + requiresAtomicReleaseSwitch: true, + requiresAllN8nProcessesRestart: true, + requiresMcpSchemaAcceptance: true, + rollbackBaselinePolicy, + }, + }; + const rollback = { + schemaVersion: "nodedc.n8n-private-extension-rollback/v2", + releaseId, + packageSha256, + mode: "engine-owned-atomic-release-switch", + baselinePolicy: rollbackBaselinePolicy, + steps: [ + "select_verified_previous_release_or_preverified_inactive_baseline", + "switch_engine_owned_mount_atomically", + "restart_all_n8n_processes", + "verify_mcp_schema_state_matches_selected_baseline", + ], + forbidden: [ + "delete_active_release", + "mutate_engine_core", + "live_npm_install", + ], + }; + await writeFile(join(releaseDir, "release.json"), `${JSON.stringify(release, null, 2)}\n`, "utf8"); + await writeFile(join(releaseDir, "rollback.json"), `${JSON.stringify(rollback, null, 2)}\n`, "utf8"); + await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=n8n-private-extension\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${relativeReleasePath}\n`, "utf8"); + + await mkdir(artifactDir, { recursive: true }); + const target = join(artifactDir, `nodedc-n8n-private-extension-${patchId}.tgz`); + const tarScript = [ + "import gzip, os, pathlib, sys, tarfile", + "root = pathlib.Path(sys.argv[2])", + "def clean(info):", + " info.uid = info.gid = 0", + " info.uname = info.gname = 'root'", + " info.mtime = 0", + " info.mode = 0o755 if info.isdir() else 0o644", + " return info", + "with open(sys.argv[1], 'wb') as output:", + " with gzip.GzipFile(filename='', mode='wb', fileobj=output, compresslevel=9, mtime=0) as compressed:", + " with tarfile.open(fileobj=compressed, mode='w', format=tarfile.PAX_FORMAT) as archive:", + " for top in ('manifest.env', 'files.txt', 'payload'):", + " path = root / top", + " archive.add(path, arcname=top, recursive=False, filter=clean)", + " if path.is_dir():", + " for child in sorted(path.rglob('*'), key=lambda item: item.as_posix()):", + " archive.add(child, arcname=child.relative_to(root).as_posix(), recursive=False, filter=clean)", + ].join("\n"); + run("python3", ["-c", tarScript, target, stage], stage); + + const artifactSha256 = createHash("sha256").update(await readFile(target)).digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact: target, + artifactSha256, + releaseId, + packageSha256, + nodeTypes: expectedRuntimeNodeTypes, + credentialTypes: expectedCredentialTypes, + activation: "blocked_pending_engine_owned_mount", + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +function assertPackageSourcePolicy(value) { + if (value.name !== "n8n-nodes-ndc") throw new Error("package_name_invalid"); + if (value.version !== expectedPackageVersion) throw new Error("package_version_invalid"); + if (value.private !== true) throw new Error("package_must_remain_private"); + if (value.dependencies !== undefined) throw new Error("runtime_dependencies_forbidden"); + for (const lifecycle of ["preinstall", "install", "postinstall", "prepack", "prepare", "postpack"]) { + if (value.scripts?.[lifecycle] !== undefined) throw new Error(`lifecycle_script_forbidden:${lifecycle}`); + } + assertExactRegistration(value.n8n?.nodes, expectedN8nNodes, "n8n_nodes"); + assertExactRegistration(value.n8n?.credentials, expectedN8nCredentials, "n8n_credentials"); +} + +function assertRuntimeNodePolicy(packageJson) { + const observedTypes = []; + for (const expected of expectedRuntimeNodes) { + const loaded = requireModule(join(packageRoot, expected.file)); + const NodeClass = loaded?.[expected.exportName]; + if (typeof NodeClass !== "function") throw new Error(`runtime_node_export_missing:${expected.exportName}`); + const description = new NodeClass()?.description; + if (!description || description.name !== expected.name) { + throw new Error(`runtime_node_name_mismatch:${expected.exportName}`); + } + if ("usableAsTool" in description) { + throw new Error(`runtime_tool_variant_forbidden:${expected.name}`); + } + observedTypes.push(`${packageJson.name}.${description.name}`); + } + if (JSON.stringify(observedTypes) !== JSON.stringify(expectedRuntimeNodeTypes)) { + throw new Error("runtime_node_types_mismatch"); + } +} + +function assertExactRegistration(actual, expected, label) { + if (!Array.isArray(actual)) throw new Error(`${label}_missing`); + if (actual.length !== new Set(actual).size) throw new Error(`${label}_duplicate`); + if (actual.length !== expected.length || expected.some((value) => !actual.includes(value))) { + throw new Error(`${label}_mismatch`); + } +} + +function assertPackedFilePolicy(metadata, sourcePackage) { + if (metadata.name !== sourcePackage.name || metadata.version !== sourcePackage.version) { + throw new Error("npm_pack_identity_mismatch"); + } + if (!Number.isSafeInteger(metadata.size) || metadata.size < 1024 || metadata.size > 32 * 1024 * 1024) { + throw new Error("npm_pack_size_invalid"); + } + if (!Array.isArray(metadata.files) || metadata.files.length > 512) throw new Error("npm_pack_file_list_invalid"); + const paths = new Set(); + for (const file of metadata.files) { + if (!file || typeof file.path !== "string" || paths.has(file.path)) throw new Error("npm_pack_file_invalid"); + paths.add(file.path); + if (!(file.path === "README.md" || file.path === "package.json" || file.path.startsWith("dist/"))) { + throw new Error(`npm_pack_path_forbidden:${file.path}`); + } + const parts = file.path.split("/"); + if ( + file.path.includes("\\") + || parts.some((part) => !part || part === "." || part === ".." || part.startsWith(".")) + || file.mode !== 0o644 + ) { + throw new Error(`npm_pack_path_unsafe:${file.path}`); + } + } + assertExactRegistration( + [...paths].filter((value) => /^dist\/nodes\/.+\.node\.js$/.test(value)), + expectedN8nNodes, + "npm_pack_nodes", + ); + assertExactRegistration( + [...paths].filter((value) => /^dist\/credentials\/.+\.credentials\.js$/.test(value)), + expectedN8nCredentials, + "npm_pack_credentials", + ); + for (const registered of [...sourcePackage.n8n.nodes, ...sourcePackage.n8n.credentials]) { + if (!paths.has(registered)) throw new Error(`npm_pack_registration_missing:${registered}`); + } +} + +function run(command, args, cwd) { + const result = spawnSync(command, args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + if (result.status !== 0) { + throw new Error(`${command}_failed:${result.stderr || result.stdout}`); + } + return result; +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 094512e..fc8b6d2 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -3,6 +3,7 @@ import argparse import json import os import re +import secrets import shutil import stat import subprocess @@ -28,6 +29,40 @@ STATE_FILE = STATE_DIR / "applied.jsonl" FAILED_STATE_FILE = STATE_DIR / "failed.jsonl" LOCK_DIR = STATE_DIR / "deploy.lock" DOCKER = Path("/usr/local/bin/docker") +MAP_GATEWAY_SECRET_DIR = Path("/volume1/docker/nodedc-platform/secrets") +MAP_GATEWAY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-gateway-admin-secret" +MAP_EGRESS_PROXY_SECRET_FILE = MAP_GATEWAY_SECRET_DIR / "map-egress-proxy-token" +PROXY_CONTUR_ENV_FILE = Path("/volume1/docker/proxy-contur/.env") +DC_AMD_PROXY_RUNTIME_DIR = Path("/volume1/docker/dc-amd-proxy/runtime") +EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-provisioner" +EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "token" +EXTERNAL_DATA_PLANE_READER_GRANTS_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-reader-grants" +FOUNDRY_BINDING_GRANTS_DIR = MAP_GATEWAY_SECRET_DIR / "foundry-binding-grants" +N8N_PRIVATE_EXTENSION_RELEASES_ROOT = Path("/volume1/docker/nodedc-platform/n8n-private-extensions") +ENGINE_N8N_SEALED_RELEASES_ROOT = Path("/volume2/nodedc-demo/n8n-private-extensions") +ENGINE_N8N_TRANSITION_DESCRIPTOR_REL = "nodedc-source/services/n8n/private-extensions/ndc-activation.json" +ENGINE_N8N_COMPOSE_OVERRIDE_REL = "nodedc-source/services/n8n/private-extensions/docker-compose.ndc-private-extension.yml" +ENGINE_N8N_SCHEMA_ROOT_REL = "nodedc-source/server/assets/n8n/schema/v2.3.2" +ENGINE_N8N_NODES_CATALOG_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/nodes.catalog.json" +ENGINE_N8N_CREDENTIALS_CATALOG_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/credentials.catalog.json" +ENGINE_N8N_SCHEMA_META_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/meta.json" +ENGINE_N8N_ICON_REL = "nodedc-source/server/assets/n8n/icons/ndc.svg" +ENGINE_N8N_DARK_ICON_REL = "nodedc-source/server/assets/n8n/icons/ndc.dark.svg" +ENGINE_N8N_VERSION = "2.3.2" +ENGINE_N8N_BASE_IMAGE = "docker.n8n.io/n8nio/n8n:2.3.2" +ENGINE_N8N_BASE_ARCHITECTURE = "amd64" +ENGINE_N8N_RUNTIME_PACKAGE_PATH = "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc" +ENGINE_N8N_ICON_SHA256 = "1c928fc996d8b82121e8f8e327d74b1dd21556c106de99c5e8d317d926c0ba17" +ENGINE_N8N_DARK_ICON_SHA256 = "ef3b8551da4ce04527736405afa9a220a2c76d047bd18f6f7124b9adb4216b0c" +ENGINE_N8N_ACTIVATION_NODES_CATALOG_JSON_SHA256 = "230f712230f7ee8debaa4b44a4358cc19c205bc43305d8ed89d5e3daac466506" +ENGINE_N8N_ACTIVATION_CREDENTIALS_CATALOG_JSON_SHA256 = "a26824ffc0e15db857c792153de3417febc0dbba4880380d6c8cd544b3c80f4d" +ENGINE_N8N_ACTIVATION_META_JSON_SHA256 = "caf7635420c61333e65f68350f5567217aed96fb51354b38764bcac2c24e7966" +ENGINE_N8N_INACTIVE_NODES_CATALOG_JSON_SHA256 = "b70d9d8130d498c55de46a5d0758c844f1242b70803457b2291ab3a9de8056f2" +ENGINE_N8N_INACTIVE_CREDENTIALS_CATALOG_JSON_SHA256 = "680e9f52aac791efbd38e3bd99bd51ef5cded9756d867897c6c2755850e87b50" +ENGINE_N8N_INACTIVE_META_JSON_SHA256 = "0ac555d7ab31cb0ca042db4f474e83cfefbf20b5bbb2e1509fead27ed21e8acb" +MAP_GATEWAY_RUNTIME_GID = 1000 +EXTERNAL_DATA_PLANE_RUNTIME_UID = 11006 +EXTERNAL_DATA_PLANE_RUNTIME_GID = 11006 MAX_ARTIFACT_BYTES = 512 * 1024 * 1024 MAX_MEMBER_COUNT = 20000 @@ -36,6 +71,40 @@ MAX_FILE_BYTES = 256 * 1024 * 1024 PATCH_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,96}$") MANIFEST_KEYS = {"id", "component", "type"} +MAP_GATEWAY_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{48,256}$") +EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{48,256}$") +N8N_PRIVATE_EXTENSION_RELEASE_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+-[a-f0-9]{16}$") +N8N_PRIVATE_EXTENSION_NODES = ( + "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", + "dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js", + "dist/nodes/NdcFoundryBinding/NdcFoundryBinding.node.js", +) +N8N_PRIVATE_EXTENSION_CREDENTIALS = ( + "dist/credentials/NdcDataProductWriterApi.credentials.js", + "dist/credentials/NdcDataProductReaderApi.credentials.js", + "dist/credentials/NdcFoundryBindingApi.credentials.js", +) +ENGINE_N8N_NODE_TYPES = ( + "n8n-nodes-ndc.ndcDataProductPublish", + "n8n-nodes-ndc.ndcDataProductRead", + "n8n-nodes-ndc.ndcFoundryBinding", +) +ENGINE_N8N_CREDENTIAL_TYPES = ( + "ndcDataProductWriterApi", + "ndcDataProductReaderApi", + "ndcFoundryBindingApi", +) +ENGINE_BASE_COMPOSE_SERVICES = ( + "postgresql", + "authentik-server", + "authentik-worker", + "n8n-postgres", + "ops-postgres", + "n8n", + "nodedc-backend", + "nodedc-frontend-build", + "app", +) COMPONENTS = { "engine": { @@ -73,9 +142,10 @@ COMPONENTS = { "compose_env_file": Path("/volume1/docker/nodedc-platform/platform/.env.synology"), "compose_files": ( Path("/volume1/docker/nodedc-platform/platform/docker-compose.platform-http.yml"), + Path("/volume1/docker/nodedc-platform/platform/docker-compose.external-data-plane.yml"), ), "compose_no_deps": True, - "services": ("notification-postgres", "notification-core", "ai-workspace-hub", "launcher", "reverse-proxy"), + "services": ("notification-postgres", "notification-core", "ai-workspace-hub", "launcher", "reverse-proxy", "external-data-plane-postgres", "external-data-plane", "map-gateway"), "healthchecks": ( "http://127.0.0.1:5185/healthz", "http://127.0.0.1:18081/healthz", @@ -135,6 +205,55 @@ COMPONENTS = { "http://127.0.0.1:18100/api/auth/session", ), }, + "n8n-private-extension": { + # This component only stages a verified, immutable offline release in + # Platform-owned storage. It deliberately has no Compose file, service, + # container mutation or Engine activation side effect. + "payload_root": N8N_PRIVATE_EXTENSION_RELEASES_ROOT, + "bootstrap_root": True, + "artifact_only": True, + "immutable_payload": True, + "services": (), + }, + "module-foundry": { + "payload_root": Path("/volume1/docker/nodedc-platform/module-foundry/source"), + "compose_root": Path("/volume1/docker/nodedc-platform/module-foundry/source/infra"), + "compose_project": "nodedc-module-foundry", + "compose_env_file": Path("/volume1/docker/nodedc-platform/module-foundry/source/.env"), + "compose_files": ( + Path("/volume1/docker/nodedc-platform/module-foundry/source/infra/docker-compose.module-foundry.yml"), + ), + "bootstrap_root": True, + "compose_build": True, + "compose_no_deps": True, + "services": ("nodedc-module-foundry",), + "healthchecks": ( + "http://172.22.0.222:9920/healthz", + ), + }, + "proxy-contur": { + "payload_root": Path("/volume1/docker/proxy-contur"), + "compose_root": Path("/volume1/docker/proxy-contur"), + "compose_project": "proxy-contur", + "compose_env_file": Path("/volume1/docker/proxy-contur/.env"), + "compose_files": ( + Path("/volume1/docker/proxy-contur/docker-compose.yml"), + ), + "compose_build": True, + "compose_no_deps": True, + "services": ("proxy-contur",), + "health_container": "proxy-contur", + }, + "dc-amd-proxy": { + "payload_root": Path("/volume1/docker/dc-amd-proxy"), + "compose_root": Path("/volume1/docker/dc-amd-proxy"), + "compose_project": "dc-amd-proxy", + "bootstrap_root": True, + "compose_build": True, + "compose_no_deps": True, + "services": ("dc-amd-proxy",), + "health_container": "dc-amd-proxy", + }, "dc-cms": { "payload_root": Path("/volume1/docker/dc-cms/source"), "compose_root": Path("/volume1/docker/dc-cms/source/infra"), @@ -209,6 +328,18 @@ def sha256_file(path): return h.hexdigest() +def sha256_json_value(value): + import hashlib + + canonical = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + def safe_name(value): return re.sub(r"[^A-Za-z0-9._-]", "_", value) @@ -234,6 +365,248 @@ def ensure_layout(): path.chmod(0o755) +def ensure_platform_runtime_secret(secret_file, secret_re, label): + # This is Platform infrastructure state, not a product setting. The + # root-owned runner creates it once and only explicitly bound service mounts + # can read it. It must never be copied into an artifact or a shared .env. + secret_dir = secret_file.parent + try: + directory_stat = secret_dir.lstat() + except FileNotFoundError: + secret_dir.mkdir(parents=True, exist_ok=False) + directory_stat = secret_dir.lstat() + + if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode): + die(f"{label} secret directory is unsafe: {secret_dir}") + os.chown(secret_dir, 0, MAP_GATEWAY_RUNTIME_GID) + secret_dir.chmod(0o710) + + try: + secret_stat = secret_file.lstat() + except FileNotFoundError: + secret_value = secrets.token_urlsafe(48) + temporary = secret_dir / f".{secret_file.name}.{os.getpid()}.{time.time_ns()}.tmp" + descriptor = None + try: + descriptor = os.open(str(temporary), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640) + os.write(descriptor, f"{secret_value}\n".encode("ascii")) + os.fsync(descriptor) + os.fchown(descriptor, 0, MAP_GATEWAY_RUNTIME_GID) + os.fchmod(descriptor, 0o640) + os.close(descriptor) + descriptor = None + os.replace(temporary, secret_file) + fsync_directory(secret_dir) + finally: + if descriptor is not None: + os.close(descriptor) + if temporary.exists(): + temporary.unlink() + return "created" + + if stat.S_ISLNK(secret_stat.st_mode) or not stat.S_ISREG(secret_stat.st_mode): + die(f"{label} secret file is unsafe: {secret_file}") + if secret_stat.st_uid != 0 or secret_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + die(f"{label} secret file has unsafe ownership or permissions: {secret_file}") + if secret_stat.st_size > 512: + die(f"{label} secret file is too large: {secret_file}") + try: + secret_value = secret_file.read_text(encoding="ascii").strip() + except UnicodeDecodeError: + die(f"{label} secret file is not ascii: {secret_file}") + if not secret_re.fullmatch(secret_value): + die(f"{label} secret file has invalid format: {secret_file}") + os.chown(secret_file, 0, MAP_GATEWAY_RUNTIME_GID) + secret_file.chmod(0o640) + return "reused" + + +def ensure_map_gateway_admin_secret(): + return ensure_platform_runtime_secret( + MAP_GATEWAY_SECRET_FILE, + MAP_GATEWAY_SECRET_RE, + "map gateway", + ) + + +def read_proxy_contur_token(): + try: + source_stat = PROXY_CONTUR_ENV_FILE.lstat() + except FileNotFoundError: + die("proxy-contur env file is required for Map egress") + if stat.S_ISLNK(source_stat.st_mode) or not stat.S_ISREG(source_stat.st_mode): + die("proxy-contur env file is unsafe") + if source_stat.st_size > 64 * 1024 or source_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + die("proxy-contur env file has unsafe permissions") + try: + lines = PROXY_CONTUR_ENV_FILE.read_text(encoding="utf-8").splitlines() + except UnicodeDecodeError: + die("proxy-contur env file is not utf-8") + + value = None + for raw in lines: + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[7:].lstrip() + if not line.startswith("PROXY_TOKEN="): + continue + value = line.partition("=")[2].strip() + if len(value) >= 2 and value[0] in ("'", '"') and value[-1] == value[0]: + value = value[1:-1] + break + + if not value or len(value) > 4096 or any(ord(char) < 33 or ord(char) == 127 for char in value): + die("proxy-contur PROXY_TOKEN is missing or invalid") + return value + + +def sync_map_egress_proxy_secret(): + # The Map Gateway gets a read-only file copy of the existing proxy token. + # The value is never printed, placed in an artifact, or exposed to Foundry. + token = read_proxy_contur_token() + try: + parent_stat = MAP_GATEWAY_SECRET_DIR.lstat() + except FileNotFoundError: + MAP_GATEWAY_SECRET_DIR.mkdir(parents=True, exist_ok=False) + parent_stat = MAP_GATEWAY_SECRET_DIR.lstat() + if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode): + die(f"map egress parent secret directory is unsafe: {MAP_GATEWAY_SECRET_DIR}") + os.chown(MAP_GATEWAY_SECRET_DIR, 0, MAP_GATEWAY_RUNTIME_GID) + MAP_GATEWAY_SECRET_DIR.chmod(0o710) + + if MAP_EGRESS_PROXY_SECRET_FILE.exists() or MAP_EGRESS_PROXY_SECRET_FILE.is_symlink(): + target_stat = MAP_EGRESS_PROXY_SECRET_FILE.lstat() + if stat.S_ISLNK(target_stat.st_mode) or not stat.S_ISREG(target_stat.st_mode): + die("map egress proxy secret file is unsafe") + + temporary = MAP_GATEWAY_SECRET_DIR / f".{MAP_EGRESS_PROXY_SECRET_FILE.name}.{os.getpid()}.{time.time_ns()}.tmp" + descriptor = None + try: + descriptor = os.open(str(temporary), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o640) + os.write(descriptor, f"{token}\n".encode("utf-8")) + os.fsync(descriptor) + os.fchown(descriptor, 0, MAP_GATEWAY_RUNTIME_GID) + os.fchmod(descriptor, 0o640) + os.close(descriptor) + descriptor = None + os.replace(temporary, MAP_EGRESS_PROXY_SECRET_FILE) + fsync_directory(MAP_GATEWAY_SECRET_DIR) + finally: + if descriptor is not None: + os.close(descriptor) + if temporary.exists(): + temporary.unlink() + return "synced" + + +def ensure_root_owned_grant_directory(path, label): + try: + parent_stat = MAP_GATEWAY_SECRET_DIR.lstat() + except FileNotFoundError: + MAP_GATEWAY_SECRET_DIR.mkdir(parents=True, exist_ok=False) + parent_stat = MAP_GATEWAY_SECRET_DIR.lstat() + if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode): + die(f"{label} parent directory is unsafe: {MAP_GATEWAY_SECRET_DIR}") + os.chown(MAP_GATEWAY_SECRET_DIR, 0, MAP_GATEWAY_RUNTIME_GID) + MAP_GATEWAY_SECRET_DIR.chmod(0o710) + + try: + directory_stat = path.lstat() + except FileNotFoundError: + path.mkdir(parents=False, exist_ok=False) + directory_stat = path.lstat() + if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode): + die(f"{label} directory is unsafe: {path}") + os.chown(path, 0, 0) + path.chmod(0o500) + + for entry in path.iterdir(): + entry_stat = entry.lstat() + if (not re.fullmatch(r"[a-f0-9]{64}", entry.name) + or stat.S_ISLNK(entry_stat.st_mode) + or not stat.S_ISREG(entry_stat.st_mode) + or entry_stat.st_uid != 0 + or entry_stat.st_gid != 0 + or stat.S_IMODE(entry_stat.st_mode) != 0o400 + or entry_stat.st_size < 2 + or entry_stat.st_size > 128 * 1024): + die(f"{label} record is unsafe: {entry}") + + +def ensure_external_data_plane_provisioner_secret(): + # Unlike the Map Gateway key, this capability can mint scoped writers. + # Keep it in an isolated child directory readable only by the dedicated + # EDP/provisioner uid, never by the shared gid 1000. + try: + parent_stat = MAP_GATEWAY_SECRET_DIR.lstat() + except FileNotFoundError: + MAP_GATEWAY_SECRET_DIR.mkdir(parents=True, exist_ok=False) + parent_stat = MAP_GATEWAY_SECRET_DIR.lstat() + if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode): + die(f"external data plane parent secret directory is unsafe: {MAP_GATEWAY_SECRET_DIR}") + os.chown(MAP_GATEWAY_SECRET_DIR, 0, MAP_GATEWAY_RUNTIME_GID) + MAP_GATEWAY_SECRET_DIR.chmod(0o710) + + try: + directory_stat = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.lstat() + except FileNotFoundError: + EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.mkdir(parents=False, exist_ok=False) + directory_stat = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.lstat() + if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(directory_stat.st_mode): + die(f"external data plane provisioner secret directory is unsafe: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR}") + os.chown(EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR, EXTERNAL_DATA_PLANE_RUNTIME_UID, EXTERNAL_DATA_PLANE_RUNTIME_GID) + EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR.chmod(0o500) + + try: + secret_stat = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE.lstat() + except FileNotFoundError: + secret_value = secrets.token_urlsafe(48) + temporary = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / f".{EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE.name}.{os.getpid()}.{time.time_ns()}.tmp" + descriptor = None + try: + descriptor = os.open(str(temporary), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o400) + os.write(descriptor, f"{secret_value}\n".encode("ascii")) + os.fsync(descriptor) + os.fchown(descriptor, EXTERNAL_DATA_PLANE_RUNTIME_UID, EXTERNAL_DATA_PLANE_RUNTIME_GID) + os.fchmod(descriptor, 0o400) + os.close(descriptor) + descriptor = None + os.replace(temporary, EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE) + fsync_directory(EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR) + finally: + if descriptor is not None: + os.close(descriptor) + if temporary.exists(): + temporary.unlink() + return "created" + + if stat.S_ISLNK(secret_stat.st_mode) or not stat.S_ISREG(secret_stat.st_mode): + die(f"external data plane provisioner secret file is unsafe: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}") + if (secret_stat.st_uid != EXTERNAL_DATA_PLANE_RUNTIME_UID or + secret_stat.st_gid != EXTERNAL_DATA_PLANE_RUNTIME_GID or + stat.S_IMODE(secret_stat.st_mode) != 0o400): + die(f"external data plane provisioner secret file has unsafe ownership or permissions: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}") + if secret_stat.st_size > 512: + die(f"external data plane provisioner secret file is too large: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}") + try: + secret_value = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE.read_text(encoding="ascii").strip() + except UnicodeDecodeError: + die(f"external data plane provisioner secret file is not ascii: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}") + if not EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_RE.fullmatch(secret_value): + die(f"external data plane provisioner secret file has invalid format: {EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}") + return "reused" + + +def fsync_directory(path): + descriptor = os.open(str(path), os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def require_root(): if os.geteuid() != 0: die("run with sudo") @@ -459,13 +832,20 @@ def denied_payload_path(component, rel): parts = lower.split("/") base = parts[-1] - if component == "engine" and rel == "docker-compose.yml": + if component == "engine" and rel in ( + "docker-compose.yml", + "nodedc-source/services/n8n/private-extensions/n8n-nodes-ndc/Dockerfile", + ): pass elif component == "platform" and rel in ( "platform/docker-compose.platform-http.yml", "platform/notification-core/Dockerfile", "platform/ai-workspace-hub/Dockerfile", "platform/ai-workspace-assistant/Dockerfile", + "platform/ontology-core/Dockerfile", + "platform/gelios-gateway/Dockerfile", + "platform/services/map-gateway/Dockerfile", + "platform/services/external-data-plane/Dockerfile", ): pass elif component == "bim-viewer" and rel in ( @@ -477,6 +857,21 @@ def denied_payload_path(component, rel): "infra/docker-compose.yml", ): pass + elif component == "module-foundry" and rel in ( + "Dockerfile", + "infra/docker-compose.module-foundry.yml", + ): + pass + elif component == "proxy-contur" and rel in ( + "Dockerfile", + "docker-compose.yml", + ): + pass + elif component == "dc-amd-proxy" and rel in ( + "Dockerfile", + "docker-compose.yml", + ): + pass elif base in (".env", "dockerfile", "docker-compose.yml", "compose.yml"): return "sensitive or infrastructure filename" @@ -492,7 +887,12 @@ def denied_payload_path(component, rel): if any(part in (".git", "node_modules") for part in parts): return "repository/build dependency directory" - if any(token in lower for token in ("secret", "token", "password")): + # `packages/tokens` is a versioned UI design-token package, not a secret + # store. Keep this narrowly scoped exception so the general filename guard + # remains in force for every other Module Foundry payload path. + if not (component == "module-foundry" and lower.startswith("packages/tokens/")) and any( + token in lower for token in ("secret", "token", "password") + ): return "secret-like path" runtime_prefixes = () @@ -577,6 +977,50 @@ def denied_payload_path(component, rel): return "bim-viewer env file" if rel.startswith(("docker-compose.beam.yml.bak",)): return "bim-viewer backup file" + elif component == "module-foundry": + runtime_prefixes = ( + "runtime-data", + "node_modules", + "apps/catalog/dist", + "server/data", + "server/logs", + "server/storage", + ) + if rel == ".env" or rel.startswith(".env."): + if rel != ".env.example": + return "module-foundry env file" + if rel.startswith(("Dockerfile.bak", "infra/docker-compose.module-foundry.yml.bak")): + return "module-foundry backup file" + elif component == "n8n-private-extension": + # An extension release is inert data at this boundary. Activation is + # Engine-owned and must never be smuggled into the artifact as a script, + # Compose file, environment file, pointer or mount configuration. + extension_parts = PurePosixPath(rel).parts + is_release_directory = ( + len(extension_parts) == 3 + and extension_parts[0] == "releases" + and extension_parts[1] == "n8n-nodes-ndc" + and N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(extension_parts[2]) + ) + if not is_release_directory and base not in ("package.tgz", "release.json", "rollback.json"): + return "unexpected private extension release member" + elif component == "proxy-contur": + runtime_prefixes = ( + "node_modules", + ) + if rel == ".env" or rel.startswith(".env."): + return "proxy-contur env file" + if rel.startswith(("Dockerfile.bak", "docker-compose.yml.bak")): + return "proxy-contur backup file" + elif component == "dc-amd-proxy": + runtime_prefixes = ( + "node_modules", + "runtime", + ) + if rel == ".env" or rel.startswith(".env."): + return "dc-amd-proxy env file" + if rel.startswith(("Dockerfile.bak", "docker-compose.yml.bak")): + return "dc-amd-proxy backup file" elif component == "dc-cms": runtime_prefixes = ( "admin/uploads", @@ -632,6 +1076,10 @@ def allowed_payload_path(component, rel): "nodedc-source/workers/ai-workspace-bridge/", )): return True + if rel == "nodedc-source/services/n8n/private-extensions" or rel.startswith( + "nodedc-source/services/n8n/private-extensions/" + ): + return True if rel == "nodedc-source/dist": return True @@ -653,6 +1101,7 @@ def allowed_payload_path(component, rel): if rel in ( "platform/Caddyfile.http", "platform/docker-compose.platform-http.yml", + "platform/docker-compose.external-data-plane.yml", ): return True if rel == "platform/notification-core" or rel.startswith("platform/notification-core/"): @@ -661,6 +1110,16 @@ def allowed_payload_path(component, rel): return True if rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/"): return True + if rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/"): + return True + if rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/"): + return True + if rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/"): + return True + if rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/"): + return True + if rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/"): + return True if rel == "authentik/custom-templates" or rel.startswith("authentik/custom-templates/"): return True @@ -748,6 +1207,69 @@ def allowed_payload_path(component, rel): )): return True + if component == "module-foundry": + if rel in ( + ".dockerignore", + ".env.example", + ".gitignore", + "Dockerfile", + "package.json", + "package-lock.json", + "tsconfig.base.json", + "infra/docker-compose.module-foundry.yml", + ): + return True + if rel in ("apps", "packages", "registry", "runtime-seed", "scripts", "server"): + return True + if rel.startswith(( + "apps/", + "packages/", + "registry/", + "runtime-seed/", + "scripts/", + "server/", + )): + return True + + if component == "n8n-private-extension": + parts = PurePosixPath(rel).parts + if ( + len(parts) == 3 + and parts[0] == "releases" + and parts[1] == "n8n-nodes-ndc" + and N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(parts[2]) + ): + return True + if ( + len(parts) == 4 + and parts[0] == "releases" + and parts[1] == "n8n-nodes-ndc" + and N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(parts[2]) + and parts[3] in ("package.tgz", "release.json", "rollback.json") + ): + return True + + if component == "proxy-contur": + if rel in ( + "Dockerfile", + "README.md", + "docker-compose.yml", + "package.json", + "package-lock.json", + "server.js", + ): + return True + + if component == "dc-amd-proxy": + if rel in ( + "Dockerfile", + "README.md", + "docker-compose.yml", + "package.json", + "server.mjs", + ): + return True + if component == "dc-cms": if rel in ( ".dockerignore", @@ -820,6 +1342,539 @@ def validate_payload_tree(component, payload_dir, entries): die(f"payload file is not covered by files.txt: {rel}") +def read_strict_json(path, label, max_bytes=64 * 1024): + try: + file_stat = path.lstat() + except FileNotFoundError: + die(f"{label} missing: {path}") + if stat.S_ISLNK(file_stat.st_mode) or not stat.S_ISREG(file_stat.st_mode): + die(f"{label} must be a regular file") + if file_stat.st_size < 2 or file_stat.st_size > max_bytes: + die(f"{label} has invalid size") + + def reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + die(f"{label} has duplicate key: {key}") + result[key] = value + return result + + try: + return json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=reject_duplicate_keys, + ) + except UnicodeDecodeError: + die(f"{label} is not utf-8") + except json.JSONDecodeError as exc: + die(f"{label} is invalid json: {exc.msg}") + + +def require_exact_json_keys(value, expected, label): + if not isinstance(value, dict): + die(f"{label} must be an object") + actual = set(value) + if actual != set(expected): + die(f"{label} keys mismatch: expected={sorted(expected)} actual={sorted(actual)}") + + +def validate_n8n_package_tarball(package_path, release): + package_stat = package_path.lstat() + if stat.S_ISLNK(package_stat.st_mode) or not stat.S_ISREG(package_stat.st_mode): + die("private extension package must be a regular file") + if package_stat.st_size < 1024 or package_stat.st_size > 32 * 1024 * 1024: + die("private extension package has invalid size") + if sha256_file(package_path) != release["package"]["sha256"]: + die("private extension package sha256 mismatch") + if package_stat.st_size != release["package"]["bytes"]: + die("private extension package byte count mismatch") + + names = set() + package_bytes = 0 + package_json_bytes = None + packed_node_sources = {} + with tarfile.open(package_path, "r:gz") as archive: + for member_count, member in enumerate(archive, 1): + if member_count > 512: + die("private extension package has too many members") + validate_posix_path(member.name) + if member.name in names: + die(f"duplicate private extension package member: {member.name}") + names.add(member.name) + if member.name != "package" and not member.name.startswith("package/"): + die(f"private extension package member escaped package root: {member.name}") + if not (member.isfile() or member.isdir()): + die(f"unsupported private extension package member: {member.name}") + if member.name == "package": + if not member.isdir(): + die("private extension package root must be a directory") + else: + package_rel = PurePosixPath(member.name).relative_to("package") + if any(part.startswith(".") or part.startswith("._") for part in package_rel.parts): + die(f"private extension package hidden member rejected: {member.name}") + if not ( + package_rel.as_posix() in ("README.md", "package.json") + or package_rel.as_posix().startswith("dist/") + ): + die(f"private extension package member is not allowlisted: {member.name}") + expected_mode = 0o644 if member.isfile() else 0o755 + if stat.S_IMODE(member.mode) != expected_mode: + die(f"unsafe private extension package mode: {member.name}") + if member.isfile(): + package_bytes += member.size + if member.size > 4 * 1024 * 1024 or package_bytes > 64 * 1024 * 1024: + die("private extension package payload is too large") + if member.name == "package/package.json": + source = archive.extractfile(member) + if source is None: + die("private extension package.json cannot be read") + package_json_bytes = source.read(64 * 1024 + 1) + if member.name.startswith("package/dist/nodes/") and member.name.endswith(".node.js"): + source = archive.extractfile(member) + if source is None: + die(f"private extension packed node cannot be read: {member.name}") + packed_node_sources[member.name[len("package/"):]] = source.read(4 * 1024 * 1024 + 1) + + if package_json_bytes is None or len(package_json_bytes) > 64 * 1024: + die("private extension package.json missing or too large") + try: + package_json = json.loads(package_json_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + die("private extension package.json is invalid") + if package_json.get("name") != "n8n-nodes-ndc": + die("private extension package name mismatch") + if package_json.get("version") != release["package"]["version"]: + die("private extension package version mismatch") + if package_json.get("private") is not True: + die("private extension package must remain private") + if package_json.get("dependencies") not in (None, {}): + die("private extension runtime dependencies are forbidden") + scripts = package_json.get("scripts") or {} + if not isinstance(scripts, dict): + die("private extension package scripts must be an object") + for lifecycle in ("preinstall", "install", "postinstall", "prepack", "prepare", "postpack"): + if lifecycle in scripts: + die(f"private extension lifecycle script is forbidden: {lifecycle}") + + n8n = package_json.get("n8n") + if not isinstance(n8n, dict): + die("private extension n8n registration missing") + registered = [] + expected_registrations = { + "nodes": N8N_PRIVATE_EXTENSION_NODES, + "credentials": N8N_PRIVATE_EXTENSION_CREDENTIALS, + } + for field, expected in expected_registrations.items(): + values = n8n.get(field) + if not isinstance(values, list) or not values: + die(f"private extension n8n.{field} registration missing") + if values != list(expected): + die(f"private extension n8n.{field} registration mismatch") + for value in values: + if not isinstance(value, str) or not value.startswith("dist/") or value not in names and f"package/{value}" not in names: + die(f"private extension n8n.{field} member missing: {value}") + registered.append(value) + if len(registered) != len(set(registered)): + die("private extension n8n registration contains duplicates") + packed_nodes = sorted( + name[len("package/"):] + for name in names + if name.startswith("package/dist/nodes/") and name.endswith(".node.js") + ) + packed_credentials = sorted( + name[len("package/"):] + for name in names + if name.startswith("package/dist/credentials/") and name.endswith(".credentials.js") + ) + if packed_nodes != sorted(N8N_PRIVATE_EXTENSION_NODES): + die("private extension packed node set mismatch") + if packed_credentials != sorted(N8N_PRIVATE_EXTENSION_CREDENTIALS): + die("private extension packed credential set mismatch") + for node_path in N8N_PRIVATE_EXTENSION_NODES: + source = packed_node_sources.get(node_path) + if source is None or len(source) > 4 * 1024 * 1024: + die(f"private extension packed node source missing or too large: {node_path}") + if b"usableAsTool" in source: + die(f"private extension packed node would generate a tool variant: {node_path}") + + +def validate_n8n_private_extension_release(payload_dir, entries): + if len(entries) != 1: + die("private extension artifact must contain exactly one immutable release entry") + release_rel = entries[0] + parts = PurePosixPath(release_rel).parts + if ( + len(parts) != 3 + or parts[0] != "releases" + or parts[1] != "n8n-nodes-ndc" + or not N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(parts[2]) + ): + die("private extension files.txt entry must be a digest-bound release directory") + + release_dir = payload_dir / release_rel + expected_files = {"package.tgz", "release.json", "rollback.json"} + actual_files = {path.name for path in release_dir.iterdir() if path.is_file()} + actual_dirs = [path.name for path in release_dir.iterdir() if path.is_dir()] + if actual_dirs or actual_files != expected_files: + die("private extension release must contain only package.tgz, release.json and rollback.json") + + release = read_strict_json(release_dir / "release.json", "private extension release manifest") + require_exact_json_keys(release, ("schemaVersion", "releaseId", "package", "storage", "activation"), "private extension release manifest") + require_exact_json_keys(release["package"], ("name", "version", "sha256", "bytes", "runtimeTypePrefix"), "private extension package manifest") + require_exact_json_keys(release["storage"], ("relativePath", "immutable"), "private extension storage manifest") + require_exact_json_keys( + release["activation"], + ( + "owner", + "status", + "requiredCommunityPackagePath", + "requiresAtomicReleaseSwitch", + "requiresAllN8nProcessesRestart", + "requiresMcpSchemaAcceptance", + "rollbackBaselinePolicy", + ), + "private extension activation manifest", + ) + require_exact_json_keys( + release["activation"]["rollbackBaselinePolicy"], + ("allowed", "firstActivation", "requiresPreActivationVerification"), + "private extension rollback baseline policy", + ) + + release_id = parts[2] + package = release["package"] + activation = release["activation"] + if release["schemaVersion"] != "nodedc.n8n-private-extension-release/v2": + die("private extension release schema mismatch") + if release["releaseId"] != release_id: + die("private extension release id mismatch") + if package["name"] != "n8n-nodes-ndc" or package["runtimeTypePrefix"] != "n8n-nodes-ndc.": + die("private extension package identity mismatch") + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", str(package["version"])): + die("private extension package version is invalid") + if not re.fullmatch(r"[a-f0-9]{64}", str(package["sha256"])): + die("private extension package sha256 is invalid") + if not isinstance(package["bytes"], int) or isinstance(package["bytes"], bool): + die("private extension package byte count is invalid") + if release_id != f"{package['version']}-{package['sha256'][:16]}": + die("private extension release id is not digest-bound") + if release["storage"] != {"relativePath": release_rel, "immutable": True}: + die("private extension storage manifest mismatch") + rollback_baseline_policy = { + "allowed": [ + "previous_verified_immutable_release", + "verified_inactive", + ], + "firstActivation": "verified_inactive", + "requiresPreActivationVerification": True, + } + if activation != { + "owner": "engine", + "status": "blocked_pending_engine_owned_mount", + "requiredCommunityPackagePath": "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc", + "requiresAtomicReleaseSwitch": True, + "requiresAllN8nProcessesRestart": True, + "requiresMcpSchemaAcceptance": True, + "rollbackBaselinePolicy": rollback_baseline_policy, + }: + die("private extension activation boundary mismatch") + + rollback = read_strict_json(release_dir / "rollback.json", "private extension rollback manifest") + require_exact_json_keys( + rollback, + ("schemaVersion", "releaseId", "packageSha256", "mode", "baselinePolicy", "steps", "forbidden"), + "private extension rollback manifest", + ) + require_exact_json_keys( + rollback["baselinePolicy"], + ("allowed", "firstActivation", "requiresPreActivationVerification"), + "private extension rollback manifest baseline policy", + ) + if rollback != { + "schemaVersion": "nodedc.n8n-private-extension-rollback/v2", + "releaseId": release_id, + "packageSha256": package["sha256"], + "mode": "engine-owned-atomic-release-switch", + "baselinePolicy": rollback_baseline_policy, + "steps": [ + "select_verified_previous_release_or_preverified_inactive_baseline", + "switch_engine_owned_mount_atomically", + "restart_all_n8n_processes", + "verify_mcp_schema_state_matches_selected_baseline", + ], + "forbidden": [ + "delete_active_release", + "mutate_engine_core", + "live_npm_install", + ], + }: + die("private extension rollback manifest mismatch") + + validate_n8n_package_tarball(release_dir / "package.tgz", release) + + +def is_engine_n8n_transition(component, entries): + return component == "engine" and entries is not None and ENGINE_N8N_TRANSITION_DESCRIPTOR_REL in entries + + +def read_engine_n8n_transition_descriptor(path, label="Engine n8n transition descriptor"): + descriptor = read_strict_json(path, label) + require_exact_json_keys( + descriptor, + ( + "schemaVersion", + "action", + "releaseId", + "packageVersion", + "packageSha256", + "n8nVersion", + "baseImage", + "baseImageArchitecture", + "baseImageIdentityPolicy", + "sealedReleaseRelativePath", + "composeOverride", + "runtimePackagePath", + "topologyServices", + "expectedCurrent", + "expectedNodeTypes", + "expectedCredentialTypes", + "rollbackBaseline", + ), + label, + ) + action = descriptor.get("action") + if descriptor.get("schemaVersion") != "nodedc.engine-n8n-private-extension-transition/v1": + die("Engine n8n transition schema mismatch") + if action not in ("activate", "rollback-inactive"): + die("Engine n8n transition action mismatch") + release_id = descriptor.get("releaseId") + package_version = descriptor.get("packageVersion") + package_sha256 = descriptor.get("packageSha256") + if not isinstance(release_id, str) or not N8N_PRIVATE_EXTENSION_RELEASE_RE.fullmatch(release_id): + die("Engine n8n transition release id is invalid") + if not isinstance(package_version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", package_version): + die("Engine n8n transition package version is invalid") + if not isinstance(package_sha256, str) or not re.fullmatch(r"[a-f0-9]{64}", package_sha256): + die("Engine n8n transition package sha256 is invalid") + if release_id != f"{package_version}-{package_sha256[:16]}": + die("Engine n8n transition release identity mismatch") + expected_sealed_path = f"n8n-private-extensions/releases/n8n-nodes-ndc/{release_id}/package" + exact_common = { + "n8nVersion": ENGINE_N8N_VERSION, + "baseImage": ENGINE_N8N_BASE_IMAGE, + "baseImageArchitecture": ENGINE_N8N_BASE_ARCHITECTURE, + "baseImageIdentityPolicy": "running-container-and-local-tag-must-match", + "sealedReleaseRelativePath": expected_sealed_path, + "composeOverride": ENGINE_N8N_COMPOSE_OVERRIDE_REL, + "runtimePackagePath": ENGINE_N8N_RUNTIME_PACKAGE_PATH, + "topologyServices": ["n8n"], + "rollbackBaseline": "verified_inactive", + } + for key, expected in exact_common.items(): + if descriptor.get(key) != expected: + die(f"Engine n8n transition {key} mismatch") + if action == "activate": + if descriptor.get("expectedCurrent") != "verified_inactive": + die("Engine n8n activation must start from the verified inactive baseline") + if descriptor.get("expectedNodeTypes") != list(ENGINE_N8N_NODE_TYPES): + die("Engine n8n activation node type set mismatch") + if descriptor.get("expectedCredentialTypes") != list(ENGINE_N8N_CREDENTIAL_TYPES): + die("Engine n8n activation credential type set mismatch") + else: + if descriptor.get("expectedCurrent") != release_id: + die("Engine n8n rollback expected-current release mismatch") + if descriptor.get("expectedNodeTypes") != [] or descriptor.get("expectedCredentialTypes") != []: + die("Engine n8n rollback must select the inactive catalog") + return descriptor + + +def expected_engine_n8n_compose_override(descriptor): + health_script = ( + "const http=require('http');const req=http.get('http://127.0.0.1:5678/healthz/readiness'," + "r=>{r.resume();process.exit(r.statusCode===200?0:1)});req.on('error',()=>process.exit(1));" + "req.setTimeout(4000,()=>{req.destroy();process.exit(1)});" + ) + health_test = json.dumps(["CMD", "node", "-e", health_script], separators=(",", ":")) + mount_source = f"/volume2/nodedc-demo/{descriptor['sealedReleaseRelativePath']}" + return "\n".join(( + "services:", + " n8n:", + f" image: {ENGINE_N8N_BASE_IMAGE}", + " platform: linux/amd64", + " pull_policy: never", + " environment:", + " N8N_USER_FOLDER: /home/node", + ' N8N_COMMUNITY_PACKAGES_ENABLED: "true"', + ' N8N_COMMUNITY_PACKAGES_PREVENT_LOADING: "false"', + ' N8N_REINSTALL_MISSING_PACKAGES: "false"', + " volumes:", + f" - {mount_source}:{ENGINE_N8N_RUNTIME_PACKAGE_PATH}:ro", + " healthcheck:", + f" test: {health_test}", + " interval: 10s", + " timeout: 5s", + " retries: 30", + " start_period: 30s", + " labels:", + f" nodedc.n8n-private-extension.release: {descriptor['releaseId']}", + f" nodedc.n8n-private-extension.package-sha256: {descriptor['packageSha256']}", + "", + )) + + +def validate_engine_n8n_catalog_payload(payload_dir, descriptor): + action = descriptor["action"] + nodes_path = payload_dir / ENGINE_N8N_NODES_CATALOG_REL + credentials_path = payload_dir / ENGINE_N8N_CREDENTIALS_CATALOG_REL + meta_path = payload_dir / ENGINE_N8N_SCHEMA_META_REL + nodes = read_strict_json( + nodes_path, + "Engine n8n node catalog", + max_bytes=32 * 1024 * 1024, + ) + credentials = read_strict_json( + credentials_path, + "Engine n8n credential catalog", + max_bytes=4 * 1024 * 1024, + ) + meta = read_strict_json(meta_path, "Engine n8n schema metadata") + if not isinstance(nodes, list) or not isinstance(credentials, list): + die("Engine n8n schema catalogs must be arrays") + private_nodes = [ + item for item in nodes + if isinstance(item, dict) and str(item.get("name") or "").startswith("n8n-nodes-ndc.") + ] + private_credentials = [ + item for item in credentials + if isinstance(item, dict) and str(item.get("name") or "") in ENGINE_N8N_CREDENTIAL_TYPES + ] + expected_node_types = list(ENGINE_N8N_NODE_TYPES) if action == "activate" else [] + expected_credential_types = list(ENGINE_N8N_CREDENTIAL_TYPES) if action == "activate" else [] + if [item.get("name") for item in private_nodes] != expected_node_types: + die("Engine n8n pinned node catalog exact set mismatch") + if [item.get("name") for item in private_credentials] != expected_credential_types: + die("Engine n8n pinned credential catalog exact set mismatch") + if any("usableAsTool" in item for item in private_nodes): + die("Engine n8n pinned node catalog would generate tool variants") + if action == "activate": + if len(nodes) != 437 or len(credentials) != 388: + die("Engine n8n activation catalog count mismatch") + if any(not str(item.get("displayName") or "").startswith("NDC ") for item in private_nodes): + die("Engine n8n private node visible name mismatch") + if any(not str(item.get("displayName") or "").startswith("NDC ") for item in private_credentials): + die("Engine n8n private credential visible name mismatch") + private_catalog_text = json.dumps( + {"nodes": private_nodes, "credentials": private_credentials}, + ensure_ascii=False, + ).lower() + if "gelios" in private_catalog_text or "robot2b" in private_catalog_text: + die("provider business identity is forbidden in the Engine n8n extension catalog") + else: + if len(nodes) != 434 or len(credentials) != 385: + die("Engine n8n inactive baseline catalog count mismatch") + if not isinstance(meta, dict): + die("Engine n8n schema metadata must be an object") + if meta.get("n8nVersion") != ENGINE_N8N_VERSION: + die("Engine n8n schema metadata version mismatch") + if meta.get("nodeCount") != len(nodes) or meta.get("credentialCount") != len(credentials): + die("Engine n8n schema metadata count mismatch") + if action == "activate": + if meta.get("source") != f"n8n-core+n8n-nodes-ndc@{descriptor['packageVersion']}": + die("Engine n8n schema metadata source mismatch") + if sha256_file(payload_dir / ENGINE_N8N_ICON_REL) != ENGINE_N8N_ICON_SHA256: + die("Engine n8n light icon sha256 mismatch") + if sha256_file(payload_dir / ENGINE_N8N_DARK_ICON_REL) != ENGINE_N8N_DARK_ICON_SHA256: + die("Engine n8n dark icon sha256 mismatch") + expected_catalog_hashes = ( + (nodes, ENGINE_N8N_ACTIVATION_NODES_CATALOG_JSON_SHA256, "node"), + (credentials, ENGINE_N8N_ACTIVATION_CREDENTIALS_CATALOG_JSON_SHA256, "credential"), + (meta, ENGINE_N8N_ACTIVATION_META_JSON_SHA256, "metadata"), + ) + else: + expected_catalog_hashes = ( + (nodes, ENGINE_N8N_INACTIVE_NODES_CATALOG_JSON_SHA256, "node"), + (credentials, ENGINE_N8N_INACTIVE_CREDENTIALS_CATALOG_JSON_SHA256, "credential"), + (meta, ENGINE_N8N_INACTIVE_META_JSON_SHA256, "metadata"), + ) + for catalog, expected_sha256, label in expected_catalog_hashes: + if sha256_json_value(catalog) != expected_sha256: + die(f"Engine n8n {action} {label} catalog sha256 mismatch") + + +def validate_engine_n8n_transition(payload_dir, entries): + descriptor = read_engine_n8n_transition_descriptor(payload_dir / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL) + activation_entries = [ + ENGINE_N8N_TRANSITION_DESCRIPTOR_REL, + ENGINE_N8N_COMPOSE_OVERRIDE_REL, + ENGINE_N8N_NODES_CATALOG_REL, + ENGINE_N8N_CREDENTIALS_CATALOG_REL, + ENGINE_N8N_SCHEMA_META_REL, + ENGINE_N8N_ICON_REL, + ENGINE_N8N_DARK_ICON_REL, + ] + rollback_entries = [ + ENGINE_N8N_TRANSITION_DESCRIPTOR_REL, + ENGINE_N8N_NODES_CATALOG_REL, + ENGINE_N8N_CREDENTIALS_CATALOG_REL, + ENGINE_N8N_SCHEMA_META_REL, + ] + expected_entries = activation_entries if descriptor["action"] == "activate" else rollback_entries + if entries != expected_entries: + die("Engine n8n transition files.txt exact set/order mismatch") + if descriptor["action"] == "activate": + override = payload_dir / ENGINE_N8N_COMPOSE_OVERRIDE_REL + try: + override_text = override.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + die("Engine n8n Compose override cannot be read") + if override_text != expected_engine_n8n_compose_override(descriptor): + die("Engine n8n Compose override mismatch") + if "N8N_CUSTOM_EXTENSIONS" in override_text or "CUSTOM." in override_text: + die("Engine n8n custom-extension loader is forbidden") + validate_engine_n8n_catalog_payload(payload_dir, descriptor) + return descriptor + + +def current_engine_n8n_transition_descriptor(): + path = component_root("engine") / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL + if not path.exists(): + return None + if path.is_symlink() or not path.is_file(): + die("installed Engine n8n transition descriptor is unsafe") + return read_engine_n8n_transition_descriptor(path, "installed Engine n8n transition descriptor") + + +def validate_engine_n8n_staged_release(descriptor): + release_rel = f"releases/n8n-nodes-ndc/{descriptor['releaseId']}" + release_dir = N8N_PRIVATE_EXTENSION_RELEASES_ROOT / release_rel + current = N8N_PRIVATE_EXTENSION_RELEASES_ROOT + for part in PurePosixPath(release_rel).parts: + current = current / part + try: + current_stat = current.lstat() + except FileNotFoundError: + die(f"staged private extension release is missing: {release_rel}") + if stat.S_ISLNK(current_stat.st_mode): + die(f"staged private extension release symlink rejected: {current}") + validate_n8n_private_extension_release(N8N_PRIVATE_EXTENSION_RELEASES_ROOT, [release_rel]) + release = read_strict_json(release_dir / "release.json", "staged private extension release manifest") + if release.get("releaseId") != descriptor["releaseId"]: + die("staged private extension release id mismatch") + if release.get("package", {}).get("version") != descriptor["packageVersion"]: + die("staged private extension package version mismatch") + if release.get("package", {}).get("sha256") != descriptor["packageSha256"]: + die("staged private extension package sha256 mismatch") + for path in [release_dir, *release_dir.rglob("*")]: + path_stat = path.lstat() + if path_stat.st_uid != 0 or path_stat.st_gid != 0: + die(f"staged private extension release ownership mismatch: {path}") + expected_mode = 0o555 if stat.S_ISDIR(path_stat.st_mode) else 0o444 + if stat.S_IMODE(path_stat.st_mode) != expected_mode: + die(f"staged private extension release mode mismatch: {path}") + return release_dir + + def load_artifact(artifact, work_dir): safe_extract(artifact, work_dir) @@ -843,6 +1898,18 @@ def load_artifact(artifact, work_dir): die(f"files.txt entry missing in payload: {rel}") validate_payload_tree(manifest["component"], payload_dir, entries) + if manifest["component"] == "n8n-private-extension": + validate_n8n_private_extension_release(payload_dir, entries) + if manifest["component"] == "engine": + touches_private_extension = any( + rel == "nodedc-source/services/n8n/private-extensions" + or rel.startswith("nodedc-source/services/n8n/private-extensions/") + for rel in entries + ) + if touches_private_extension and not is_engine_n8n_transition(manifest["component"], entries): + die("Engine private-extension payload requires the canonical transition descriptor") + if is_engine_n8n_transition(manifest["component"], entries): + validate_engine_n8n_transition(payload_dir, entries) return manifest, entries, payload_dir @@ -898,7 +1965,17 @@ def component_compose_project(component): return COMPONENTS[component].get("compose_project") +def component_artifact_only(component): + return bool(COMPONENTS[component].get("artifact_only")) + + def component_services(component, entries=None): + if is_engine_n8n_transition(component, entries): + # The actual Engine topology has one n8n process and no separately + # deployed worker/webhook services. The release barrier therefore + # force-recreates exactly this service and never touches its database. + return ("n8n",) + if component == "dc-cms" and entries is not None: selected = [] @@ -951,14 +2028,28 @@ def component_services(component, entries=None): touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries) touches_ai_workspace = any(rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/") for rel in entries) touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries) + touches_ontology = any(rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/") for rel in entries) + touches_gelios = any(rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/") for rel in entries) + touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries) + touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries) touches_authentik = any(rel == "authentik/custom-templates" or rel.startswith("authentik/custom-templates/") for rel in entries) + map_gateway_only_compose = touches_compose and touches_map_gateway and not any((touches_notification, touches_ai_workspace, touches_ai_workspace_assistant, touches_ontology, touches_gelios, touches_external_data_plane, touches_authentik)) + restart_all_for_compose = touches_compose and not map_gateway_only_compose - if touches_notification or touches_compose: + if touches_notification or restart_all_for_compose: add("notification-postgres", "notification-core", "launcher", "reverse-proxy") - if touches_ai_workspace or touches_compose: + if touches_ai_workspace or restart_all_for_compose: add("ai-workspace-hub", "reverse-proxy") - if touches_ai_workspace_assistant or touches_compose: + if touches_ai_workspace_assistant or restart_all_for_compose: add("ai-workspace-postgres", "ai-workspace-assistant") + if touches_ontology or restart_all_for_compose: + add("ontology-core", "ai-workspace-hub") + if touches_gelios or restart_all_for_compose: + add("gelios-postgres", "gelios-gateway") + if touches_map_gateway: + add("map-gateway") + if touches_external_data_plane: + add("external-data-plane-postgres", "external-data-plane") if touches_authentik: add("authentik-server", "authentik-worker", "reverse-proxy") if touches_caddy: @@ -981,6 +2072,22 @@ def component_compose_no_deps(component, entries=None): def component_compose_files(component): + if component == "engine": + root = component_root(component) + files = [root / "docker-compose.yml"] + descriptor = current_engine_n8n_transition_descriptor() + if descriptor and descriptor["action"] == "activate": + override = root / ENGINE_N8N_COMPOSE_OVERRIDE_REL + if override.is_symlink() or not override.is_file(): + die("active Engine n8n Compose override is missing or unsafe") + try: + override_text = override.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + die("active Engine n8n Compose override cannot be read") + if override_text != expected_engine_n8n_compose_override(descriptor): + die("installed Engine n8n Compose override drift detected") + files.append(override) + return tuple(files) return COMPONENTS[component].get("compose_files", ()) @@ -1010,22 +2117,48 @@ def component_builds(component, entries=None): touches_notification = any(rel == "platform/notification-core" or rel.startswith("platform/notification-core/") for rel in entries) touches_ai_workspace = any(rel == "platform/ai-workspace-hub" or rel.startswith("platform/ai-workspace-hub/") for rel in entries) touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries) + touches_ontology = any(rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/") for rel in entries) + touches_gelios = any(rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/") for rel in entries) + touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries) + touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries) + map_gateway_only_compose = touches_compose and touches_map_gateway and not any((touches_notification, touches_ai_workspace, touches_ai_workspace_assistant, touches_ontology, touches_gelios, touches_external_data_plane)) + rebuild_all_for_compose = touches_compose and not map_gateway_only_compose builds = [] - if touches_notification or touches_compose: + if touches_notification or rebuild_all_for_compose: builds.append(( Path("/volume1/docker/nodedc-platform/platform/notification-core"), ("build", "--no-cache", "-t", "nodedc/notification-core:local", "."), )) - if touches_ai_workspace or touches_compose: + if touches_ai_workspace or rebuild_all_for_compose: builds.append(( Path("/volume1/docker/nodedc-platform/platform/ai-workspace-hub"), ("build", "--no-cache", "-t", "nodedc/ai-workspace-hub:local", "."), )) - if touches_ai_workspace_assistant or touches_compose: + if touches_ai_workspace_assistant or rebuild_all_for_compose: builds.append(( Path("/volume1/docker/nodedc-platform/platform/ai-workspace-assistant"), ("build", "--no-cache", "-t", "nodedc/ai-workspace-assistant:local", "."), )) + if touches_ontology or rebuild_all_for_compose: + builds.append(( + Path("/volume1/docker/nodedc-platform/platform/ontology-core"), + ("build", "--no-cache", "-t", "nodedc/ontology-core:local", "."), + )) + if touches_gelios or rebuild_all_for_compose: + builds.append(( + Path("/volume1/docker/nodedc-platform/platform/gelios-gateway"), + ("build", "--no-cache", "-t", "nodedc/gelios-gateway:local", "."), + )) + if touches_map_gateway: + builds.append(( + Path("/volume1/docker/nodedc-platform/platform/services/map-gateway"), + ("build", "--no-cache", "-t", "nodedc/map-gateway:local", "."), + )) + if touches_external_data_plane: + builds.append(( + Path("/volume1/docker/nodedc-platform/platform"), + ("build", "--no-cache", "-f", "services/external-data-plane/Dockerfile", "-t", "nodedc/external-data-plane:local", "."), + )) return tuple(builds) if component == "tasker": @@ -1068,13 +2201,462 @@ def component_builds(component, entries=None): return () +def docker_json(args, label): + result = subprocess.run( + [str(DOCKER), *args], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + die(f"{label} failed: exit={result.returncode}") + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + die(f"{label} returned invalid JSON") + + +def engine_n8n_container_ids(): + result = subprocess.run( + [*compose_base_cmd("engine"), "ps", "-q", "n8n"], + cwd=str(component_compose_root("engine")), + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + die("Engine n8n container lookup failed") + container_ids = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if len(container_ids) != 1: + die(f"Engine n8n topology mismatch: expected=1 actual={len(container_ids)}") + return container_ids + + +def inspect_engine_n8n_base_image(): + images = docker_json(["image", "inspect", ENGINE_N8N_BASE_IMAGE], "Engine n8n base image inspect") + if not isinstance(images, list) or len(images) != 1 or not isinstance(images[0], dict): + die("Engine n8n base image inspect shape mismatch") + image = images[0] + image_id = image.get("Id") + if not isinstance(image_id, str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", image_id): + die("Engine n8n base image id is invalid") + if image.get("Architecture") != ENGINE_N8N_BASE_ARCHITECTURE or image.get("Os") != "linux": + die("Engine n8n base image platform mismatch") + return { + "id": image_id, + "architecture": image.get("Architecture"), + "repo_digests": tuple(image.get("RepoDigests") or ()), + } + + +def inspect_container(container_id): + containers = docker_json(["inspect", container_id], "Engine n8n container inspect") + if not isinstance(containers, list) or len(containers) != 1 or not isinstance(containers[0], dict): + die("Engine n8n container inspect shape mismatch") + return containers[0] + + +def engine_n8n_version(container_id): + result = subprocess.run( + [str(DOCKER), "exec", container_id, "n8n", "--version"], + check=False, + capture_output=True, + text=True, + timeout=60, + ) + version = result.stdout.strip() + if result.returncode != 0 or version != ENGINE_N8N_VERSION: + die("Engine n8n runtime version mismatch") + return version + + +def engine_n8n_private_loader_catalog(container_id): + container = inspect_container(container_id) + package_mounts = [ + mount for mount in (container.get("Mounts") or ()) + if mount.get("Destination") == ENGINE_N8N_RUNTIME_PACKAGE_PATH + ] + if not package_mounts: + return {"node_types": [], "credential_types": []} + if len(package_mounts) != 1 or package_mounts[0].get("RW") is not False: + die("Engine n8n package-loader probe found an unsafe package mount") + script = ( + "const {PackageDirectoryLoader}=require('/usr/local/lib/node_modules/n8n/node_modules/n8n-core');" + "(async()=>{const loader=new PackageDirectoryLoader('" + + ENGINE_N8N_RUNTIME_PACKAGE_PATH + + "');await loader.loadAll();process.stdout.write(JSON.stringify({packageName:loader.packageName," + "nodes:loader.types.nodes.map(x=>x.name),credentials:loader.types.credentials.map(x=>x.name)}))})()" + ".catch(()=>process.exit(1));" + ) + result = subprocess.run( + [str(DOCKER), "exec", container_id, "node", "-e", script], + check=False, + capture_output=True, + text=True, + timeout=120, + ) + if result.returncode != 0: + die("Engine n8n package-loader probe failed") + try: + catalog = json.loads(result.stdout) + except json.JSONDecodeError: + die("Engine n8n package-loader probe returned invalid JSON") + require_exact_json_keys(catalog, ("packageName", "nodes", "credentials"), "Engine n8n package-loader probe") + if catalog.get("packageName") != "n8n-nodes-ndc": + die("Engine n8n package-loader probe package identity mismatch") + nodes = catalog.get("nodes") + credentials = catalog.get("credentials") + if not isinstance(nodes, list) or not all(isinstance(value, str) for value in nodes): + die("Engine n8n package-loader probe node set is invalid") + if not isinstance(credentials, list) or not all(isinstance(value, str) for value in credentials): + die("Engine n8n package-loader probe credential set is invalid") + return { + "node_types": [f"n8n-nodes-ndc.{value}" for value in nodes], + "credential_types": credentials, + } + + +def engine_n8n_current_state(): + descriptor = current_engine_n8n_transition_descriptor() + if descriptor and descriptor["action"] == "activate": + return descriptor["releaseId"], descriptor + return "verified_inactive", descriptor + + +def validate_engine_n8n_base_compose_source(): + compose_path = component_root("engine") / "docker-compose.yml" + try: + compose = compose_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + die("Engine base Compose cannot be read") + services = re.findall(r"(?m)^ ([A-Za-z0-9_-]+):\s*$", compose) + if tuple(services) != ENGINE_BASE_COMPOSE_SERVICES: + die("Engine base Compose exact service topology mismatch") + match = re.search(r"(?ms)^ n8n:\s*\n(?P.*?)(?=^ [A-Za-z0-9_-]+:\s*$)", compose) + if not match: + die("Engine base Compose n8n service cannot be isolated") + n8n_service = match.group("body") + expected_image = "image: docker.n8n.io/n8nio/n8n:${N8N_IMAGE_TAG:-2.3.2}" + if expected_image not in n8n_service: + die("Engine base Compose n8n version mismatch") + required_fragments = ( + "./n8n-data:/home/node/.n8n", + "./nodedc-source/services/n8n/runtime-plugin:/opt/nodedc/runtime-plugin:ro", + "n8n-postgres:", + ) + if any(fragment not in n8n_service for fragment in required_fragments): + die("Engine base Compose n8n persistence/runtime dependency mismatch") + forbidden_fragments = ( + "build:", + "N8N_CUSTOM_EXTENSIONS", + "CUSTOM.", + ENGINE_N8N_RUNTIME_PACKAGE_PATH, + "N8N_USER_FOLDER", + "N8N_COMMUNITY_PACKAGES_ENABLED", + "N8N_COMMUNITY_PACKAGES_PREVENT_LOADING", + "N8N_REINSTALL_MISSING_PACKAGES", + ) + if any(fragment in n8n_service for fragment in forbidden_fragments): + die("Engine base Compose contains a non-baseline n8n activation setting") + + +def preflight_engine_n8n_transition(descriptor, enforce_expected_current): + validate_engine_n8n_base_compose_source() + if descriptor["action"] == "activate": + validate_engine_n8n_staged_release(descriptor) + base_image = inspect_engine_n8n_base_image() + container_id = engine_n8n_container_ids()[0] + container = inspect_container(container_id) + state = container.get("State") or {} + if state.get("Status") != "running": + die("Engine n8n baseline container is not running") + if container.get("Image") != base_image["id"]: + die("Engine n8n running image does not match the local 2.3.2 base image") + version = engine_n8n_version(container_id) + current_state, current_descriptor = engine_n8n_current_state() + state_matches = current_state == descriptor["expectedCurrent"] + current_catalog = engine_n8n_private_loader_catalog(container_id) + current_types = current_catalog["node_types"] + current_credentials = current_catalog["credential_types"] + expected_current_types = list(ENGINE_N8N_NODE_TYPES) if current_state != "verified_inactive" else [] + expected_current_credentials = list(ENGINE_N8N_CREDENTIAL_TYPES) if current_state != "verified_inactive" else [] + if current_types != expected_current_types: + die("Engine n8n current live private-node set does not match its transition state") + if current_credentials != expected_current_credentials: + die("Engine n8n current live private-credential set does not match its transition state") + catalog_descriptor = ( + current_descriptor + if current_state != "verified_inactive" and current_descriptor is not None + else inactive_engine_n8n_descriptor(descriptor) + ) + validate_engine_n8n_catalog_payload(component_root("engine"), catalog_descriptor) + if enforce_expected_current and not state_matches: + die( + "Engine n8n transition stale current state: " + f"expected={descriptor['expectedCurrent']} actual={current_state}" + ) + if current_descriptor and current_state != "verified_inactive": + if current_descriptor.get("packageSha256") != descriptor["packageSha256"]: + if enforce_expected_current: + die("Engine n8n current release package sha256 mismatch") + return { + "base_image_id": base_image["id"], + "base_image_architecture": base_image["architecture"], + "base_image_repo_digests": base_image["repo_digests"], + "container_id": container_id, + "current_state": current_state, + "current_types": current_types, + "current_credentials": current_credentials, + "n8n_version": version, + "state_matches": state_matches, + } + + +def add_engine_n8n_sealed_member_paths(expected_paths, member_name): + parts = PurePosixPath(member_name).parts + if not parts or parts[0] != "package": + die("Engine sealed n8n package member root mismatch") + for depth in range(1, len(parts) + 1): + expected_paths.add(PurePosixPath(*parts[:depth]).as_posix()) + + +def validate_engine_n8n_sealed_release(release_dir, staged_release_dir, descriptor): + if release_dir.is_symlink() or not release_dir.is_dir(): + die("Engine sealed n8n release is missing or unsafe") + for filename in ("package.tgz", "release.json", "rollback.json"): + installed = release_dir / filename + staged = staged_release_dir / filename + if installed.is_symlink() or not installed.is_file(): + die(f"Engine sealed n8n release member is unsafe: {filename}") + if sha256_file(installed) != sha256_file(staged): + die(f"Engine sealed n8n release member mismatch: {filename}") + package_root = release_dir / "package" + if package_root.is_symlink() or not package_root.is_dir(): + die("Engine sealed n8n package tree is missing or unsafe") + expected_paths = {"package", "package.tgz", "release.json", "rollback.json"} + with tarfile.open(staged_release_dir / "package.tgz", "r:gz") as archive: + for member in archive: + # npm package tarballs do not have to carry explicit directory + # members. Extraction still creates those parent directories, so + # they are part of the exact sealed tree rather than unexpected + # content. + add_engine_n8n_sealed_member_paths(expected_paths, member.name) + target = release_dir.joinpath(*PurePosixPath(member.name).parts) + target_stat = target.lstat() + if stat.S_ISLNK(target_stat.st_mode): + die(f"Engine sealed n8n package symlink rejected: {member.name}") + if member.isdir(): + if not stat.S_ISDIR(target_stat.st_mode): + die(f"Engine sealed n8n package directory mismatch: {member.name}") + else: + if not stat.S_ISREG(target_stat.st_mode) or target_stat.st_size != member.size: + die(f"Engine sealed n8n package file mismatch: {member.name}") + source = archive.extractfile(member) + if source is None: + die(f"Engine sealed n8n package file unreadable: {member.name}") + import hashlib + expected_sha = hashlib.sha256(source.read()).hexdigest() + if sha256_file(target) != expected_sha: + die(f"Engine sealed n8n package content mismatch: {member.name}") + actual_paths = { + path.relative_to(release_dir).as_posix() + for path in release_dir.rglob("*") + } + if actual_paths != expected_paths: + die("Engine sealed n8n release tree exact member set mismatch") + for path in [release_dir, *release_dir.rglob("*")]: + path_stat = path.lstat() + if path_stat.st_uid != 0 or path_stat.st_gid != 0: + die(f"Engine sealed n8n release ownership mismatch: {path}") + expected_mode = 0o555 if stat.S_ISDIR(path_stat.st_mode) else 0o444 + if stat.S_IMODE(path_stat.st_mode) != expected_mode: + die(f"Engine sealed n8n release mode mismatch: {path}") + package_json = read_strict_json(package_root / "package.json", "Engine sealed n8n package.json") + if package_json.get("name") != "n8n-nodes-ndc" or package_json.get("version") != descriptor["packageVersion"]: + die("Engine sealed n8n package identity mismatch") + + +def ensure_engine_n8n_sealed_release(descriptor, current_stamp): + staged_release_dir = validate_engine_n8n_staged_release(descriptor) + release_parent = ENGINE_N8N_SEALED_RELEASES_ROOT / "releases" / "n8n-nodes-ndc" + release_parent.mkdir(parents=True, exist_ok=True) + for path in ( + ENGINE_N8N_SEALED_RELEASES_ROOT, + ENGINE_N8N_SEALED_RELEASES_ROOT / "releases", + release_parent, + ): + path_stat = path.lstat() + if stat.S_ISLNK(path_stat.st_mode) or not stat.S_ISDIR(path_stat.st_mode): + die(f"Engine sealed n8n release parent is unsafe: {path}") + os.chown(path, 0, 0) + path.chmod(0o755) + release_dir = release_parent / descriptor["releaseId"] + if release_dir.exists() or release_dir.is_symlink(): + validate_engine_n8n_sealed_release(release_dir, staged_release_dir, descriptor) + return release_dir + + next_dir = release_parent / f".{descriptor['releaseId']}.next-{current_stamp}" + if next_dir.exists() or next_dir.is_symlink(): + die(f"Engine sealed n8n release staging path already exists: {next_dir}") + next_dir.mkdir(mode=0o700) + for filename in ("package.tgz", "release.json", "rollback.json"): + shutil.copy2(staged_release_dir / filename, next_dir / filename) + with tarfile.open(staged_release_dir / "package.tgz", "r:gz") as archive: + for member in archive: + relative = PurePosixPath(member.name) + target = next_dir.joinpath(*relative.parts) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + source = archive.extractfile(member) + if source is None: + die(f"Engine sealed n8n package member unreadable: {member.name}") + with target.open("xb") as output: + shutil.copyfileobj(source, output) + for path in [next_dir, *next_dir.rglob("*")]: + path_stat = path.lstat() + if stat.S_ISLNK(path_stat.st_mode): + die(f"Engine sealed n8n release symlink rejected: {path}") + os.chown(path, 0, 0) + if stat.S_ISDIR(path_stat.st_mode): + path.chmod(0o555) + elif stat.S_ISREG(path_stat.st_mode): + path.chmod(0o444) + else: + die(f"Engine sealed n8n release special member rejected: {path}") + if release_dir.exists() or release_dir.is_symlink(): + die("Engine sealed n8n release appeared concurrently") + next_dir.rename(release_dir) + validate_engine_n8n_sealed_release(release_dir, staged_release_dir, descriptor) + return release_dir + + +def wait_engine_n8n_readiness(container_id): + script = ( + "const http=require('http');const req=http.get('http://127.0.0.1:5678/healthz/readiness'," + "r=>{r.resume();process.exit(r.statusCode===200?0:1)});req.on('error',()=>process.exit(1));" + "req.setTimeout(4000,()=>{req.destroy();process.exit(1)});" + ) + for _attempt in range(1, 61): + result = subprocess.run( + [str(DOCKER), "exec", container_id, "node", "-e", script], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0: + return + time.sleep(5) + die("Engine n8n readiness check failed") + + +def accept_engine_n8n_runtime(descriptor): + base_image = inspect_engine_n8n_base_image() + container_id = engine_n8n_container_ids()[0] + container = inspect_container(container_id) + state = container.get("State") or {} + if state.get("Status") != "running" or container.get("Image") != base_image["id"]: + die("Engine n8n runtime image/state acceptance failed") + wait_engine_n8n_readiness(container_id) + engine_n8n_version(container_id) + config = container.get("Config") or {} + env = set(config.get("Env") or ()) + mounts = container.get("Mounts") or () + package_mounts = [mount for mount in mounts if mount.get("Destination") == ENGINE_N8N_RUNTIME_PACKAGE_PATH] + if descriptor["action"] == "activate": + expected_source = f"/volume2/nodedc-demo/{descriptor['sealedReleaseRelativePath']}" + if len(package_mounts) != 1: + die("Engine n8n package mount count mismatch") + package_mount = package_mounts[0] + if package_mount.get("Source") != expected_source or package_mount.get("RW") is not False: + die("Engine n8n sealed package mount mismatch") + required_env = { + "N8N_USER_FOLDER=/home/node", + "N8N_COMMUNITY_PACKAGES_ENABLED=true", + "N8N_COMMUNITY_PACKAGES_PREVENT_LOADING=false", + "N8N_REINSTALL_MISSING_PACKAGES=false", + } + if not required_env.issubset(env): + die("Engine n8n package loader environment mismatch") + labels = config.get("Labels") or {} + if labels.get("nodedc.n8n-private-extension.release") != descriptor["releaseId"]: + die("Engine n8n release label mismatch") + if labels.get("nodedc.n8n-private-extension.package-sha256") != descriptor["packageSha256"]: + die("Engine n8n package label mismatch") + else: + if package_mounts: + die("Engine n8n inactive baseline still has the private package mount") + inactive_forbidden = ( + "N8N_USER_FOLDER=", + "N8N_COMMUNITY_PACKAGES_ENABLED=", + "N8N_COMMUNITY_PACKAGES_PREVENT_LOADING=", + "N8N_REINSTALL_MISSING_PACKAGES=", + ) + if any(item.startswith(inactive_forbidden) for item in env): + die("Engine n8n inactive baseline still has private package loader flags") + if any(item.startswith("N8N_CUSTOM_EXTENSIONS=") or item.startswith("CUSTOM.") for item in env): + die("Engine n8n custom extension environment is forbidden") + + started_at = str(state.get("StartedAt") or "") + logs = subprocess.run( + [str(DOCKER), "logs", "--since", started_at, container_id], + check=False, + capture_output=True, + text=True, + timeout=60, + ) + if logs.returncode != 0: + die("Engine n8n loader log inspection failed") + scoped_logs = f"{logs.stdout}\n{logs.stderr}".lower() + error_patterns = ( + r"n8n-nodes-ndc.{0,400}(?:cannot find|error|failed|duplicate|eacces|syntaxerror)", + r"(?:cannot find|error loading|failed to load|duplicate|eacces|syntaxerror).{0,400}n8n-nodes-ndc", + ) + if any(re.search(pattern, scoped_logs, re.DOTALL) for pattern in error_patterns): + die("Engine n8n private extension loader log acceptance failed") + + expected_types = list(ENGINE_N8N_NODE_TYPES) if descriptor["action"] == "activate" else [] + expected_credentials = list(ENGINE_N8N_CREDENTIAL_TYPES) if descriptor["action"] == "activate" else [] + loader_catalog = engine_n8n_private_loader_catalog(container_id) + if loader_catalog["node_types"] != expected_types: + die("Engine n8n live package-loader node catalog acceptance failed") + if loader_catalog["credential_types"] != expected_credentials: + die("Engine n8n live package-loader credential catalog acceptance failed") + validate_engine_n8n_catalog_payload(component_root("engine"), descriptor) + first_restart_count = int(container.get("RestartCount") or 0) + time.sleep(2) + stable_container = inspect_container(container_id) + if stable_container.get("State", {}).get("Status") != "running": + die("Engine n8n container did not remain running") + if int(stable_container.get("RestartCount") or 0) != first_restart_count: + die("Engine n8n container restarted during acceptance") + return { + "container_id": container_id, + "image_id": base_image["id"], + "node_types": expected_types, + "credential_types": list(descriptor["expectedCredentialTypes"]), + } + + def plan_artifact(artifact): validate_artifact_location(artifact) ensure_layout() sha = sha256_file(artifact) with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: - manifest, entries, _payload_dir = load_artifact(artifact, Path(tmp)) + manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) + transition_descriptor = None + transition_preflight = None + if is_engine_n8n_transition(manifest["component"], entries): + transition_descriptor = read_engine_n8n_transition_descriptor( + payload_dir / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL + ) + transition_preflight = preflight_engine_n8n_transition( + transition_descriptor, + enforce_expected_current=False, + ) component = manifest["component"] root = component_root(component) @@ -1097,6 +2679,49 @@ def plan_artifact(artifact): print(f"build_root={build_root}") print(f"build={' '.join((str(DOCKER),) + tuple(build_args))}") print(f"services={' '.join(services)}") + if transition_descriptor: + print(f"n8n_transition={transition_descriptor['action']}") + print(f"n8n_version={transition_descriptor['n8nVersion']}") + print(f"n8n_release={transition_descriptor['releaseId']}") + print(f"n8n_package_sha256={transition_descriptor['packageSha256']}") + print(f"n8n_base_image={transition_descriptor['baseImage']}") + print(f"n8n_base_image_id={transition_preflight['base_image_id']}") + print(f"n8n_base_image_architecture={transition_preflight['base_image_architecture']}") + for repo_digest in transition_preflight["base_image_repo_digests"]: + print(f"n8n_base_image_repo_digest={repo_digest}") + print(f"n8n_current_state={transition_preflight['current_state']}") + print(f"n8n_expected_current={transition_descriptor['expectedCurrent']}") + print(f"n8n_state_matches={'yes' if transition_preflight['state_matches'] else 'not-yet'}") + print(f"n8n_sealed_package=/volume2/nodedc-demo/{transition_descriptor['sealedReleaseRelativePath']}") + print("n8n_build=none") + print("n8n_pull=never") + print("n8n_force_recreate=yes") + print("n8n_persistent_data=preserved:/volume2/nodedc-demo/n8n-data") + print("n8n_database=preserved:n8n-postgres") + print(f"n8n_expected_node_types={','.join(transition_descriptor['expectedNodeTypes'])}") + print(f"n8n_expected_credential_types={','.join(transition_descriptor['expectedCredentialTypes'])}") + print("n8n_rollback_baseline=verified_inactive") + touches_map_gateway = component == "platform" and any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries) + touches_external_data_plane = component == "platform" and any( + rel == "platform/services/external-data-plane" + or rel.startswith("platform/services/external-data-plane/") + or rel == "platform/packages/external-provider-contract" + or rel.startswith("platform/packages/external-provider-contract/") + or rel == "platform/docker-compose.external-data-plane.yml" + for rel in entries + ) + if component == "module-foundry" or touches_map_gateway: + print(f"runtime_secret=runner-managed:{MAP_GATEWAY_SECRET_FILE}") + if component == "module-foundry": + print(f"runtime_grants=runner-managed:{EXTERNAL_DATA_PLANE_READER_GRANTS_DIR}") + print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}") + if touches_external_data_plane: + print(f"runtime_secret=runner-managed:{EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}") + print(f"runtime_grants=runner-managed:{EXTERNAL_DATA_PLANE_READER_GRANTS_DIR}") + if component in ("proxy-contur", "dc-amd-proxy") or touches_map_gateway: + print(f"runtime_secret=runner-synced:{MAP_EGRESS_PROXY_SECRET_FILE}") + if component == "dc-amd-proxy": + print(f"runtime_state=runner-prepared:{DC_AMD_PROXY_RUNTIME_DIR}") if state_has_sha(sha): print("state=sha-already-applied") elif state_has_patch_id(manifest["id"]): @@ -1135,6 +2760,73 @@ def create_backup(root, backup_dir, entries, include_nginx_html): (backup_dir / "missing-files.txt").write_text("\n".join(missing) + ("\n" if missing else ""), encoding="utf-8") +def read_backup_path_list(path): + if not path.is_file(): + die(f"deploy backup path list missing: {path}") + return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def inactive_engine_n8n_descriptor(descriptor): + inactive = dict(descriptor) + inactive["action"] = "rollback-inactive" + inactive["expectedCurrent"] = descriptor["releaseId"] + inactive["expectedNodeTypes"] = [] + inactive["expectedCredentialTypes"] = [] + return inactive + + +def restore_engine_n8n_transition(root, backup_dir, entries, descriptor, current_stamp): + existing = set(read_backup_path_list(backup_dir / "existing-files.txt")) + missing = set(read_backup_path_list(backup_dir / "missing-files.txt")) + if existing | missing != set(entries): + die("Engine n8n rollback backup entry set mismatch") + required_baseline = { + ENGINE_N8N_NODES_CATALOG_REL, + ENGINE_N8N_CREDENTIALS_CATALOG_REL, + ENGINE_N8N_SCHEMA_META_REL, + } + if not required_baseline.issubset(existing): + die("Engine n8n rollback baseline catalogs were not present before apply") + + with tempfile.TemporaryDirectory(prefix="engine-n8n-rollback-", dir=TMP_DIR) as tmp: + restore_root = Path(tmp) + with tarfile.open(backup_dir / "source-before.tgz", "r:gz") as archive: + for rel in entries: + if rel not in existing: + continue + try: + member = archive.getmember(rel) + except KeyError: + die(f"Engine n8n rollback backup member missing: {rel}") + if not member.isfile(): + die(f"Engine n8n rollback backup member is not a file: {rel}") + source = archive.extractfile(member) + if source is None: + die(f"Engine n8n rollback backup member unreadable: {rel}") + staged = restore_root / rel + staged.parent.mkdir(parents=True, exist_ok=True) + with staged.open("xb") as output: + shutil.copyfileobj(source, output) + replace_file(staged, root / rel, f"{current_stamp}-rollback") + + if ENGINE_N8N_TRANSITION_DESCRIPTOR_REL not in existing: + inactive = inactive_engine_n8n_descriptor(descriptor) + descriptor_path = root / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL + descriptor_path.parent.mkdir(parents=True, exist_ok=True) + next_path = descriptor_path.with_name(f"{descriptor_path.name}.next-{current_stamp}-rollback") + if next_path.exists() or next_path.is_symlink(): + die(f"Engine n8n rollback descriptor staging path exists: {next_path}") + next_path.write_text(json.dumps(inactive, indent=2) + "\n", encoding="utf-8") + os.replace(next_path, descriptor_path) + + restored_descriptor = current_engine_n8n_transition_descriptor() + if restored_descriptor is None: + die("Engine n8n rollback did not restore an explicit baseline descriptor") + run_compose("engine", ("n8n",), entries) + accept_engine_n8n_runtime(restored_descriptor) + return restored_descriptor["action"] + + def replace_file(src, dst, current_stamp): dst.parent.mkdir(parents=True, exist_ok=True) tmp = dst.with_name(f"{dst.name}.next-{current_stamp}") @@ -1172,6 +2864,23 @@ def copy_payload_path(payload_dir, root, rel, current_stamp): die(f"payload path is neither file nor directory: {rel}") +def seal_n8n_private_extension_release(root, rel): + release_dir = root / rel + if not release_dir.is_dir() or release_dir.is_symlink(): + die("private extension release was not installed as a directory") + for path in [release_dir, *release_dir.rglob("*")]: + path_stat = path.lstat() + if stat.S_ISLNK(path_stat.st_mode): + die(f"private extension installed symlink rejected: {path}") + os.chown(path, 0, 0) + if stat.S_ISDIR(path_stat.st_mode): + path.chmod(0o555) + elif stat.S_ISREG(path_stat.st_mode): + path.chmod(0o444) + else: + die(f"private extension installed special file rejected: {path}") + + def publish_engine_dist(root, current_stamp): dist = root / "nodedc-source" / "dist" if not dist.is_dir(): @@ -1189,6 +2898,17 @@ def publish_engine_dist(root, current_stamp): next_path.rename(dst) +def component_publish_dist(component, entries=None): + if not COMPONENTS[component].get("publish_dist"): + return False + if component == "engine": + return entries is not None and any( + rel == "nodedc-source/dist" or rel.startswith("nodedc-source/dist/") + for rel in entries + ) + return True + + def run_build(component, entries=None): for build_root, build_args in component_builds(component, entries): dockerfile = "Dockerfile" @@ -1223,6 +2943,8 @@ def compose_base_cmd(component): def run_compose(component, services, entries=None): compose_root = component_compose_root(component) cmd = [*compose_base_cmd(component), "up", "-d", "--force-recreate"] + if is_engine_n8n_transition(component, entries): + cmd.extend(["--pull", "never"]) if COMPONENTS[component].get("compose_build"): cmd.append("--build") if component_compose_no_deps(component, entries): @@ -1240,7 +2962,81 @@ def run_compose(component, services, entries=None): subprocess.run([*compose_base_cmd(component), "ps"], cwd=str(compose_root), check=True) -def prepare_component_runtime(component): +def prepare_component_runtime(component, entries=None): + if is_engine_n8n_transition(component, entries): + descriptor = current_engine_n8n_transition_descriptor() + if descriptor is None: + die("installed Engine n8n transition descriptor is missing") + if descriptor["action"] == "activate": + ensure_engine_n8n_sealed_release(descriptor, stamp()) + return + + if component == "module-foundry": + ensure_map_gateway_admin_secret() + ensure_root_owned_grant_directory( + EXTERNAL_DATA_PLANE_READER_GRANTS_DIR, + "external data plane reader grants", + ) + ensure_root_owned_grant_directory( + FOUNDRY_BINDING_GRANTS_DIR, + "foundry binding grants", + ) + return + + if component == "proxy-contur": + sync_map_egress_proxy_secret() + return + + if component == "dc-amd-proxy": + # The connector access value arrives only through the one-time pairing + # endpoint. The artifact must never carry it, and the service user is + # the only non-root identity that can write this directory. + sync_map_egress_proxy_secret() + try: + runtime_stat = DC_AMD_PROXY_RUNTIME_DIR.lstat() + except FileNotFoundError: + DC_AMD_PROXY_RUNTIME_DIR.mkdir(parents=True, exist_ok=False) + runtime_stat = DC_AMD_PROXY_RUNTIME_DIR.lstat() + if stat.S_ISLNK(runtime_stat.st_mode) or not stat.S_ISDIR(runtime_stat.st_mode): + die(f"dc-amd-proxy runtime directory is unsafe: {DC_AMD_PROXY_RUNTIME_DIR}") + os.chown(DC_AMD_PROXY_RUNTIME_DIR, MAP_GATEWAY_RUNTIME_GID, MAP_GATEWAY_RUNTIME_GID) + DC_AMD_PROXY_RUNTIME_DIR.chmod(0o700) + + connector_state = DC_AMD_PROXY_RUNTIME_DIR / "connector-access" + if connector_state.exists() or connector_state.is_symlink(): + connector_stat = connector_state.lstat() + if ( + stat.S_ISLNK(connector_stat.st_mode) + or not stat.S_ISREG(connector_stat.st_mode) + or connector_stat.st_uid != MAP_GATEWAY_RUNTIME_GID + or connector_stat.st_mode & (stat.S_IRWXG | stat.S_IRWXO) + or connector_stat.st_size > 512 + ): + die("dc-amd-proxy connector pair state is unsafe") + return + + if component == "platform" and entries is not None: + touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries) + touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries) + if touches_map_gateway: + # These NAS directories are the only canonical persistent stores + # for all Map Page instances. Creation lives in the root-owned + # runner, never in a user-supplied artifact or a Compose side effect. + cache_root = Path("/volume1/docker/nodedc-platform/map-gateway") + for cache_dir in (cache_root / "live-tile-cache", cache_root / "offline-snapshot"): + cache_dir.mkdir(parents=True, exist_ok=True) + os.chown(cache_dir, 1000, 1000) + os.chmod(cache_dir, 0o750) + ensure_map_gateway_admin_secret() + sync_map_egress_proxy_secret() + if touches_external_data_plane: + ensure_external_data_plane_provisioner_secret() + ensure_root_owned_grant_directory( + EXTERNAL_DATA_PLANE_READER_GRANTS_DIR, + "external data plane reader grants", + ) + return + if component in ("dc-cms", "dc-cms-site-nodedc"): (Path("/volume1/docker/dc-cms/sites") / "nodedc").mkdir(parents=True, exist_ok=True) return @@ -1254,8 +3050,10 @@ def prepare_component_runtime(component): def run_component_runtime(component, entries, services): + if component_artifact_only(component): + return run_build(component, entries) - prepare_component_runtime(component) + prepare_component_runtime(component, entries) run_compose(component, services, entries) @@ -1286,18 +3084,65 @@ def healthcheck_url(check): def component_healthchecks(component, entries=None): + if is_engine_n8n_transition(component, entries): + # The transition does not restart the Engine UI/backend generation. + # Its own acceptance below verifies n8n readiness, image, mount, logs, + # live node export and the pinned Engine MCP catalogs. + return () checks = list(COMPONENTS[component].get("healthchecks", ())) if component == "platform" and entries is not None: touches_compose = any(rel == "platform/docker-compose.platform-http.yml" for rel in entries) touches_ai_workspace_assistant = any(rel == "platform/ai-workspace-assistant" or rel.startswith("platform/ai-workspace-assistant/") for rel in entries) - if touches_compose or touches_ai_workspace_assistant: + touches_ontology = any(rel == "platform/ontology-core" or rel.startswith("platform/ontology-core/") for rel in entries) + touches_gelios = any(rel == "platform/gelios-gateway" or rel.startswith("platform/gelios-gateway/") for rel in entries) + touches_map_gateway = any(rel == "platform/services/map-gateway" or rel.startswith("platform/services/map-gateway/") for rel in entries) + touches_external_data_plane = any(rel == "platform/services/external-data-plane" or rel.startswith("platform/services/external-data-plane/") or rel == "platform/packages/external-provider-contract" or rel.startswith("platform/packages/external-provider-contract/") or rel == "platform/docker-compose.external-data-plane.yml" for rel in entries) + map_gateway_only_compose = touches_compose and touches_map_gateway and not any((touches_ai_workspace_assistant, touches_ontology, touches_gelios, touches_external_data_plane)) + healthcheck_all_for_compose = touches_compose and not map_gateway_only_compose + if healthcheck_all_for_compose or touches_ai_workspace_assistant: checks.append("http://127.0.0.1:18082/healthz") + if healthcheck_all_for_compose or touches_ontology: + checks.append("http://127.0.0.1:18104/healthz") + if healthcheck_all_for_compose or touches_gelios: + checks.append("http://127.0.0.1:18105/healthz") + if touches_map_gateway: + checks.append({"url": "http://127.0.0.1:18103/healthz", "headers": {"x-nodedc-user-id": "deploy-healthcheck"}}) + if touches_external_data_plane: + checks.append("http://127.0.0.1:18106/healthz") return tuple(checks) +def healthcheck_container(container_name): + last_status = "unknown" + for _attempt in range(1, 61): + result = subprocess.run( + [str(DOCKER), "inspect", "--format", "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", container_name], + check=False, + capture_output=True, + text=True, + ) + status_text = result.stdout.strip().lower() + last_status = status_text or result.stderr.strip() or f"inspect-exit-{result.returncode}" + if status_text in ("healthy", "running"): + return + if status_text in ("unhealthy", "exited", "dead"): + break + time.sleep(5) + die(f"container healthcheck failed for {container_name}: {last_status}") + + def run_healthchecks(component, entries=None): + if is_engine_n8n_transition(component, entries): + descriptor = current_engine_n8n_transition_descriptor() + if descriptor is None: + die("installed Engine n8n transition descriptor is missing during acceptance") + accept_engine_n8n_runtime(descriptor) + return for url in component_healthchecks(component, entries): healthcheck_url(url) + container_name = COMPONENTS[component].get("health_container") + if container_name: + healthcheck_container(container_name) def move_artifact(src, target_dir, suffix=""): @@ -1317,7 +3162,12 @@ def apply_artifact(artifact): sha = sha256_file(artifact) backup_id = None + backup_dir = None manifest = None + entries = None + component = None + root = None + transition_descriptor = None apply_started = False current_stamp = stamp() @@ -1332,6 +3182,7 @@ def apply_artifact(artifact): compose_root = component_compose_root(component) bootstrap_root = bool(COMPONENTS[component].get("bootstrap_root")) services = component_services(component, entries) + artifact_only = component_artifact_only(component) if state_has_sha(sha): die(f"artifact sha already applied: {sha}") @@ -1342,45 +3193,68 @@ def apply_artifact(artifact): root.mkdir(parents=True, exist_ok=True) else: die(f"component payload root not found: {root}") - if not compose_root.is_dir(): + if not artifact_only and not compose_root.is_dir(): if bootstrap_root and is_relative_to(compose_root.resolve(strict=False), root.resolve(strict=False)): compose_root.mkdir(parents=True, exist_ok=True) else: die(f"component compose root not found: {compose_root}") compose_files = component_compose_files(component) - if compose_files: + if not artifact_only and compose_files: for compose_file in compose_files: if not compose_file.is_file(): compose_entry = None if is_relative_to(compose_file.resolve(strict=False), root.resolve(strict=False)): compose_entry = compose_file.resolve(strict=False).relative_to(root.resolve(strict=False)).as_posix() - if not bootstrap_root or compose_entry not in entries: + if compose_entry not in entries: die(f"compose file not found: {compose_file}") - elif not (compose_root / "docker-compose.yml").is_file(): - die(f"docker-compose.yml not found in component compose root: {compose_root}") + elif not artifact_only and not (compose_root / "docker-compose.yml").is_file(): + # A bootstrap component begins with no payload root. Its + # compose file may therefore be supplied by the same + # reviewed overlay that is about to create that root. + compose_entry = None + if is_relative_to(compose_root.resolve(strict=False), root.resolve(strict=False)): + compose_entry = compose_root.resolve(strict=False).relative_to(root.resolve(strict=False)).joinpath("docker-compose.yml").as_posix() + if not (bootstrap_root and compose_entry in entries): + die(f"docker-compose.yml not found in component compose root: {compose_root}") compose_env_file = component_compose_env_file(component) - if compose_env_file and not compose_env_file.is_file(): + if not artifact_only and compose_env_file and not compose_env_file.is_file(): die(f"compose env file not found: {compose_env_file}") - if not DOCKER.is_file(): + if not artifact_only and not DOCKER.is_file(): die(f"docker not found: {DOCKER}") + if is_engine_n8n_transition(component, entries): + transition_descriptor = read_engine_n8n_transition_descriptor( + payload_dir / ENGINE_N8N_TRANSITION_DESCRIPTOR_REL + ) + preflight_engine_n8n_transition( + transition_descriptor, + enforce_expected_current=True, + ) + + if COMPONENTS[component].get("immutable_payload"): + for rel in entries: + destination = root / rel + if destination.exists() or destination.is_symlink(): + die(f"immutable release already exists: {rel}") + backup_id = safe_name(f"{component}-{patch_id}-{current_stamp}") backup_dir = BACKUPS_DIR / backup_id backup_dir.mkdir(parents=True, exist_ok=False) (backup_dir / "manifest.env").write_text((work / "manifest.env").read_text(encoding="utf-8"), encoding="utf-8") (backup_dir / "files.txt").write_text((work / "files.txt").read_text(encoding="utf-8"), encoding="utf-8") - include_nginx_html = component == "engine" and any( - rel == "nodedc-source/dist" or rel.startswith("nodedc-source/dist/") - for rel in entries - ) + include_nginx_html = component == "engine" and component_publish_dist(component, entries) create_backup(root, backup_dir, entries, include_nginx_html) apply_started = True for rel in entries: copy_payload_path(payload_dir, root, rel, current_stamp) - if COMPONENTS[component].get("publish_dist"): + if component == "n8n-private-extension": + for rel in entries: + seal_n8n_private_extension_release(root, rel) + + if component_publish_dist(component, entries): publish_engine_dist(root, current_stamp) run_component_runtime(component, entries, services) @@ -1398,6 +3272,27 @@ def apply_artifact(artifact): }) print(f"deploy-ok patch={patch_id} component={component} backup={backup_id}") except Exception as exc: + rollback_status = "not-required" + if ( + apply_started + and backup_dir is not None + and root is not None + and transition_descriptor is not None + and is_engine_n8n_transition(component, entries) + ): + try: + restored_action = restore_engine_n8n_transition( + root, + backup_dir, + entries, + transition_descriptor, + current_stamp, + ) + rollback_status = f"ok:{restored_action}" + print(f"engine-n8n-automatic-rollback={rollback_status}", file=sys.stderr) + except Exception as rollback_exc: + rollback_status = f"failed:{type(rollback_exc).__name__}" + print("engine-n8n-automatic-rollback=failed", file=sys.stderr) failed_path = None if artifact.exists(): try: @@ -1412,6 +3307,7 @@ def apply_artifact(artifact): "failed_at": utc_now(), "id": manifest.get("id") if manifest else None, "message": str(exc), + "rollback_status": rollback_status, "sha256": sha, "started_apply": apply_started, "status": "failed", diff --git a/infra/deploy-runner/test_engine_n8n_private_extension.py b/infra/deploy-runner/test_engine_n8n_private_extension.py new file mode 100644 index 0000000..d20697f --- /dev/null +++ b/infra/deploy-runner/test_engine_n8n_private_extension.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +PLATFORM_ROOT = SCRIPT_DIR.parent.parent +ENGINE_ROOT = PLATFORM_ROOT.parent / "NODEDC_ENGINE_INFRA" +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +BUILDER_PATH = SCRIPT_DIR / "build-engine-n8n-private-extension-artifact.mjs" +STAGE_ARTIFACT = ( + PLATFORM_ROOT + / "infra/deploy-artifacts" + / "nodedc-n8n-private-extension-n8n-nodes-ndc-release-20260716-003.tgz" +) +EXPECTED_NODES = [ + "n8n-nodes-ndc.ndcDataProductPublish", + "n8n-nodes-ndc.ndcDataProductRead", + "n8n-nodes-ndc.ndcFoundryBinding", +] +EXPECTED_CREDENTIALS = [ + "ndcDataProductWriterApi", + "ndcDataProductReaderApi", + "ndcFoundryBindingApi", +] + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader("nodedc_engine_deploy_under_test", str(RUNNER_PATH)) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class EngineN8nPrivateExtensionTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temporary = tempfile.TemporaryDirectory(prefix="nodedc-engine-n8n-policy-") + cls.root = Path(cls.temporary.name) + cls.results = [] + for index in range(2): + output = cls.root / f"build-{index}" + output.mkdir() + env = os.environ.copy() + env["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(output) + env["NODEDC_N8N_EXTENSION_STAGE_ARTIFACT"] = str(STAGE_ARTIFACT) + result = subprocess.run( + ["node", str(BUILDER_PATH)], + cwd=PLATFORM_ROOT, + env=env, + check=True, + capture_output=True, + text=True, + ) + cls.results.append(json.loads(result.stdout)) + + @classmethod + def tearDownClass(cls): + cls.temporary.cleanup() + + def artifact(self, build_index, kind): + return Path(self.results[build_index][kind]["artifact"]) + + def test_engine_artifacts_are_byte_reproducible(self): + for kind in ("activation", "rollback"): + first = self.artifact(0, kind).read_bytes() + second = self.artifact(1, kind).read_bytes() + self.assertEqual(first, second) + self.assertEqual(self.results[0][kind]["sha256"], self.results[1][kind]["sha256"]) + self.assertEqual(first[4:8], b"\0\0\0\0") + self.assertEqual(first[3] & 0x08, 0) + + def test_activation_and_rollback_pass_strict_runner_policy(self): + expected = {"activation": ("activate", 7), "rollback": ("rollback-inactive", 4)} + for kind, (action, entry_count) in expected.items(): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: + manifest, entries, payload = RUNNER.load_artifact( + self.artifact(0, kind), + Path(directory), + ) + descriptor = RUNNER.read_engine_n8n_transition_descriptor( + payload / RUNNER.ENGINE_N8N_TRANSITION_DESCRIPTOR_REL + ) + self.assertEqual(manifest["component"], "engine") + self.assertEqual(descriptor["action"], action) + self.assertEqual(len(entries), entry_count) + self.assertEqual(RUNNER.component_services("engine", entries), ("n8n",)) + self.assertEqual(RUNNER.component_builds("engine", entries), ()) + self.assertFalse(RUNNER.component_publish_dist("engine", entries)) + + def test_activation_catalog_is_exact_three_without_tool_variants(self): + with tempfile.TemporaryDirectory() as directory: + _manifest, _entries, payload = RUNNER.load_artifact( + self.artifact(0, "activation"), + Path(directory), + ) + nodes = json.loads((payload / RUNNER.ENGINE_N8N_NODES_CATALOG_REL).read_text()) + credentials = json.loads((payload / RUNNER.ENGINE_N8N_CREDENTIALS_CATALOG_REL).read_text()) + private_nodes = [item for item in nodes if item.get("name", "").startswith("n8n-nodes-ndc.")] + private_credentials = [item for item in credentials if item.get("name") in EXPECTED_CREDENTIALS] + self.assertEqual([item["name"] for item in private_nodes], EXPECTED_NODES) + self.assertEqual([item["name"] for item in private_credentials], EXPECTED_CREDENTIALS) + self.assertTrue(all("usableAsTool" not in item for item in private_nodes)) + self.assertEqual((len(nodes), len(credentials)), (437, 388)) + + def test_rollback_catalog_restores_verified_inactive_baseline(self): + with tempfile.TemporaryDirectory() as directory: + _manifest, _entries, payload = RUNNER.load_artifact( + self.artifact(0, "rollback"), + Path(directory), + ) + nodes = json.loads((payload / RUNNER.ENGINE_N8N_NODES_CATALOG_REL).read_text()) + credentials = json.loads((payload / RUNNER.ENGINE_N8N_CREDENTIALS_CATALOG_REL).read_text()) + self.assertEqual([item for item in nodes if item.get("name", "").startswith("n8n-nodes-ndc.")], []) + self.assertEqual([item for item in credentials if item.get("name") in EXPECTED_CREDENTIALS], []) + self.assertEqual((len(nodes), len(credentials)), (434, 385)) + + def test_compose_override_is_runner_derived_and_offline(self): + with tempfile.TemporaryDirectory() as directory: + _manifest, _entries, payload = RUNNER.load_artifact( + self.artifact(0, "activation"), + Path(directory), + ) + descriptor = RUNNER.read_engine_n8n_transition_descriptor( + payload / RUNNER.ENGINE_N8N_TRANSITION_DESCRIPTOR_REL + ) + override = (payload / RUNNER.ENGINE_N8N_COMPOSE_OVERRIDE_REL).read_text() + self.assertEqual(override, RUNNER.expected_engine_n8n_compose_override(descriptor)) + self.assertIn("pull_policy: never", override) + self.assertIn("N8N_USER_FOLDER: /home/node", override) + self.assertIn(":/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc:ro", override) + self.assertNotIn("N8N_CUSTOM_EXTENSIONS", override) + self.assertNotIn("build:", override) + + def test_unknown_descriptor_key_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + RUNNER.safe_extract(self.artifact(0, "activation"), work) + descriptor_path = work / "payload" / RUNNER.ENGINE_N8N_TRANSITION_DESCRIPTOR_REL + descriptor = json.loads(descriptor_path.read_text()) + descriptor["unexpected"] = True + descriptor_path.write_text(json.dumps(descriptor)) + with self.assertRaisesRegex(RUNNER.DeployError, "keys mismatch"): + RUNNER.validate_engine_n8n_transition( + work / "payload", + RUNNER.parse_files_list(work / "files.txt"), + ) + + def test_core_catalog_substitution_is_rejected_even_when_counts_match(self): + cases = ( + ("activation", RUNNER.ENGINE_N8N_NODES_CATALOG_REL, "node"), + ("activation", RUNNER.ENGINE_N8N_CREDENTIALS_CATALOG_REL, "credential"), + ("rollback", RUNNER.ENGINE_N8N_NODES_CATALOG_REL, "node"), + ("rollback", RUNNER.ENGINE_N8N_CREDENTIALS_CATALOG_REL, "credential"), + ) + for kind, relative_path, label in cases: + with self.subTest(kind=kind, catalog=label), tempfile.TemporaryDirectory() as directory: + work = Path(directory) + RUNNER.safe_extract(self.artifact(0, kind), work) + catalog_path = work / "payload" / relative_path + catalog = json.loads(catalog_path.read_text()) + if label == "node": + candidate = next( + item for item in catalog + if not item.get("name", "").startswith("n8n-nodes-ndc.") + ) + candidate["name"] = "n8n-nodes-unreviewed.hiddenNode" + else: + candidate = next( + item for item in catalog + if item.get("name") not in EXPECTED_CREDENTIALS + ) + candidate["name"] = "unreviewedCredential" + catalog_path.write_text(json.dumps(catalog, indent=2) + "\n") + with self.assertRaisesRegex(RUNNER.DeployError, "catalog sha256 mismatch"): + RUNNER.validate_engine_n8n_transition( + work / "payload", + RUNNER.parse_files_list(work / "files.txt"), + ) + + def test_old_engine_artifact_is_rejected_by_descriptor_gate(self): + old = PLATFORM_ROOT / "infra/deploy-artifacts/nodedc-engine-n8n-private-extension-20260715-002.tgz" + if not old.is_file(): + self.skipTest("rejected historical artifact not present") + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(RUNNER.DeployError, "canonical transition descriptor"): + RUNNER.load_artifact(old, Path(directory)) + + def test_engine_source_keeps_base_compose_and_separate_override(self): + original_root = RUNNER.COMPONENTS["engine"]["payload_root"] + original_compose = RUNNER.COMPONENTS["engine"]["compose_root"] + try: + RUNNER.COMPONENTS["engine"]["payload_root"] = ENGINE_ROOT + RUNNER.COMPONENTS["engine"]["compose_root"] = ENGINE_ROOT + RUNNER.validate_engine_n8n_base_compose_source() + finally: + RUNNER.COMPONENTS["engine"]["payload_root"] = original_root + RUNNER.COMPONENTS["engine"]["compose_root"] = original_compose + + def test_additional_n8n_runtime_service_is_rejected(self): + original_root = RUNNER.COMPONENTS["engine"]["payload_root"] + original_compose = RUNNER.COMPONENTS["engine"]["compose_root"] + try: + with tempfile.TemporaryDirectory() as directory: + engine_root = Path(directory) + compose = (ENGINE_ROOT / "docker-compose.yml").read_text() + compose += ( + "\n n8n-worker-2:\n" + " image: docker.n8n.io/n8nio/n8n:${N8N_IMAGE_TAG:-2.3.2}\n" + ) + (engine_root / "docker-compose.yml").write_text(compose) + RUNNER.COMPONENTS["engine"]["payload_root"] = engine_root + RUNNER.COMPONENTS["engine"]["compose_root"] = engine_root + with self.assertRaisesRegex(RUNNER.DeployError, "exact service topology mismatch"): + RUNNER.validate_engine_n8n_base_compose_source() + finally: + RUNNER.COMPONENTS["engine"]["payload_root"] = original_root + RUNNER.COMPONENTS["engine"]["compose_root"] = original_compose + + def test_sealed_release_exact_set_includes_implicit_directories(self): + release_relative = Path( + "payload/releases/n8n-nodes-ndc/0.1.2-05e4b38b14b4a019" + ) + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + outer = work / "outer" + RUNNER.safe_extract(STAGE_ARTIFACT, outer) + staged_release = outer / release_relative + sealed_release = work / "sealed" + sealed_release.mkdir() + for filename in ("package.tgz", "release.json", "rollback.json"): + (sealed_release / filename).write_bytes( + (staged_release / filename).read_bytes() + ) + + expected_paths = { + "package", + "package.tgz", + "release.json", + "rollback.json", + } + with tarfile.open(staged_release / "package.tgz", "r:gz") as archive: + for member in archive: + RUNNER.add_engine_n8n_sealed_member_paths( + expected_paths, + member.name, + ) + target = sealed_release.joinpath(*Path(member.name).parts) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + source = archive.extractfile(member) + self.assertIsNotNone(source) + target.write_bytes(source.read()) + + actual_paths = { + path.relative_to(sealed_release).as_posix() + for path in sealed_release.rglob("*") + } + self.assertEqual(actual_paths, expected_paths) + self.assertIn("package/dist/nodes", expected_paths) + self.assertIn("package/dist/credentials", expected_paths) + + def test_tar_members_have_no_appledouble_or_special_types(self): + for kind in ("activation", "rollback"): + with tarfile.open(self.artifact(0, kind), "r:gz") as archive: + for member in archive: + self.assertFalse(Path(member.name).name.startswith("._")) + self.assertTrue(member.isfile() or member.isdir()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/infra/deploy-runner/test_n8n_private_extension.py b/infra/deploy-runner/test_n8n_private_extension.py new file mode 100644 index 0000000..1ceb194 --- /dev/null +++ b/infra/deploy-runner/test_n8n_private_extension.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import io +import json +import os +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent +PLATFORM_ROOT = SCRIPT_DIR.parent.parent +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +BUILDER_PATH = SCRIPT_DIR / "build-n8n-private-extension-artifact.mjs" +PATCH_ID = "n8n-nodes-ndc-release-20260716-003" +EXPECTED_NODES = [ + "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", + "dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js", + "dist/nodes/NdcFoundryBinding/NdcFoundryBinding.node.js", +] +EXPECTED_NODE_TYPES = [ + "n8n-nodes-ndc.ndcDataProductPublish", + "n8n-nodes-ndc.ndcDataProductRead", + "n8n-nodes-ndc.ndcFoundryBinding", +] +EXPECTED_CREDENTIALS = [ + "dist/credentials/NdcDataProductWriterApi.credentials.js", + "dist/credentials/NdcDataProductReaderApi.credentials.js", + "dist/credentials/NdcFoundryBindingApi.credentials.js", +] +EXPECTED_CREDENTIAL_TYPES = [ + "ndcDataProductWriterApi", + "ndcDataProductReaderApi", + "ndcFoundryBindingApi", +] + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader("nodedc_deploy_under_test", str(RUNNER_PATH)) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class N8nPrivateExtensionPolicyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temporary = tempfile.TemporaryDirectory(prefix="nodedc-extension-policy-") + cls.root = Path(cls.temporary.name) + cls.artifacts = [] + cls.results = [] + for index in range(2): + output_dir = cls.root / f"build-{index}" + output_dir.mkdir() + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(output_dir) + command = ["node", str(BUILDER_PATH)] + if index == 0: + command.append(PATCH_ID) + result = subprocess.run( + command, + cwd=PLATFORM_ROOT, + env=environment, + check=True, + capture_output=True, + text=True, + ) + metadata = json.loads(result.stdout) + artifact = Path(metadata["artifact"]) + cls.results.append(metadata) + cls.artifacts.append(artifact) + + @classmethod + def tearDownClass(cls): + cls.temporary.cleanup() + + def test_runner_source_compiles(self): + compile(RUNNER_PATH.read_text(encoding="utf-8"), str(RUNNER_PATH), "exec") + + def test_builder_is_byte_reproducible(self): + first = self.artifacts[0].read_bytes() + second = self.artifacts[1].read_bytes() + self.assertEqual(first, second) + self.assertEqual(self.results[0]["artifactSha256"], self.results[1]["artifactSha256"]) + self.assertEqual(self.results[0]["patchId"], PATCH_ID) + self.assertEqual(self.results[0]["nodeTypes"], EXPECTED_NODE_TYPES) + self.assertEqual(self.results[0]["credentialTypes"], EXPECTED_CREDENTIAL_TYPES) + self.assertEqual(first[4:8], b"\0\0\0\0", "gzip mtime must be zero") + self.assertEqual(first[3] & 0x08, 0, "gzip header must not carry a host filename") + + inner = self._inner_package_bytes() + self.assertEqual(inner[4:8], b"\0\0\0\0", "inner package gzip mtime must be zero") + self.assertEqual(inner[3] & 0x08, 0, "inner package gzip header must not carry a host filename") + + def test_positive_release_passes_runner_policy(self): + with tempfile.TemporaryDirectory(prefix="nodedc-extension-load-") as directory: + manifest, entries, _payload = RUNNER.load_artifact(self.artifacts[0], Path(directory)) + self.assertEqual(manifest["component"], "n8n-private-extension") + self.assertEqual(len(entries), 1) + self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.2-[a-f0-9]{16}$") + + with tarfile.open(self.artifacts[0], "r:gz") as archive: + names = archive.getnames() + self.assertFalse(any(Path(name).name.startswith("._") for name in names)) + self.assertFalse(any(".." in Path(name).parts for name in names)) + + def test_release_has_exactly_three_non_tool_runtime_types(self): + package = self._inner_package_bytes() + with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive: + package_json_member = archive.getmember("package/package.json") + package_json = json.loads(archive.extractfile(package_json_member).read()) + self.assertEqual(package_json["version"], "0.1.2") + self.assertEqual(package_json["n8n"]["nodes"], EXPECTED_NODES) + self.assertEqual(package_json["n8n"]["credentials"], EXPECTED_CREDENTIALS) + for node_path in EXPECTED_NODES: + source = archive.extractfile(archive.getmember(f"package/{node_path}")).read().decode("utf-8") + self.assertNotRegex(source, r"\busableAsTool\b") + + def test_inner_package_stream_and_members_are_canonical(self): + package = self._inner_package_bytes() + self.assertEqual(package[4:8], b"\0\0\0\0") + self.assertEqual(package[3] & 0x08, 0) + with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive: + members = archive.getmembers() + self.assertEqual([member.name for member in members], sorted(member.name for member in members)) + for member in members: + self.assertEqual((member.uid, member.gid, member.uname, member.gname), (0, 0, "root", "root")) + self.assertEqual(member.mtime, 0) + self.assertEqual(member.mode, 0o755 if member.isdir() else 0o644) + + def test_v2_first_activation_uses_verified_inactive_baseline(self): + with tempfile.TemporaryDirectory(prefix="nodedc-extension-v2-") as directory: + work = Path(directory) + RUNNER.safe_extract(self.artifacts[0], work) + entries = RUNNER.parse_files_list(work / "files.txt") + release_dir = work / "payload" / entries[0] + release_path = release_dir / "release.json" + rollback_path = release_dir / "rollback.json" + release = json.loads(release_path.read_text(encoding="utf-8")) + rollback = json.loads(rollback_path.read_text(encoding="utf-8")) + + expected_policy = { + "allowed": ["previous_verified_immutable_release", "verified_inactive"], + "firstActivation": "verified_inactive", + "requiresPreActivationVerification": True, + } + self.assertEqual(release["schemaVersion"], "nodedc.n8n-private-extension-release/v2") + self.assertEqual(release["package"]["version"], "0.1.2") + self.assertEqual(release["activation"]["rollbackBaselinePolicy"], expected_policy) + self.assertEqual(rollback["schemaVersion"], "nodedc.n8n-private-extension-rollback/v2") + self.assertEqual(rollback["baselinePolicy"], expected_policy) + + rollback["baselinePolicy"]["requiresPreActivationVerification"] = False + rollback_path.write_text(json.dumps(rollback), encoding="utf-8") + with self.assertRaisesRegex(RUNNER.DeployError, "rollback manifest mismatch"): + RUNNER.validate_n8n_private_extension_release(work / "payload", entries) + + def test_outer_appledouble_and_traversal_are_rejected(self): + with self.assertRaisesRegex(RUNNER.DeployError, "unexpected tar member"): + RUNNER.validate_tar_member(tarfile.TarInfo("._manifest.env")) + with self.assertRaisesRegex(RUNNER.DeployError, "path escape rejected"): + RUNNER.validate_tar_member(tarfile.TarInfo("payload/../escape")) + + def test_extra_custom_node_is_rejected(self): + def mutate(package_json): + package_json["n8n"]["nodes"].append("dist/nodes/Extra/Extra.node.js") + + package_path, release = self._tampered_package( + mutate, + {"package/dist/nodes/Extra/Extra.node.js": b"module.exports = {};\n"}, + ) + with self.assertRaisesRegex(RUNNER.DeployError, "n8n.nodes registration mismatch"): + RUNNER.validate_n8n_package_tarball(package_path, release) + + def test_lifecycle_script_is_rejected(self): + def mutate(package_json): + package_json.setdefault("scripts", {})["install"] = "echo forbidden" + + package_path, release = self._tampered_package(mutate) + with self.assertRaisesRegex(RUNNER.DeployError, "lifecycle script is forbidden"): + RUNNER.validate_n8n_package_tarball(package_path, release) + + def test_inner_appledouble_is_rejected(self): + package_path, release = self._tampered_package( + lambda _package_json: None, + {"package/dist/._NdcDataProductPublish.node.js": b"forbidden\n"}, + ) + with self.assertRaisesRegex(RUNNER.DeployError, "hidden member rejected"): + RUNNER.validate_n8n_package_tarball(package_path, release) + + def test_tool_variant_marker_is_rejected_by_runner(self): + target = EXPECTED_NODES[0] + package_path, release = self._tampered_package( + lambda _package_json: None, + file_mutations={target: lambda source: source + b"\n// usableAsTool: true\n"}, + ) + with self.assertRaisesRegex(RUNNER.DeployError, "would generate a tool variant"): + RUNNER.validate_n8n_package_tarball(package_path, release) + + def _tampered_package(self, mutate, extra_files=None, file_mutations=None): + package_bytes = self._inner_package_bytes() + members = [] + with tarfile.open(fileobj=io.BytesIO(package_bytes), mode="r:gz") as archive: + for member in archive: + if not member.isfile(): + continue + source = archive.extractfile(member) + members.append((member.name, source.read())) + + rewritten = [] + for name, content in members: + if name == "package/package.json": + package_json = json.loads(content) + mutate(package_json) + content = (json.dumps(package_json, sort_keys=True) + "\n").encode("utf-8") + package_rel = name.removeprefix("package/") + if package_rel in (file_mutations or {}): + content = file_mutations[package_rel](content) + rewritten.append((name, content)) + rewritten.extend((extra_files or {}).items()) + + target = self.root / f"tampered-{len(list(self.root.glob('tampered-*.tgz')))}.tgz" + with tarfile.open(target, "w:gz", format=tarfile.PAX_FORMAT) as archive: + for name, content in rewritten: + member = tarfile.TarInfo(name) + member.mode = 0o644 + member.mtime = 0 + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) + + with tarfile.open(target, "r:gz") as archive: + package_member = next(member for member in archive if member.name == "package/package.json") + version = json.loads(archive.extractfile(package_member).read())["version"] + release = { + "package": { + "version": version, + "sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + "bytes": target.stat().st_size, + } + } + return target, release + + def _inner_package_bytes(self): + with tarfile.open(self.artifacts[0], "r:gz") as archive: + member = next(item for item in archive if item.name.endswith("/package.tgz")) + source = archive.extractfile(member) + return source.read() + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index 8be69a9..fda87a0 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -152,9 +152,11 @@ services: AI_WORKSPACE_HUB_FALLBACK_URLS: ${AI_WORKSPACE_HUB_FALLBACK_URLS:-} AI_WORKSPACE_ASSISTANT_ACTION_RELAY_ENABLED: ${AI_WORKSPACE_ASSISTANT_ACTION_RELAY_ENABLED:-true} AI_WORKSPACE_ASSISTANT_ACTION_RELAY_ID: ${AI_WORKSPACE_ASSISTANT_ACTION_RELAY_ID:-local-dev} + AI_WORKSPACE_ONTOLOGY_MCP_ENABLED: ${AI_WORKSPACE_ONTOLOGY_MCP_ENABLED:-true} AI_WORKSPACE_OPS_ENTITLEMENT_URL: ${AI_WORKSPACE_OPS_ENTITLEMENT_URL:-} AI_WORKSPACE_OPS_ENTITLEMENT_TOKEN: ${AI_WORKSPACE_OPS_ENTITLEMENT_TOKEN:-} AI_WORKSPACE_OPS_ENTITLEMENT_REQUIRED: ${AI_WORKSPACE_OPS_ENTITLEMENT_REQUIRED:-false} + AI_WORKSPACE_ENTITLEMENT_ADAPTERS_JSON: ${AI_WORKSPACE_ENTITLEMENT_ADAPTERS_JSON:-} AI_WORKSPACE_OPS_GATEWAY_BASE_URL: ${AI_WORKSPACE_OPS_GATEWAY_BASE_URL:-} AI_WORKSPACE_OPS_DEFAULT_WORKSPACE_SLUG: ${AI_WORKSPACE_OPS_DEFAULT_WORKSPACE_SLUG:-} AI_WORKSPACE_OPS_DEFAULT_PROJECT_ID: ${AI_WORKSPACE_OPS_DEFAULT_PROJECT_ID:-} @@ -168,6 +170,86 @@ services: ai-workspace-postgres: condition: service_healthy + ontology-core: + image: nodedc/ontology-core:local + build: + context: ../services/ontology-core + restart: unless-stopped + environment: + NODE_ENV: production + PORT: 18104 + NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} + expose: + - "18104" + healthcheck: + test: + - CMD + - node + - -e + - "fetch('http://127.0.0.1:18104/healthz').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))" + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + + # External-provider telemetry is isolated from Platform identity state. The + # database is not published; only the Gateway is exposed on the local host + # for operator checks and Engine development. + gelios-postgres: + image: ${GELIOS_TIMESCALE_IMAGE:-timescale/timescaledb-ha:pg16.14-ts2.28.2-all} + restart: unless-stopped + environment: + POSTGRES_DB: ${GELIOS_PG_DB:-nodedc_gelios} + POSTGRES_USER: ${GELIOS_PG_USER:-nodedc_gelios} + POSTGRES_PASSWORD: ${GELIOS_PG_PASS:-change-me-generate-with-infra-scripts-init-dev-env} + healthcheck: + test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 30s + volumes: + - gelios-database:/home/postgres/pgdata/data + networks: + - gelios-data + + gelios-gateway: + image: nodedc/gelios-gateway:local + build: + context: ../services/gelios-gateway + restart: unless-stopped + environment: + NODE_ENV: production + PORT: 18105 + DATABASE_URL: ${GELIOS_DATABASE_URL:-postgresql://nodedc_gelios:change-me-generate-with-infra-scripts-init-dev-env@gelios-postgres:5432/nodedc_gelios} + NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} + GELIOS_TENANT_ID: ${GELIOS_TENANT_ID:-robot2b} + GELIOS_CONNECTION_ID: ${GELIOS_CONNECTION_ID:-gelios-primary} + GELIOS_UNIT_SCOPE: ${GELIOS_UNIT_SCOPE:-allowlist} + GELIOS_ALLOWED_UNIT_IDS: ${GELIOS_ALLOWED_UNIT_IDS:-} + GELIOS_INTAKE_ENABLED: ${GELIOS_INTAKE_ENABLED:-false} + GELIOS_RAW_RETENTION_DAYS: ${GELIOS_RAW_RETENTION_DAYS:-14} + expose: + - "18105" + ports: + - "${GELIOS_GATEWAY_HOST_BIND:-127.0.0.1:18105}:18105" + depends_on: + gelios-postgres: + condition: service_healthy + healthcheck: + test: + - CMD + - node + - -e + - "fetch('http://127.0.0.1:18105/healthz').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))" + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + networks: + - default + - gelios-data + ai-workspace-hub: image: nodedc/ai-workspace-hub:local build: @@ -180,10 +262,14 @@ services: AI_WORKSPACE_HUB_WS_PATH: /api/ai-workspace/hub NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} NODEDC_AI_WORKSPACE_ASSISTANT_URL: http://ai-workspace-assistant:18082 + NODEDC_ONTOLOGY_CORE_URL: http://ontology-core:18104 expose: - "18081" ports: - "${AI_WORKSPACE_HUB_HOST_BIND:-127.0.0.1:18081}:18081" + depends_on: + ontology-core: + condition: service_healthy map-gateway: image: nodedc/map-gateway:local @@ -194,6 +280,10 @@ services: NODE_ENV: production PORT: 18103 CESIUM_ION_TOKEN: ${CESIUM_ION_TOKEN:-} + # Foundry signs its private token-admin requests with the existing + # Platform internal credential. The value stays outside this compose + # file; this line only wires the local development runtime. + NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} CESIUM_ION_ASSET_ALLOWLIST: ${CESIUM_ION_ASSET_ALLOWLIST:-1,2,96188} MAP_GATEWAY_UPSTREAM_ALLOWLIST: ${MAP_GATEWAY_UPSTREAM_ALLOWLIST:-api.cesium.com,assets.ion.cesium.com,tile.openstreetmap.org,dev.virtualearth.net,ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtualearth.net,ecn.t2.tiles.virtualearth.net,ecn.t3.tiles.virtualearth.net} # Offline access is selected explicitly with the cache profile, rather @@ -236,6 +326,7 @@ volumes: authentik-certs: notification-database: ai-workspace-database: + gelios-database: caddy-data: caddy-config: # Cache data is runtime state, not Compose lifecycle state. These volumes are @@ -247,3 +338,7 @@ volumes: map-offline-snapshot: external: true name: ${NODEDC_MAP_OFFLINE_SNAPSHOT_VOLUME:-nodedc-platform_map-offline-snapshot} + +networks: + gelios-data: + internal: true diff --git a/infra/scripts/check-local-test-system.sh b/infra/scripts/check-local-test-system.sh index b39d556..26eadb4 100755 --- a/infra/scripts/check-local-test-system.sh +++ b/infra/scripts/check-local-test-system.sh @@ -93,6 +93,7 @@ run_sh_quiet() { section "Static syntax" run_cmd "Hub server syntax" node --check "$HUB_DIR/src/server.mjs" run_cmd "Assistant server syntax" node --check "$ASSISTANT_DIR/src/server.mjs" +run_cmd "Ontology MCP server syntax" node --check "$ONTOLOGY_DIR/src/mcp-server.mjs" run_cmd "Local environment safety checker syntax" node --check "$INFRA_DIR/scripts/check-local-environment-safety.mjs" run_sh "Shell scripts syntax" "$PLATFORM_ROOT" \ "sh -n infra/scripts/init-dev-env.sh infra/scripts/check-local-test-system.sh infra/scripts/check-ai-workspace-topology.sh infra/scripts/check-ai-workspace-config-contract.sh infra/scripts/check-ai-workspace-release-gates.sh" @@ -128,8 +129,10 @@ fi section "AI Workspace smokes" run_sh_quiet "Assistant run-profile smoke" "$ASSISTANT_DIR" "npm run smoke:run-profile" +run_sh_quiet "AI Hub Ontology MCP proxy smoke" "$HUB_DIR" "npm run smoke:ontology-mcp-proxy" if [ -d "$ONTOLOGY_DIR" ]; then run_sh_quiet "Ontology assistant caller smoke" "$ONTOLOGY_DIR" "npm run smoke:assistant-caller" + run_sh_quiet "Ontology MCP smoke" "$ONTOLOGY_DIR" "npm run smoke:mcp" else skip "Ontology assistant caller smoke (missing $ONTOLOGY_DIR)" fi diff --git a/infra/synology/.env.synology.example b/infra/synology/.env.synology.example index ab1db58..2306c6f 100644 --- a/infra/synology/.env.synology.example +++ b/infra/synology/.env.synology.example @@ -33,6 +33,24 @@ TASK_INTERNAL_LOGOUT_URL=https://ops.nodedc.ru/api/internal/nodedc/logout/ NODEDC_ENGINE_DOCKER_NETWORK=nodedc-demo_default NODEDC_ENGINE_INTERNAL_URL=http://nodedc-demo-nodedc-backend-1:3001 +# Platform Map Gateway — private service and one shared persistent TileCache. +# Optional one-time legacy bootstrap only. Preferred operation: an authorized +# Foundry admin saves the token through Page Library → Platform settings; it is +# then stored in the private NAS cache volume and never exposed to Foundry UI, +# deploy artifact or browser env. Do not paste a real token into this example. +CESIUM_ION_TOKEN= +CESIUM_ION_ASSET_ALLOWLIST=1,2,96188 +MAP_CACHE_MODE=readwrite +MAP_CACHE_MAX_MB=20480 +MAP_GATEWAY_HOST_BIND=127.0.0.1:18103 +# Created by the root-owned deploy runner on first Map Gateway rollout. +# These are NAS paths, visible through SMB as nodedc-platform/map-gateway/. +MAP_LIVE_CACHE_HOST_DIR=/volume1/docker/nodedc-platform/map-gateway/live-tile-cache +MAP_OFFLINE_SNAPSHOT_HOST_DIR=/volume1/docker/nodedc-platform/map-gateway/offline-snapshot +MAP_GATEWAY_UPSTREAM_ALLOWLIST=api.cesium.com,assets.ion.cesium.com,tile.openstreetmap.org,dev.virtualearth.net,ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtualearth.net,ecn.t2.tiles.virtualearth.net,ecn.t3.tiles.virtualearth.net +MAP_GATEWAY_LEGACY_CACHE_HOSTS= +MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST= + LAUNCHER_OIDC_ISSUER=https://id.nodedc.ru/application/o/launcher/ LAUNCHER_OIDC_CLIENT_ID=nodedc-launcher LAUNCHER_OIDC_CLIENT_SECRET=replace-with-random-synology-secret @@ -55,6 +73,34 @@ SESSION_SECRET=replace-with-random-synology-secret COOKIE_DOMAIN=.nas.nodedc COOKIE_SECURE=false +# External Data Plane — provider-neutral storage. Generate a distinct database +# password; it must never reuse NODEDC_INTERNAL_ACCESS_TOKEN or a provider key. +EXTERNAL_DATA_PLANE_TIMESCALE_IMAGE=timescale/timescaledb-ha:pg16.14-ts2.28.2-all +EXTERNAL_DATA_PLANE_PG_DB=nodedc_data_plane +EXTERNAL_DATA_PLANE_PG_USER=nodedc_data_plane +EXTERNAL_DATA_PLANE_PG_PASS=replace-with-random-synology-secret +EXTERNAL_DATA_PLANE_HOST_BIND=127.0.0.1:18106 +EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE=10 +EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS=14 +EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES=5242880 +EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH=5000 +EXTERNAL_DATA_PLANE_MAX_ATTRIBUTES_BYTES_PER_FACT=65536 +EXTERNAL_DATA_PLANE_MAX_PATCH_OPERATIONS=500 +EXTERNAL_DATA_PLANE_MAX_PATCH_BYTES=262144 +EXTERNAL_DATA_PLANE_PATCH_RETENTION_MS=3600000 +EXTERNAL_DATA_PLANE_RECEIPT_RETENTION_MS=604800000 +EXTERNAL_DATA_PLANE_RETENTION_DELETE_LIMIT=10000 +EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS=20000 +EXTERNAL_DATA_PLANE_STREAM_POLL_MS=1000 +EXTERNAL_DATA_PLANE_MAX_READER_STREAMS=10 +EXTERNAL_DATA_PLANE_WRITER_BINDING_MAX_TTL_DAYS=90 +EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS=300 +EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS=3600000 +EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED=false +# The writer-provisioner secret is a root-owned file under +# /volume1/docker/nodedc-platform/secrets/external-data-plane-provisioner/, +# never a shared .env value. + NOTIFICATION_PG_DB=nodedc_notifications NOTIFICATION_PG_USER=nodedc_notifications NOTIFICATION_PG_PASS=replace-with-random-synology-secret @@ -68,9 +114,41 @@ NODEDC_AI_WORKSPACE_ASSISTANT_URL=http://ai-workspace-assistant:18082 AI_WORKSPACE_OPS_ENTITLEMENT_URL=http://172.22.0.222:18190/api/internal/v1/ai-workspace/entitlements AI_WORKSPACE_OPS_ENTITLEMENT_TOKEN=replace-with-ops-agent-gateway-internal-token AI_WORKSPACE_OPS_ENTITLEMENT_REQUIRED=false +# Set only after Foundry passes Launcher handoff and Authentik access checks. +# Keep every existing entitlement adapter in this same JSON object. The adapter +# uses the existing NODE.DC internal service credential; no Foundry browser or +# worker secret is required. +# {"module-foundry":{"url":"https://foundry.nodedc.ru/api/ai-workspace/entitlements","required":false}} +AI_WORKSPACE_ENTITLEMENT_ADAPTERS_JSON= AI_WORKSPACE_HUB_TOKEN=replace-with-random-synology-secret AI_WORKSPACE_HUB_HOST_BIND=0.0.0.0:18081 AI_WORKSPACE_HUB_PUBLIC_URL=wss://ai-hub.nodedc.ru/api/ai-workspace/hub AI_WORKSPACE_HUB_INTERNAL_URL=https://ai-hub.nodedc.ru AI_WORKSPACE_HUB_FALLBACK_URLS= +AI_WORKSPACE_ONTOLOGY_MCP_ENABLED=true +# Optional public HTTP origin for the dynamic worker MCP URL. Normally it is +# derived from AI_WORKSPACE_HUB_PUBLIC_URL, so leave this empty. +AI_WORKSPACE_ONTOLOGY_MCP_PUBLIC_URL= +ONTOLOGY_CORE_HOST_BIND=127.0.0.1:18104 + +# Gelios Gateway — storage/read service. Provider credentials stay in the +# protected Engine Collector. Intake remains disabled until scope is approved. +GELIOS_TIMESCALE_IMAGE=timescale/timescaledb-ha:pg16.14-ts2.28.2-all +GELIOS_PG_DB=nodedc_gelios +GELIOS_PG_USER=nodedc_gelios +GELIOS_PG_PASS=replace-with-random-synology-secret +# URL-encode reserved characters in GELIOS_PG_PASS when forming this URL. +GELIOS_DATABASE_URL=postgresql://nodedc_gelios:replace-with-url-encoded-synology-secret@gelios-postgres:5432/nodedc_gelios +GELIOS_GATEWAY_HOST_BIND=127.0.0.1:18105 +# Explicit tenant and connection identifiers are deployment configuration; +# do not encode a customer or pilot name in source defaults. +GELIOS_TENANT_ID=replace-with-tenant-id +GELIOS_CONNECTION_ID=gelios-connection-id +# `all` accepts every unit returned by this approved tenant + connection. +GELIOS_UNIT_SCOPE=allowlist +GELIOS_ALLOWED_UNIT_IDS= +GELIOS_INTAKE_ENABLED=false +GELIOS_RAW_RETENTION_DAYS=14 +# Presentation state only; it does not change provider collection cadence. +GELIOS_POSITION_STALE_AFTER_MS=300000 diff --git a/infra/synology/README.md b/infra/synology/README.md index 5a1f78b..2609ef8 100644 --- a/infra/synology/README.md +++ b/infra/synology/README.md @@ -202,6 +202,26 @@ TASKER_SYNC_SOURCE=1 ./infra/synology/deploy-current.sh Если emergency-fix был сделан прямо на Synology в этих env-файлах, перенести sanitized-значение в `.env.synology.example`/docs, а секрет оставить только в live env. +### Gelios: сбор всех units доверенного подключения + +`GELIOS_UNIT_SCOPE=all` означает **все текущие и будущие units только одной уже +настроенной пары `GELIOS_TENANT_ID` + `GELIOS_CONNECTION_ID`**. Это не wildcard +на другие tenants или providers. Provider credential остаётся только в Engine. +Пропавший из очередного ответа unit не удаляется: его stable provider ID и +`last_seen_at` сохраняются; видимость на карте — отдельная логика витрины. + +Перед каноническим `nodedc-deploy apply` включить политику на Synology (скрипт +создаёт backup и не выводит секреты): + +```bash +sudo bash /volume1/docker/nodedc-deploy/inbox/prepare-gelios-all-units-env.sh +``` + +Затем применить узкий Platform-артефакт, собранный с `--gateway-only`, через +`nodedc-deploy`. В его plan должны быть только `gelios-postgres` и +`gelios-gateway`; общий `docker-compose` в такой архив не входит. После apply +Gateway начинает принимать весь состав units этого подключения. + ## AI Hub relay-only deploy Для локального тестирования с внешней Codex-машиной нельзя деплоить Launcher, Engine, Ops, Authentik или Tasker. Единственная допустимая удалённая точка в этой схеме — `ai-workspace-hub`, потому что удалённый worker должен иметь публичный WebSocket/HTTP relay. diff --git a/infra/synology/apply-current-runtime.sh b/infra/synology/apply-current-runtime.sh index 69f00a8..ba1cb84 100755 --- a/infra/synology/apply-current-runtime.sh +++ b/infra/synology/apply-current-runtime.sh @@ -34,12 +34,28 @@ echo "== ai workspace assistant image build ==" cd "${PLATFORM_DIR}/ai-workspace-assistant" "${DOCKER_BIN}" build --no-cache -t nodedc/ai-workspace-assistant:local . +echo "== ontology core image build ==" +cd "${PLATFORM_DIR}/ontology-core" +"${DOCKER_BIN}" build --no-cache -t nodedc/ontology-core:local . + +echo "== gelios gateway image build ==" +cd "${PLATFORM_DIR}/gelios-gateway" +"${DOCKER_BIN}" build --no-cache -t nodedc/gelios-gateway:local . + echo "== platform services recreate ==" cd "${PLATFORM_DIR}" "${DOCKER_BIN}" compose \ --env-file "${ENV_FILE}" \ -f "${COMPOSE_FILE}" \ - up -d --force-recreate reverse-proxy authentik-server authentik-worker notification-postgres notification-core ai-workspace-postgres ai-workspace-assistant ai-workspace-hub launcher + up -d --force-recreate reverse-proxy authentik-server authentik-worker notification-postgres notification-core ai-workspace-postgres ontology-core ai-workspace-assistant ai-workspace-hub gelios-postgres gelios-gateway launcher + +echo "== ontology core health check ==" +"${DOCKER_BIN}" exec nodedc-platform-ontology-core-1 sh -lc \ + 'node -e '"'"'fetch("http://127.0.0.1:18104/healthz").then(async (response) => { console.log(await response.text()); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(error); process.exit(1); })'"'"'' + +echo "== gelios gateway health check ==" +"${DOCKER_BIN}" exec nodedc-platform-gelios-gateway-1 sh -lc \ + 'node -e '"'"'fetch("http://127.0.0.1:18105/healthz").then(async (response) => { console.log(await response.text()); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(error); process.exit(1); })'"'"'' echo "== notification core health check ==" "${DOCKER_BIN}" exec nodedc-platform-notification-core-1 sh -lc \ diff --git a/infra/synology/backup-current.sh b/infra/synology/backup-current.sh index f7f30aa..90fd46c 100755 --- a/infra/synology/backup-current.sh +++ b/infra/synology/backup-current.sh @@ -48,6 +48,7 @@ rsync_dir "${NAS_ROOT}/platform/authentik/" "${BACKUP_DIR}/files/platform/authen rsync_dir "${NAS_ROOT}/platform/notification-core/" "${BACKUP_DIR}/files/platform/notification-core/" rsync_dir "${NAS_ROOT}/platform/ai-workspace-hub/" "${BACKUP_DIR}/files/platform/ai-workspace-hub/" rsync_dir "${NAS_ROOT}/platform/ai-workspace-assistant/" "${BACKUP_DIR}/files/platform/ai-workspace-assistant/" +rsync_dir "${NAS_ROOT}/platform/ontology-core/" "${BACKUP_DIR}/files/platform/ontology-core/" rsync_dir "${NAS_ROOT}/bim-viewer/source/server/data/" "${BACKUP_DIR}/files/bim-viewer/server-data/" rsync_file "${NAS_ROOT}/platform/.env.synology" "${BACKUP_DIR}/files/platform/" @@ -81,6 +82,7 @@ Contains: - Notification Core source/config: platform/notification-core, platform compose/env - AI Workspace Hub source/config: platform/ai-workspace-hub, platform compose/env - AI Workspace Assistant source/config: platform/ai-workspace-assistant, platform compose/env +- Ontology Core source/catalog: platform/ontology-core, platform compose/env - Tasker runtime config: tasker/plane-app/.env.synology, compose, Synology override - Ops Agents Gateway runtime config: ops-agents/.env, compose - BIM Viewer runtime data/config: bim-viewer/source/server/data, .env, compose diff --git a/infra/synology/deploy-current.sh b/infra/synology/deploy-current.sh index 67e1401..fd8ca09 100755 --- a/infra/synology/deploy-current.sh +++ b/infra/synology/deploy-current.sh @@ -84,6 +84,22 @@ rsync -av "${RSYNC_METADATA_ARGS[@]}" --delete \ "${PLATFORM_REPO}/services/ontology-core/" \ "${NAS_ROOT}/platform/ai-workspace-assistant/ontology-core/" +mkdir -p "${NAS_ROOT}/platform/ontology-core" +rsync -av "${RSYNC_METADATA_ARGS[@]}" --delete \ + --exclude='node_modules/' \ + --exclude='.env' \ + --exclude='.env.*' \ + "${PLATFORM_REPO}/services/ontology-core/" \ + "${NAS_ROOT}/platform/ontology-core/" + +mkdir -p "${NAS_ROOT}/platform/gelios-gateway" +rsync -av "${RSYNC_METADATA_ARGS[@]}" --delete \ + --exclude='node_modules/' \ + --exclude='.env' \ + --exclude='.env.*' \ + "${PLATFORM_REPO}/services/gelios-gateway/" \ + "${NAS_ROOT}/platform/gelios-gateway/" + if [[ "${SYNC_AUTHENTIK_TEMPLATES}" == "1" ]]; then mkdir -p "${NAS_ROOT}/authentik/custom-templates" rsync -av "${RSYNC_METADATA_ARGS[@]}" --delete \ @@ -240,7 +256,20 @@ fi cat <<'EOF' -Synced files to NAS mount. Run on Synology to apply runtime changes: +Synced source to the NAS mount and staged the root-owned runner candidate. + +For a canonical deploy, do not use the legacy direct runtime scripts below. +An administrator first installs/verifies the staged runner, then plans and +applies a data-only artifact from /volume1/docker/nodedc-deploy/inbox: + +sudo install -o root -g root -m 0755 \ + /volume1/docker/nodedc-deploy/runner-install/nodedc-deploy \ + /usr/local/sbin/nodedc-deploy +sudo /usr/local/sbin/nodedc-deploy verify-install +sudo /usr/local/sbin/nodedc-deploy plan /volume1/docker/nodedc-deploy/inbox/.tgz +sudo /usr/local/sbin/nodedc-deploy apply /volume1/docker/nodedc-deploy/inbox/.tgz + +Legacy direct runtime scripts (emergency/manual recovery only): cd /volume1/docker/nodedc-platform/platform sudo bash apply-current-runtime.sh @@ -276,14 +305,14 @@ cd /volume1/docker/nodedc-platform/platform sudo /usr/local/bin/docker compose \ --env-file /volume1/docker/nodedc-platform/platform/.env.synology \ -f /volume1/docker/nodedc-platform/platform/docker-compose.platform-http.yml \ - up -d --build --force-recreate --no-deps ai-workspace-hub ai-workspace-assistant notification-core launcher + up -d --build --force-recreate --no-deps ontology-core ai-workspace-hub ai-workspace-assistant notification-core launcher Optional Platform/Auth infra apply, only after deliberate compose/proxy/Auth templates changes: sudo /usr/local/bin/docker compose \ --env-file /volume1/docker/nodedc-platform/platform/.env.synology \ -f /volume1/docker/nodedc-platform/platform/docker-compose.platform-http.yml \ - up -d --build --force-recreate --no-deps reverse-proxy authentik-server authentik-worker ai-workspace-hub ai-workspace-assistant notification-core launcher + up -d --build --force-recreate --no-deps reverse-proxy authentik-server authentik-worker ontology-core ai-workspace-hub ai-workspace-assistant notification-core launcher After Authentik template rollout, clear global Brand custom CSS from the live DB. NODE.DC login CSS must be template-scoped, not stored in Brand.branding_custom_css: diff --git a/infra/synology/docker-compose.external-data-plane.yml b/infra/synology/docker-compose.external-data-plane.yml new file mode 100644 index 0000000..76eff31 --- /dev/null +++ b/infra/synology/docker-compose.external-data-plane.yml @@ -0,0 +1,96 @@ +name: nodedc-platform + +# This is intentionally an overlay, not a provider service. It is composed +# with docker-compose.platform-http.yml by the canonical deploy runner. +services: + external-data-plane-postgres: + image: ${EXTERNAL_DATA_PLANE_TIMESCALE_IMAGE:-timescale/timescaledb-ha:pg16.14-ts2.28.2-all} + restart: unless-stopped + env_file: + - ${NODEDC_SYNOLOGY_ENV_FILE:-.env.synology} + environment: + POSTGRES_DB: ${EXTERNAL_DATA_PLANE_PG_DB:-nodedc_data_plane} + POSTGRES_USER: ${EXTERNAL_DATA_PLANE_PG_USER:-nodedc_data_plane} + POSTGRES_PASSWORD: ${EXTERNAL_DATA_PLANE_PG_PASS:?external data plane database password required} + healthcheck: + test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 30s + volumes: + - external-data-plane-database:/home/postgres/pgdata/data + networks: + - external-data-plane + + external-data-plane: + image: nodedc/external-data-plane:local + restart: unless-stopped + env_file: + - ${NODEDC_SYNOLOGY_ENV_FILE:-.env.synology} + environment: + NODE_ENV: production + PORT: 18106 + EXTERNAL_DATA_PLANE_DATABASE_URL: postgresql://${EXTERNAL_DATA_PLANE_PG_USER:-nodedc_data_plane}:${EXTERNAL_DATA_PLANE_PG_PASS:?external data plane database password required}@external-data-plane-postgres:5432/${EXTERNAL_DATA_PLANE_PG_DB:-nodedc_data_plane} + EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE: ${EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE:-10} + EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS: ${EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS:-14} + EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES: ${EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES:-5242880} + EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH: ${EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH:-5000} + EXTERNAL_DATA_PLANE_MAX_ATTRIBUTES_BYTES_PER_FACT: ${EXTERNAL_DATA_PLANE_MAX_ATTRIBUTES_BYTES_PER_FACT:-65536} + EXTERNAL_DATA_PLANE_MAX_PATCH_OPERATIONS: ${EXTERNAL_DATA_PLANE_MAX_PATCH_OPERATIONS:-500} + EXTERNAL_DATA_PLANE_MAX_PATCH_BYTES: ${EXTERNAL_DATA_PLANE_MAX_PATCH_BYTES:-262144} + EXTERNAL_DATA_PLANE_PATCH_RETENTION_MS: ${EXTERNAL_DATA_PLANE_PATCH_RETENTION_MS:-3600000} + EXTERNAL_DATA_PLANE_RECEIPT_RETENTION_MS: ${EXTERNAL_DATA_PLANE_RECEIPT_RETENTION_MS:-604800000} + EXTERNAL_DATA_PLANE_RETENTION_DELETE_LIMIT: ${EXTERNAL_DATA_PLANE_RETENTION_DELETE_LIMIT:-10000} + EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS: ${EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS:-20000} + EXTERNAL_DATA_PLANE_STREAM_POLL_MS: ${EXTERNAL_DATA_PLANE_STREAM_POLL_MS:-1000} + EXTERNAL_DATA_PLANE_MAX_READER_STREAMS: ${EXTERNAL_DATA_PLANE_MAX_READER_STREAMS:-10} + EXTERNAL_DATA_PLANE_WRITER_BINDING_MAX_TTL_DAYS: ${EXTERNAL_DATA_PLANE_WRITER_BINDING_MAX_TTL_DAYS:-90} + EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS: ${EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS:-300} + EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: ${EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS:-3600000} + # Migration-only shared-token routes stay closed in a canonical install. + EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED: ${EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED:-false} + NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} + # Runner-owned secret file. This credential must never live in the + # shared .env.synology that is inherited by unrelated services. + EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: /run/nodedc-secrets/external-data-plane-provisioner-token + # This stays false until the dedicated Engine provisioner is deployed and + # receives the same read-only secret mount. + EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "false" + volumes: + - type: bind + source: /volume1/docker/nodedc-platform/secrets/external-data-plane-provisioner/token + target: /run/nodedc-secrets/external-data-plane-provisioner-token + read_only: true + bind: + create_host_path: false + expose: + - "18106" + ports: + - "${EXTERNAL_DATA_PLANE_HOST_BIND:-127.0.0.1:18106}:18106" + depends_on: + external-data-plane-postgres: + condition: service_healthy + healthcheck: + test: + - CMD + - node + - -e + - "fetch('http://127.0.0.1:18106/healthz').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))" + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + networks: + - engine + - external-data-plane + +networks: + engine: + external: true + name: ${NODEDC_ENGINE_DOCKER_NETWORK:-nodedc-demo_default} + external-data-plane: + internal: true + +volumes: + external-data-plane-database: diff --git a/infra/synology/docker-compose.platform-http.yml b/infra/synology/docker-compose.platform-http.yml index fc07c2d..ecde896 100644 --- a/infra/synology/docker-compose.platform-http.yml +++ b/infra/synology/docker-compose.platform-http.yml @@ -142,6 +142,8 @@ services: AI_WORKSPACE_HUB_INTERNAL_URL: ${AI_WORKSPACE_HUB_INTERNAL_URL:-https://ai-hub.nodedc.ru} AI_WORKSPACE_HUB_TOKEN: ${AI_WORKSPACE_HUB_TOKEN:?ai workspace hub token required} AI_WORKSPACE_HUB_FALLBACK_URLS: ${AI_WORKSPACE_HUB_FALLBACK_URLS:-} + AI_WORKSPACE_ONTOLOGY_MCP_ENABLED: ${AI_WORKSPACE_ONTOLOGY_MCP_ENABLED:-true} + AI_WORKSPACE_ENTITLEMENT_ADAPTERS_JSON: ${AI_WORKSPACE_ENTITLEMENT_ADAPTERS_JSON:-} NDC_ONTOLOGY_LAUNCHER_BASE_URL: http://launcher:5173 expose: - "18082" @@ -154,6 +156,160 @@ services: - identity - engine + ontology-core: + image: nodedc/ontology-core:local + build: + context: ./ontology-core + restart: unless-stopped + env_file: + - ${NODEDC_SYNOLOGY_ENV_FILE:-.env.synology} + environment: + NODE_ENV: production + PORT: 18104 + NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} + expose: + - "18104" + ports: + - "${ONTOLOGY_CORE_HOST_BIND:-127.0.0.1:18104}:18104" + healthcheck: + test: + - CMD + - node + - -e + - "fetch('http://127.0.0.1:18104/healthz').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))" + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + networks: + - engine + + # The provider database is private. Gateway is the only component that can + # reach it; the host bind is loopback-only for audited operator diagnostics. + gelios-postgres: + image: ${GELIOS_TIMESCALE_IMAGE:-timescale/timescaledb-ha:pg16.14-ts2.28.2-all} + restart: unless-stopped + env_file: + - ${NODEDC_SYNOLOGY_ENV_FILE:-.env.synology} + environment: + POSTGRES_DB: ${GELIOS_PG_DB:-nodedc_gelios} + POSTGRES_USER: ${GELIOS_PG_USER:-nodedc_gelios} + POSTGRES_PASSWORD: ${GELIOS_PG_PASS:?gelios database password required} + healthcheck: + test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 30s + volumes: + - gelios-database:/home/postgres/pgdata/data + networks: + - gelios-data + + gelios-gateway: + image: nodedc/gelios-gateway:local + build: + context: ./gelios-gateway + restart: unless-stopped + env_file: + - ${NODEDC_SYNOLOGY_ENV_FILE:-.env.synology} + environment: + NODE_ENV: production + PORT: 18105 + DATABASE_URL: ${GELIOS_DATABASE_URL:?gelios database URL required} + NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} + GELIOS_TENANT_ID: ${GELIOS_TENANT_ID:?gelios tenant id required} + GELIOS_CONNECTION_ID: ${GELIOS_CONNECTION_ID:?gelios connection id required} + GELIOS_UNIT_SCOPE: ${GELIOS_UNIT_SCOPE:-allowlist} + GELIOS_ALLOWED_UNIT_IDS: ${GELIOS_ALLOWED_UNIT_IDS:-} + GELIOS_INTAKE_ENABLED: ${GELIOS_INTAKE_ENABLED:-false} + GELIOS_RAW_RETENTION_DAYS: ${GELIOS_RAW_RETENTION_DAYS:-14} + GELIOS_POSITION_STALE_AFTER_MS: ${GELIOS_POSITION_STALE_AFTER_MS:-300000} + expose: + - "18105" + ports: + - "${GELIOS_GATEWAY_HOST_BIND:-127.0.0.1:18105}:18105" + depends_on: + gelios-postgres: + condition: service_healthy + healthcheck: + test: + - CMD + - node + - -e + - "fetch('http://127.0.0.1:18105/healthz').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))" + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + networks: + - engine + - gelios-data + + # One private Platform gateway and one NAS-resident persistent cache serve + # every Foundry Application/Map Page instance. It is not published to a browser. + map-gateway: + image: nodedc/map-gateway:local + build: + context: ../../services/map-gateway + restart: unless-stopped + env_file: + - ${NODEDC_SYNOLOGY_ENV_FILE:-.env.synology} + environment: + NODE_ENV: production + PORT: 18103 + # Optional legacy bootstrap. The canonical mutable value, when an + # authorized Foundry admin saves it, is the private NAS secret file + # inside MAP_CACHE_DIR and never a deploy artifact or browser env. + CESIUM_ION_TOKEN: ${CESIUM_ION_TOKEN:-} + # Runner-owned file mount; the secret itself is never an env value. + NODEDC_MAP_GATEWAY_ADMIN_SECRET_FILE: /run/nodedc-secrets/map-gateway-admin-secret + # DC AMD Proxy preserves the AMD workstation's VPN-only Cesium route. + # The fixed NAS LAN listener is authenticated by the same runner-managed + # file token; neither token is a browser or Foundry env. + MAP_GATEWAY_EGRESS_URL: http://172.22.0.222:8790 + MAP_GATEWAY_EGRESS_PROXY_TOKEN_FILE: /run/nodedc-secrets/map-egress-proxy-token + CESIUM_ION_ASSET_ALLOWLIST: ${CESIUM_ION_ASSET_ALLOWLIST:-1,2,96188} + MAP_CACHE_DIR: /var/lib/nodedc-map-live-cache + MAP_OFFLINE_SNAPSHOT_DIR: /var/lib/nodedc-map-offline-snapshot + MAP_CACHE_MODE: ${MAP_CACHE_MODE:-readwrite} + MAP_CACHE_MAX_MB: ${MAP_CACHE_MAX_MB:-20480} + MAP_GATEWAY_ALLOW_ANONYMOUS: "false" + MAP_GATEWAY_TRUSTED_SUBJECT_HEADER: x-nodedc-user-id + MAP_GATEWAY_UPSTREAM_ALLOWLIST: ${MAP_GATEWAY_UPSTREAM_ALLOWLIST:-api.cesium.com,assets.ion.cesium.com,tile.openstreetmap.org,dev.virtualearth.net,ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtualearth.net,ecn.t2.tiles.virtualearth.net,ecn.t3.tiles.virtualearth.net} + MAP_GATEWAY_LEGACY_CACHE_HOSTS: ${MAP_GATEWAY_LEGACY_CACHE_HOSTS:-} + MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST: ${MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST:-} + volumes: + - type: bind + source: ${MAP_LIVE_CACHE_HOST_DIR:-/volume1/docker/nodedc-platform/map-gateway/live-tile-cache} + target: /var/lib/nodedc-map-live-cache + - type: bind + source: ${MAP_OFFLINE_SNAPSHOT_HOST_DIR:-/volume1/docker/nodedc-platform/map-gateway/offline-snapshot} + target: /var/lib/nodedc-map-offline-snapshot + read_only: true + - type: bind + source: /volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret + target: /run/nodedc-secrets/map-gateway-admin-secret + read_only: true + - type: bind + source: /volume1/docker/nodedc-platform/secrets/map-egress-proxy-token + target: /run/nodedc-secrets/map-egress-proxy-token + read_only: true + # Operator-only loopback binding for the root-owned deploy healthcheck. + # Foundry reaches the service over the private Docker network; browsers + # never receive this address. + ports: + - "${MAP_GATEWAY_HOST_BIND:-127.0.0.1:18103}:18103" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:18103/healthz',{headers:{'x-nodedc-user-id':'healthcheck'}}).then((response)=>process.exit(response.ok?0:1)).catch(()=>process.exit(1))"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + networks: + - engine + - map-egress + ai-workspace-hub: image: nodedc/ai-workspace-hub:local build: @@ -167,11 +323,15 @@ services: AI_WORKSPACE_HUB_TOKEN: ${AI_WORKSPACE_HUB_TOKEN:?ai workspace hub token required} NODEDC_INTERNAL_ACCESS_TOKEN: ${NODEDC_INTERNAL_ACCESS_TOKEN:-} NODEDC_AI_WORKSPACE_ASSISTANT_URL: http://ai-workspace-assistant:18082 + NODEDC_ONTOLOGY_CORE_URL: http://ontology-core:18104 AI_WORKSPACE_HUB_WS_PATH: /api/ai-workspace/hub expose: - "18081" ports: - "${AI_WORKSPACE_HUB_HOST_BIND:-0.0.0.0:18081}:18081" + depends_on: + ontology-core: + condition: service_healthy networks: - edge - engine @@ -256,6 +416,11 @@ networks: name: ${NODEDC_ENGINE_DOCKER_NETWORK:-nodedc-demo_default} identity: internal: true + gelios-data: + internal: true + map-egress: + external: true + name: nodedc-map-egress volumes: authentik-database: @@ -263,5 +428,6 @@ volumes: authentik-certs: notification-database: ai-workspace-database: + gelios-database: caddy-data: caddy-config: diff --git a/infra/synology/prepare-external-data-plane-env.sh b/infra/synology/prepare-external-data-plane-env.sh new file mode 100755 index 0000000..1cb52e9 --- /dev/null +++ b/infra/synology/prepare-external-data-plane-env.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu + +ENV_FILE=${1:-/volume1/docker/nodedc-platform/platform/.env.synology} + +if [ ! -f "$ENV_FILE" ]; then + echo "ERROR: env file not found: $ENV_FILE" >&2 + exit 1 +fi + +if grep -q '^EXTERNAL_DATA_PLANE_PG_PASS=' "$ENV_FILE"; then + echo "External Data Plane database credential already exists; no change made." + exit 0 +fi + +if ! command -v openssl >/dev/null 2>&1; then + echo "ERROR: openssl is required to generate the database password." >&2 + exit 1 +fi + +umask 077 +TMP_FILE=$(mktemp "${ENV_FILE}.external-data-plane.XXXXXX") +trap 'rm -f "$TMP_FILE"' EXIT HUP INT TERM +PASSWORD=$(openssl rand -hex 32) + +cp "$ENV_FILE" "$TMP_FILE" +printf '\n# External Data Plane — generated locally; do not reuse an internal API token.\n' >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_TIMESCALE_IMAGE=timescale/timescaledb-ha:pg16.14-ts2.28.2-all\n' >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_PG_DB=nodedc_data_plane\n' >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_PG_USER=nodedc_data_plane\n' >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_PG_PASS=%s\n' "$PASSWORD" >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_HOST_BIND=127.0.0.1:18106\n' >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE=10\n' >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS=14\n' >> "$TMP_FILE" +printf 'EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES=5242880\n' >> "$TMP_FILE" + +mv "$TMP_FILE" "$ENV_FILE" +trap - EXIT HUP INT TERM +echo "External Data Plane database configuration was added. The generated password was not printed." diff --git a/infra/synology/prepare-gelios-all-units-env.sh b/infra/synology/prepare-gelios-all-units-env.sh new file mode 100755 index 0000000..05af078 --- /dev/null +++ b/infra/synology/prepare-gelios-all-units-env.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script is intentionally run on the Synology host, before the canonical +# platform artifact apply. It changes collection policy only; provider +# credentials remain in the protected Engine credential store. +NAS_ROOT="${NAS_ROOT:-/volume1/docker/nodedc-platform}" +PLATFORM_ENV="${PLATFORM_ENV:-${NAS_ROOT}/platform/.env.synology}" +BACKUP_ROOT="${BACKUP_ROOT:-/volume1/docker/nodedc-deploy/backups/gelios-policy}" +TIMESTAMP="${TIMESTAMP:-$(date +%Y%m%d-%H%M%S)}" +BACKUP_DIR="${BACKUP_DIR:-${BACKUP_ROOT}/${TIMESTAMP}}" + +read_env() { + local file="$1" + local key="$2" + awk -F= -v key="${key}" ' + $0 ~ "^[[:space:]]*#" { next } + $1 == key { + value = substr($0, index($0, "=") + 1) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + gsub(/^"|"$/, "", value) + print value + exit + } + ' "${file}" +} + +set_env_value() { + local file="$1" + local key="$2" + local value="$3" + local tmp + tmp="$(mktemp "${file}.tmp.XXXXXX")" + awk -v key="${key}" -v value="${value}" ' + BEGIN { replaced = 0 } + $0 ~ "^[[:space:]]*#" { print; next } + index($0, key "=") == 1 { + print key "=" value + replaced = 1 + next + } + { print } + END { + if (!replaced) print key "=" value + } + ' "${file}" > "${tmp}" + chmod 600 "${tmp}" + mv "${tmp}" "${file}" +} + +if [[ ! -f "${PLATFORM_ENV}" ]]; then + echo "missing Platform env: ${PLATFORM_ENV}" >&2 + exit 1 +fi + +tenant_id="$(read_env "${PLATFORM_ENV}" GELIOS_TENANT_ID || true)" +connection_id="$(read_env "${PLATFORM_ENV}" GELIOS_CONNECTION_ID || true)" +if [[ -z "${tenant_id}" || -z "${connection_id}" || "${tenant_id}" == replace-with-* || "${connection_id}" == replace-with-* ]]; then + echo "GELIOS_TENANT_ID and GELIOS_CONNECTION_ID must be configured before enabling all-unit collection" >&2 + exit 1 +fi + +mkdir -p "${BACKUP_DIR}" +backup_path="${BACKUP_DIR}/platform.env.synology.before-gelios-all-units" +cp "${PLATFORM_ENV}" "${backup_path}" +chmod 600 "${backup_path}" + +# `all` is constrained to the one tenant + connection validated above; it is +# not a wildcard across Gelios tenants or other providers. Existing stable +# unit rows are never deleted by this policy. +set_env_value "${PLATFORM_ENV}" GELIOS_UNIT_SCOPE "all" +set_env_value "${PLATFORM_ENV}" GELIOS_INTAKE_ENABLED "true" + +cat < { console.log(await response.text()); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(error); process.exit(1); })'"'"'' +echo "== ontology core health check ==" +"${DOCKER_BIN}" exec nodedc-platform-ontology-core-1 sh -lc \ + 'node -e '"'"'fetch("http://127.0.0.1:18104/healthz").then(async (response) => { console.log(await response.text()); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(error); process.exit(1); })'"'"'' + +echo "== gelios gateway health check ==" +"${DOCKER_BIN}" exec nodedc-platform-gelios-gateway-1 sh -lc \ + 'node -e '"'"'fetch("http://127.0.0.1:18105/healthz").then(async (response) => { console.log(await response.text()); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(error); process.exit(1); })'"'"'' + echo "== ai workspace assistant hub target check ==" "${DOCKER_BIN}" exec nodedc-platform-ai-workspace-assistant-1 sh -lc \ 'test "$AI_WORKSPACE_HUB_PUBLIC_URL" = "wss://ai-hub.nodedc.ru/api/ai-workspace/hub" && test -z "$AI_WORKSPACE_HUB_FALLBACK_URLS" && echo ai-workspace-prod-hub-target-ok'