From 569b8762e6522a585dd70ec9023ccb5f3adbde6a Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 02:23:34 +0300 Subject: [PATCH] feat(data-plane): add provider contracts and ontology delivery --- docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md | 143 +++ docs/ADR_GELIOS_DATA_PLANE.md | 107 ++ docs/ADR_L2_OWNED_EXTERNAL_CONNECTORS.md | 274 +++++ packages/external-provider-contract/README.md | 302 +++++ .../examples/gelios-positions-current.v1.mjs | 76 ++ .../external-provider-contract/package.json | 10 + .../src/data-product.mjs | 197 ++++ .../src/engine-credential-sink.mjs | 662 +++++++++++ .../src/engine-private-extension.mjs | 721 ++++++++++++ .../external-provider-contract/src/index.mjs | 481 ++++++++ .../test/contract.test.mjs | 267 +++++ .../test/data-product.test.mjs | 111 ++ .../test/engine-credential-sink.test.mjs | 242 ++++ .../test/engine-private-extension.test.mjs | 370 ++++++ services/external-data-plane/Dockerfile | 32 + .../fleet.positions.current.v1.json | 30 + .../external-data-plane/package-lock.json | 1017 +++++++++++++++++ services/external-data-plane/package.json | 18 + services/external-data-plane/src/config.mjs | 75 ++ .../src/data-product-delivery.mjs | 527 +++++++++ .../src/data-product-policy.mjs | 118 ++ .../external-data-plane/src/definitions.mjs | 43 + .../external-data-plane/src/intake-policy.mjs | 37 + .../src/reader-binding.mjs | 90 ++ services/external-data-plane/src/schema.mjs | 206 ++++ services/external-data-plane/src/server.mjs | 822 +++++++++++++ .../src/writer-binding.mjs | 182 +++ .../test/api.integration.test.mjs | 280 +++++ .../external-data-plane/test/config.test.mjs | 51 + ...data-product-delivery.integration.test.mjs | 184 +++ .../test/data-product-policy.test.mjs | 81 ++ .../test/definitions.test.mjs | 32 + .../test/intake-policy.test.mjs | 26 + .../test/reader-binding.test.mjs | 25 + .../test/writer-binding.test.mjs | 117 ++ services/gelios-gateway/Dockerfile | 23 + services/gelios-gateway/README.md | 70 ++ services/gelios-gateway/package-lock.json | 1008 ++++++++++++++++ services/gelios-gateway/package.json | 15 + services/gelios-gateway/src/capabilities.mjs | 26 + services/gelios-gateway/src/config.mjs | 80 ++ services/gelios-gateway/src/data-products.mjs | 80 ++ services/gelios-gateway/src/normalize.mjs | 92 ++ services/gelios-gateway/src/schema.mjs | 116 ++ services/gelios-gateway/src/scripts/smoke.mjs | 59 + services/gelios-gateway/src/server.mjs | 406 +++++++ services/ontology-core/Dockerfile | 16 + services/ontology-core/README.md | 6 + services/ontology-core/catalog/aliases.json | 3 +- .../domain-packages/gelios/aliases.json | 29 + .../domain-packages/gelios/entities.json | 34 + .../domain-packages/gelios/evidence.json | 49 + .../domain-packages/gelios/guardrails.json | 57 + .../domain-packages/gelios/package.json | 7 + .../domain-packages/gelios/relations.json | 44 + .../domain-packages/integration/aliases.json | 11 + .../domain-packages/integration/entities.json | 22 + .../domain-packages/integration/evidence.json | 12 + .../integration/guardrails.json | 18 + .../domain-packages/integration/package.json | 7 + .../integration/relations.json | 20 + .../catalog/domain-packages/map/entities.json | 2 +- services/ontology-core/catalog/entities.json | 12 +- services/ontology-core/catalog/relations.json | 8 +- .../ontology-core/catalog/resolver-rules.json | 2 +- .../ontology-core/docs/CONTEXT_BINDINGS.md | 2 +- .../docs/GELIOS_DOMAIN_ONTOLOGY.md | 109 ++ .../ontology-core/docs/MAP_DOMAIN_ONTOLOGY.md | 27 + .../docs/baseline/CANONICAL_ENTITY_CATALOG.md | 13 +- .../EVIDENCE_HARDENING_PLAN_V0_3_1.md | 10 +- .../docs/baseline/EVIDENCE_LEDGER.md | 5 +- ...VIDENCE_LEDGER_ENGINE_DIRTY_BOUNDARY_P1.md | 18 +- .../EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md | 23 +- .../ontology-core/docs/baseline/GUARDRAILS.md | 3 +- .../docs/baseline/RELATION_CATALOG.md | 9 +- .../docs/baseline/TERMS_GLOSSARY.md | 9 +- .../engine-workflow-manifest.example.json | 4 +- .../gelios-map-moving-object.fixture.json | 25 + services/ontology-core/package.json | 2 + services/ontology-core/src/catalog.mjs | 11 +- services/ontology-core/src/mcp-server.mjs | 581 ++++++++++ services/ontology-core/src/registry.mjs | 2 +- services/ontology-core/src/resolver.mjs | 2 +- .../ontology-core/src/scripts/smoke-mcp.mjs | 95 ++ 84 files changed, 11170 insertions(+), 70 deletions(-) create mode 100644 docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md create mode 100644 docs/ADR_GELIOS_DATA_PLANE.md create mode 100644 docs/ADR_L2_OWNED_EXTERNAL_CONNECTORS.md create mode 100644 packages/external-provider-contract/README.md create mode 100644 packages/external-provider-contract/examples/gelios-positions-current.v1.mjs create mode 100644 packages/external-provider-contract/package.json create mode 100644 packages/external-provider-contract/src/data-product.mjs create mode 100644 packages/external-provider-contract/src/engine-credential-sink.mjs create mode 100644 packages/external-provider-contract/src/engine-private-extension.mjs create mode 100644 packages/external-provider-contract/src/index.mjs create mode 100644 packages/external-provider-contract/test/contract.test.mjs create mode 100644 packages/external-provider-contract/test/data-product.test.mjs create mode 100644 packages/external-provider-contract/test/engine-credential-sink.test.mjs create mode 100644 packages/external-provider-contract/test/engine-private-extension.test.mjs create mode 100644 services/external-data-plane/Dockerfile create mode 100644 services/external-data-plane/definitions/fleet.positions.current.v1.json create mode 100644 services/external-data-plane/package-lock.json create mode 100644 services/external-data-plane/package.json create mode 100644 services/external-data-plane/src/config.mjs create mode 100644 services/external-data-plane/src/data-product-delivery.mjs create mode 100644 services/external-data-plane/src/data-product-policy.mjs create mode 100644 services/external-data-plane/src/definitions.mjs create mode 100644 services/external-data-plane/src/intake-policy.mjs create mode 100644 services/external-data-plane/src/reader-binding.mjs create mode 100644 services/external-data-plane/src/schema.mjs create mode 100644 services/external-data-plane/src/server.mjs create mode 100644 services/external-data-plane/src/writer-binding.mjs create mode 100644 services/external-data-plane/test/api.integration.test.mjs create mode 100644 services/external-data-plane/test/config.test.mjs create mode 100644 services/external-data-plane/test/data-product-delivery.integration.test.mjs create mode 100644 services/external-data-plane/test/data-product-policy.test.mjs create mode 100644 services/external-data-plane/test/definitions.test.mjs create mode 100644 services/external-data-plane/test/intake-policy.test.mjs create mode 100644 services/external-data-plane/test/reader-binding.test.mjs create mode 100644 services/external-data-plane/test/writer-binding.test.mjs create mode 100644 services/gelios-gateway/Dockerfile create mode 100644 services/gelios-gateway/README.md create mode 100644 services/gelios-gateway/package-lock.json create mode 100644 services/gelios-gateway/package.json create mode 100644 services/gelios-gateway/src/capabilities.mjs create mode 100644 services/gelios-gateway/src/config.mjs create mode 100644 services/gelios-gateway/src/data-products.mjs create mode 100644 services/gelios-gateway/src/normalize.mjs create mode 100644 services/gelios-gateway/src/schema.mjs create mode 100644 services/gelios-gateway/src/scripts/smoke.mjs create mode 100644 services/gelios-gateway/src/server.mjs create mode 100644 services/ontology-core/Dockerfile create mode 100644 services/ontology-core/catalog/domain-packages/gelios/aliases.json create mode 100644 services/ontology-core/catalog/domain-packages/gelios/entities.json create mode 100644 services/ontology-core/catalog/domain-packages/gelios/evidence.json create mode 100644 services/ontology-core/catalog/domain-packages/gelios/guardrails.json create mode 100644 services/ontology-core/catalog/domain-packages/gelios/package.json create mode 100644 services/ontology-core/catalog/domain-packages/gelios/relations.json create mode 100644 services/ontology-core/catalog/domain-packages/integration/aliases.json create mode 100644 services/ontology-core/catalog/domain-packages/integration/entities.json create mode 100644 services/ontology-core/catalog/domain-packages/integration/evidence.json create mode 100644 services/ontology-core/catalog/domain-packages/integration/guardrails.json create mode 100644 services/ontology-core/catalog/domain-packages/integration/package.json create mode 100644 services/ontology-core/catalog/domain-packages/integration/relations.json create mode 100644 services/ontology-core/docs/GELIOS_DOMAIN_ONTOLOGY.md create mode 100644 services/ontology-core/examples/gelios-map-moving-object.fixture.json create mode 100644 services/ontology-core/src/mcp-server.mjs create mode 100644 services/ontology-core/src/scripts/smoke-mcp.mjs diff --git a/docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md b/docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md new file mode 100644 index 0000000..292d5ac --- /dev/null +++ b/docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md @@ -0,0 +1,143 @@ +# ADR: канон External Provider Data Plane + +Статус: **superseded**. +Дата: 2026-07-13. +Владелец решения: NODE.DC Platform. + +> Заменено 2026-07-14 документом +> [`ADR_L2_OWNED_EXTERNAL_CONNECTORS.md`](ADR_L2_OWNED_EXTERNAL_CONNECTORS.md). +> Этот черновик неверно помещал provider adapter, normalisation и collection +> policy в `services/-gateway`. Новое правило: adapter принадлежит +> изолированному L2 workflow; Platform Data Plane остаётся provider-neutral. + +## Контекст + +Клиент может подключить к NODE.DC любой внешний продукт: телеметрию, ERP, +роботов, энергетику, BIM-систему или иной источник данных. Gelios Pro для +Gelios — первый конкретный поставщик для проверки канона, а не исключительная +архитектурная ветка. Нельзя превращать Engine workflow, Ontology Core или общую БД Platform +в место, куда попадают токены, raw payloads и частная логика каждого API. + +Нужна повторяемая форма, в которой новый provider добавляется как отдельный +adapter, но получает общие правила scope, секретов, collection, хранения, +read-model, аудита и безопасной публикации в NDC. + +## Решение + +1. В `platform/packages/external-provider-contract` живёт общий versioned + контракт интеграции. Это не runtime и не база данных. Он задаёт форму + `provider`, `connection`, `capability catalog`, `credential reference`, + `access scope`, `field policy`, `collection profile`, `retention policy`, + `read model` и красный command-domain. +2. Каждый provider получает самостоятельный app-owned adapter в + `platform/services/-gateway`. Первый экземпляр — + `platform/services/gelios-gateway`. Adapter владеет intake contract, rate + budget, normalisation, collection policy, storage и своим internal + read/realtime API. Сам provider secret остаётся в Engine Credentials и + доступен только назначенному защищённому execution workflow. +3. Один provider service может обслуживать много клиентов. Каждая строка, + cursor, audit-event и read-model обязана иметь `tenant_id` и + `connection_id`; видимость provider account сама по себе не является + продуктовым scope. Для клиента с отдельными требованиями изоляции допустим + отдельный deployment/database profile без изменения контракта. +4. База принадлежит adapter-сервису, а не Ontology Core, Engine, Tasker или + общему Platform Postgres. Для пространственно-временного Gelios-кейса + базой служит PostgreSQL 16 + TimescaleDB + PostGIS (`gelios-postgres`). + Другой provider может выбрать иной storage engine только через явный ADR, + сохранив внешний контракт. +5. Ontology Core хранит только provider-neutral и provider-specific смыслы, + связи, правила и контракты. Он не хранит credentials, runtime telemetry, + customer raw payloads или renderer objects. Enforcement остаётся в + gateway/adapters. Engine Credentials хранит provider secret в границе + специально назначенного execution workflow; значение не сериализуется в + граф, ontology, логи, read-model или UI. +6. Engine L2 Collector использует credential reference и получает разрешённые + данные поставщика. Он передаёт в adapter только аутентифицированный нормализованный + intake payload. Остальные L2 workflow получают исключительно scoped + internal API/event contract и не читают gateway DB напрямую. + +## Каноническая форма нового подключения + +```text +Client / tenant + -> Engine Credential + protected Collector workflow + -> provider connection instance + -> provider adapter (safe intake policy + normalizer) + -> provider-owned storage and projections + -> internal read/realtime contract + -> L2 workflow / approved interface binding + -> renderer adapter +``` + +Каждый новый provider добавляет только свой adapter package, capability +catalog, schema mappings, scrubbed fixtures и domain ontology package. Он не +добавляет отдельную схему доступа к Engine/Studio и не создаёт прямой путь из +browser в provider API. + +## Полнота данных без неконтролируемого объёма + +«Предусмотреть все данные» означает каталогизировать каждую provider +capability и поле, а не опрашивать весь account на максимальной частоте. +Collection profile явно решает, какие safe-read capabilities, поля, scope и +частота включены в конкретной connection instance. + +| Слой | Что хранится | Режим | +| --- | --- | --- | +| Capability catalog | documented endpoint/read-field/command capability и его риск | versioned source + ontology | +| Inventory/configuration | units, devices, groups, sensors, custom definitions | медленный reconcile | +| Current projection | последняя разрешённая позиция, состояние и display fields | idempotent upsert | +| Event history | нормализованные события и approved measurements | append-only, partitioned | +| Raw envelope | полный safe-read ответ с provenance и hash | restricted cold layer, retention-bound | +| Aggregates/features | rollups и признаки для аналитики/предиктива | derived, replaceable | + +Raw envelope не выдаётся UI и не становится таблицей «всё в JSON навсегда». +Вначале он может быть compressed/partitioned storage с метаданными в БД; при +реальном объёме переносится в object storage, а PostgreSQL хранит immutable +index, hash, policy и ссылку. Retention, raw depth и downsampling утверждаются +после замера сообщений/сек, размера payload, требуемой истории, RPO/RTO и +стоимости. Так инженер может запросить ранее не показанное поле из +каталога/архива, не раздувая горячую read-модель. + +## Realtime и интерфейс + +Частота provider collection, обновления `current projection` и выдачи в UI — +три разные настройки. Например, map consumer может получать выбранную +read-модель раз в 3 секунды, но это не даёт ему права опрашивать Gelios раз в +3 секунды или создавать отдельный polling loop на каждого зрителя. + +Gateway сначала обновляет одну current projection и публикует change event. +L2/Map binding затем может sampling/throttle этот поток по утверждённой +настройке интерфейса. Источник истины для live state — gateway storage, не +долгоживущий workflow и не Cesium session. + +## Commands: моделируются, но не подключаются + +Command templates, параметры, delivery states и audit входят в capability +catalog и ontology полностью. Read adapter не содержит send route и не +использует command capability. В будущем command execution создаётся только +как отдельный `provider-command-gateway`/red-domain deployment с явным +человеческим подтверждением, role/scope check, idempotency, audit и отдельным +security review. До этого команда не может быть отправлена из collector, L2, +Map или AI Workspace. + +## Обязательные артефакты каждого adapter + +- `provider manifest`: provider id, adapter version, auth modes, rate limits; +- capability and field catalog: read/write classification, source evidence, + pagination and error semantics; +- connection profile: tenant, secret reference, approved scope, field policy, + collection and retention profile; +- normalised contract and migrations; scrubbed fixtures and contract tests; +- health/metrics/audit without secrets or raw personal data; +- ontology package with stable subjects, relations and guardrails; +- internal read/realtime API contract; no browser/provider bypass. + +## Не решено этим ADR + +- конкретный deployment topology и HA/PITR target для каждого volume; +- выбор object storage после real-volume measurement; +- L2 stream execution contract и Module Studio binding implementation; +- параметры first Gelios collection profile и owner-approved connection scope. + +Gelios-specific применение этого решения описано в +`docs/ADR_GELIOS_DATA_PLANE.md`. diff --git a/docs/ADR_GELIOS_DATA_PLANE.md b/docs/ADR_GELIOS_DATA_PLANE.md new file mode 100644 index 0000000..6e54cb8 --- /dev/null +++ b/docs/ADR_GELIOS_DATA_PLANE.md @@ -0,0 +1,107 @@ +# ADR: Gelios adapter в External Provider Data Plane + +Статус: **superseded**. +Дата: 2026-07-13. +Владелец решения: NODE.DC Platform. + +> Заменено 2026-07-14 документом +> [`ADR_L2_OWNED_EXTERNAL_CONNECTORS.md`](ADR_L2_OWNED_EXTERNAL_CONNECTORS.md). +> Gelios остаётся domain/ontology примером, но не получает отдельный +> provider-owned gateway: fetch, mapping и collection policy живут в его L2 +> connector instance; Platform предоставляет нейтральный Data Plane. + +## Контекст + +Gelios Pro поставляет непрерывную телеметрию, конфигурацию устройств и пространственные данные. В текущем доступе подтверждены 107 видимых units, из них 105 с `lastMsg`; окончательный connection scope должен быть зафиксирован allowlist-ом владельца. Это не данные Ontology Core и не данные Tasker. Они требуют отдельного контура для текущего состояния, истории, геозапросов, аналитики и будущих прогнозов. + +Gelios является первым provider adapter, который подчиняется общему +`docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md`: connection instance хранит +connection scope/policy, Engine Credentials владеет provider secret, gateway +владеет storage/read-моделью, а ontology описывает значения, но не runtime +data. + +Физические trike-команды существуют в домене, но **не входят в ingestion или тестирование**. Для них позднее потребуется отдельный, ручной и аудируемый контур. + +## Решение + +1. Создать отдельный сервис `platform/services/gelios-gateway` как storage/read gateway: + - не хранит и не получает provider token; + - принимает только аутентифицированный safe-read intake от защищённого Engine L2 Collector workflow; + - применяет allowlist и field policy до записи; + - нормализует ответ в сущности пакета `gelios`; + - публикует read-модель для Engine L2 workflow и Map View. +2. В Engine появляется отдельный Gelios Credential и NDC Agent L2 collection profile: + - credential скрывает access/refresh pair и его lifecycle; + - credential привязан только к намеренно пошаренному Collector workflow; + - Collector содержит allowlist read-capabilities и не имеет command transport; + - ни секрет, ни provider response без нормализации не передаются в ontology, UI, Gateway или другие workflow. +3. Выделить сервису собственную БД `gelios-postgres`, не деля её с Tasker, Authentik, Notification Core или Ontology Core. В будущей multi-tenant форме эта БД обслуживает несколько Gelios connection instances, но все records разделены `tenant_id` и `connection_id`. +4. Базовый движок: PostgreSQL 16 с расширениями TimescaleDB и PostGIS. + - Timescale hypertable хранит временные ряды и автоматически делит их по времени. + - PostGIS хранит нормализованные точки и геометрии зон, а не renderer-объекты Cesium. + - Данный выбор покрывает транзакционную конфигурацию, realtime upsert, исторические запросы, SQL-аналитику и географию одним контуром. +5. **Не** использовать RabbitMQ Tasker как общую шину Gelios. Если измерения покажут, что прямой writer или число независимых потребителей не справляются, добавить в Gelios-контур NATS JetStream с durable pull consumers. Он даст replay, acknowledgement и контролируемое удержание сообщений. +6. Engine level-2 Collector — единственная точка provider access; остальные workflow являются потребителями read-модели. Ontology Core остаётся только словарём и контрактами. + +## Целевой поток + +```text +Gelios REST (safe read only) + -> NDC Agent L2 collection profile + protected credential + -> authenticated normalized intake + -> Gelios Gateway: scope -> field policy -> normalizer + -> gelios-postgres: current state + immutable telemetry history + -> [при необходимости] NATS JetStream + -> `fleet.positions.current.v1` read API / realtime subscription + -> NDC workflow level 2 + -> Map View semantic binding + -> Cesium renderer adapter +``` + +Ни один шаг не получает права отправить команду устройству. Красный command-domain находится вне этого потока; metadata каталога команд сохраняется, но send transport не создаётся. + +## Модель хранения v0 + +| Слой | Назначение | Минимальные записи | +| --- | --- | --- | +| Контроль | граница и повторяемость сбора | `access_scope`, `collection_run`, `ingestion_cursor`, endpoint/response metrics | +| Каталог | стабильные сущности и их конфигурация | `unit`, `unit_group`, `tracker_device`, sensor/maintenance/custom-field definitions | +| Current state | одна актуальная запись на unit для карты и интерфейса | `unit_current`, `position_fix`, approved operational status | +| History | неизменяемые события с временем наблюдения и получения | `telemetry_snapshot`, selective `sensor_reading`, restricted raw payload reference | +| Spatial | геометрии для запросов, не Cesium graphics | point/track/geozone with SRID 4326 | +| Analytics | роллапы и признаки, не запросы по всему raw | hourly/daily aggregates, feature sets, model runs | +| Audit | попытки, ошибки, политика, будущие команды | collection audit; separate command audit later | + +`unit_current` обновляется idempotently по `unitSubjectId`. История записывается append-only с ключом дедупликации, включающим provider unit id, observed time и fingerprint сообщения. Все временные таблицы имеют `observed_at` и `received_at`: задержка поставщика не должна переписывать фактическое время на карте. + +## Индексы и жизненный цикл + +- Основной путь истории: `(unit_subject_id, observed_at DESC)`. +- Пространственный индекс только для нормализованной geography/geometry; рендер-кэши в БД не храним. +- Сырые `params`/raw messages — restricted, отдельно от публичной Studio read-model. +- Политики retention, downsampling и резервного копирования должны быть утверждены до запуска history backfill. Они зависят от фактических msg/s, размера payload, нужной глубины истории, RPO/RTO и стоимости хранения. +- Для предиктива держать recent/raw слой и отдельные часовые/дневные агрегаты. Timescale continuous aggregates позволяют сохранять длительную агрегированную историю после сокращения raw при корректно согласованных refresh и retention политиках. + +## Нулевая итерация без лишней инфраструктуры + +1. Зафиксировать owner-approved allowlist и видимые поля. +2. Сделать только safe-read Engine Collector с ограничением по scope, rate limit, paging и cursor. +3. В течение согласованного окна измерить: сообщений/сек, размер ответа, lag, дубликаты, задержку записи и нагрузку запросов карты. +4. На фактах включить Timescale hypertables, PostGIS и retention policy; после этого решить, нужен ли JetStream сразу. +5. Подать только `fleet.positions.current.v1`/approved current position в Map View. Историю и raw не отдавать в renderer напрямую. + +## Что не решено этим ADR + +- окончательное правило connection scope; +- частота polling/возможность provider push; +- сроки хранения raw, нормализованной истории и агрегатов; +- RPO/RTO, репликация и production backup plan; +- допуск к ручному command gateway. До отдельного решения отправка команд запрещена. + +## Обоснование и источники + +- [Timescale hypertables](https://docs.timescale.com/use-timescale/latest/hypertables/) — временные таблицы PostgreSQL автоматически партиционируются по времени. +- [Timescale self-hosted installation](https://docs.timescale.com/self-hosted/latest/install/) — расширение разворачивается как self-hosted PostgreSQL-контур; production требует backup/PITR и HA-плана. +- [Retention и continuous aggregates](https://docs.timescale.com/use-timescale/latest/data-retention/data-retention-with-continuous-aggregates/) — raw и агрегаты требуют согласованных lifecycle-политик. +- [PostGIS](https://postgis.net/docs/en/) — PostgreSQL-расширение для spatial types и GiST R-tree индексов. +- [NATS JetStream consumers](https://docs.nats.io/nats-concepts/jetstream/consumers) — durable consumers дают acknowledgement, повторную доставку и recovery; рекомендуются pull consumers для новых масштабируемых обработчиков. diff --git a/docs/ADR_L2_OWNED_EXTERNAL_CONNECTORS.md b/docs/ADR_L2_OWNED_EXTERNAL_CONNECTORS.md new file mode 100644 index 0000000..289dc7b --- /dev/null +++ b/docs/ADR_L2_OWNED_EXTERNAL_CONNECTORS.md @@ -0,0 +1,274 @@ +# ADR: L2-owned external connectors and provider-neutral Data Plane + +Статус: **accepted**. +Дата: 2026-07-14. +Владелец решения: NODE.DC Platform. + +## Контекст + +NODE.DC — платформа разработки. Подключение внешнего API не должно создавать +ещё один provider-specific сервис, в котором навсегда зашиты логика клиента, +нормализация и правила отображения. Иначе каждый новый account или новый тип +объекта превращается в изменение Platform runtime. + +Нужна одна повторяемая форма: Platform даёт безопасные границы, контракт +хранения и выдачи данных; изолированный workflow NDC Agent L2 является +адаптером конкретного API и остаётся редактируемым через предоставленный +Codex/Engine MCP. + +Это решение заменяет provider-adapter часть +`ADR_EXTERNAL_PROVIDER_DATA_PLANE.md` и все provider-specific runtime +предписания `ADR_GELIOS_DATA_PLANE.md`. Они сохраняются как исторические +черновики, но не являются каноном для новой реализации. + +## Решение + +### 1. Адаптер внешнего API принадлежит L2 + +Один изолированный L2 workflow представляет одно connection instance и +является единственным местом для: + +- обращения к API поставщика через ссылку на credential из Engine; +- выбора read-capability, cursor/pagination, rate limit, retry и расписания; +- получения raw envelope, разбиения большого ответа на native batch-операции; +- семантического mapping к версии ontology и формирования canonical facts; +- idempotency key, watermark и lifecycle источника; +- deploy/run/execution/trace через Engine MCP. + +L2 не хранит значение provider secret или writer token: он использует только +Engine-bound credential references. Plaintext writer token выдаётся trusted +provisioner ровно один раз при create/rotate binding и затем хранится только в +Engine credential store, привязанном к этому L2; External Data Plane хранит +лишь его hash. Provisioner читает отдельный runner-owned secret file, который +read-only монтируется только в EDP и позднее в dedicated Engine provisioner, +работающий под выделенным UID/GID `11006`; это не shared `.env`, не +`NODEDC_INTERNAL_ACCESS_TOKEN` и не provider credential. До deployment этого +provisioner `EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED=false`, поэтому issuance +routes fail closed. Token не попадает в L2 graph/profile, contract artifact, +execution log/trace, raw envelope, Foundry или UI. Command/write-capabilities +не получают transport route в read workflow. + +Для новой учётной записи создаётся новый instance/profile этого workflow с +другой credential reference и connection configuration. Это не требует +изменения Platform source. Если поставщик раскрывает новый объект или поле, +сначала расширяется ontology/capability contract, затем изменяется именно +соответствующий L2 workflow. + +### 2. Platform Data Plane нейтрален к provider и домену + +Platform предоставляет один versioned **External Data Plane**. Он принимает +batch по внутреннему аутентифицированному контракту, хранит raw envelope с +retention/provenance, canonical facts, current/history projections и отдаёт +scoped data products/realtime updates. В нём запрещены: + +- ветвления по названию provider, customer, account, vehicle или renderer; +- provider field mapping, unit allowlist, business filtering и visual rules; +- provider token, прямой browser-to-provider доступ и command transport. + +Его canonical write-форма, которую Data Plane валидирует и сохраняет после +авторизации, содержит только нейтральные элементы: + +```text +source: providerId + tenantId + connectionId +contract: contractVersion + ontologyRevision + dataProductId +batch: runId + sequence + idempotencyKey + receivedAt +raw: optional restricted ref + contentType + hash + retention policy +facts: stable sourceId + semanticType + observedAt + attributes + geometry? +``` + +Для нового L2 writer-а это не wire-форма. Он вызывает +`POST /internal/data-plane/v1/data-products/:dataProductId/publish` с bearer +writer token и формой `nodedc.data-product.publish/v1`, в которой вообще нет +`source`, contract metadata, transport или persistence policy. Любой caller +scope и scope headers запрещены. + +External Data Plane разрешает token в собственный immutable writer binding +`(tenantId, connectionId, providerId, allowedDataProductIds, expiresAt)`, +проверяет active/non-expired binding, совпадение `providerId` и разрешение +`dataProductId`, затем сам materializes canonical `source` с tenant/connection +и только после этого валидирует и сохраняет batch. Присланный caller scope +отклоняется, а не merge/override-ится. Изменение tenant, connection, provider, +allowed data products или срока требует нового binding; для существующего +binding допустимы только token rotation и revoke. + +Выпуск, rotation и revoke binding принимаются только от dedicated provisioning +principal с runner-owned secret file; он не монтируется в shared `.env` или L2. +Shared legacy bearer не может создавать writer capability. + +Идентификатор источника стабилен в пределах `(providerId, connectionId, +sourceId)`. Отсутствие объекта в очередном ответе помечается lifecycle-state +или временем последнего наблюдения; запись и её история не удаляются. + +### 3. Граница данных и интерфейса + +```text +Provider API (safe read) + -> Engine provider-credential reference + -> L2 connector instance (adapter + mapping + batches) + -> Engine writer-credential reference + -> External Data Plane Data Product publish (scope materialized server-side) + -> External Data Plane (generic persistence + projections) + -> scoped data product / realtime stream + -> Foundry binding / user interface +``` + +Ontology Core описывает semantic types, связи, capability catalog и mapping +versions, но не хранит runtime payloads или секреты. Foundry получает только +approved data product и решает presentation: pins, visibility, layers and +filters. Оно не получает provider endpoint или credential. + +### 4. Полнота без entity allowlist + +Collection profile выбирает разрешённые **read-capabilities**, а не список +конкретных объектов. Если выбранный read endpoint возвращает все доступные +источнику сущности, L2 передаёт все валидные элементы. Новые сущности +добавляются автоматически; скрытие/отображение — задача data product/UI, а не +сбора. Технические лимиты существуют только как размер batch, backpressure, +quota и защита от повреждённого ответа, но не как бизнес-фильтр по ID. + +### 5. Масштабирование и native NDC Agent nodes + +L2 не запускает отдельный execution на каждую сущность. Он использует native +nodes NDC Agent: HTTP Request, Split Out, Split In Batches/Loop Over Items, +Aggregate, Postgres (когда нужен private workflow state), Set/If/Merge, +Schedule Trigger and Respond to Webhook. Code node допустим только как малый +преобразователь boundary-shape, когда эквивалентной native node нет; он не +становится скрытым сервисом или provider database. + +Shared telemetry/history не записывается L2 напрямую в физические таблицы +Platform Postgres: это создало бы coupling к schema. Direct Postgres допустим +для private state конкретного workflow или отдельной project-owned базы. +Общий контур использует только External Data Plane contract и bulk batches. + +### 5.1. Каталог custom nodes NODE.DC + +Встроенные узлы runtime сохраняют свои штатные названия (`HTTP Request`, +`Schedule Trigger`, `Loop Over Items` и т. п.). Любой узел, код которого +принадлежит NODE.DC, обязан одновременно выполнять три условия: + +- видимое имя начинается с `NDC `; +- runtime type принадлежит package namespace `n8n-nodes-ndc.*`; +- package-level test отклоняет публикацию узла без этого префикса и namespace. + +Первый канонический набор состоит из: + +- `NDC Data Product Publish` — runtime write boundary L2 → Data Plane; +- `NDC Data Product Read` — scoped current snapshot read по reader grant; +- `NDC Foundry Binding` — deploy/control-plane связь Data Product с + `Application → Page → typed slot`, но не транспорт каждого realtime tick. + +`NDC Foundry Output` не используется как техническое имя: оно ошибочно +подразумевает прямой transport L2 → renderer. Provider-specific adapter, +mapping и collection profile остаются логикой конкретного L2 workflow на +штатных nodes; они не оформляются как custom NODE.DC nodes и не добавляют +provider branch в Data Plane или Foundry. + +`NDC Foundry Binding` отправляет только +`nodedc.foundry.binding-upsert/v1`. Её постоянный L2 credential — отдельный +revocable `ndc_fndbg_*` workload grant с allowlist точных +`Application → Page → Binding → Slot → Data Product` targets. Короткоживущая +`fnd1.*` capability интерактивного Foundry MCP, browser session, EDP +writer/reader grant и `NODEDC_INTERNAL_ACCESS_TOKEN` для этой ноды запрещены. + +Runtime node не содержит URL, endpoint, provider ID, tenant ID, connection ID, +token, contract version или history cadence. Узел выбирает только разрешённый +Data Product; всё остальное materializes из opaque writer/reader binding и +immutable Data Product definition. + +Package `n8n-nodes-ndc` устанавливается как штатный private community package в +`/home/node/.n8n/nodes/node_modules`, чтобы n8n `PackageDirectoryLoader` +сохранил namespace `n8n-nodes-ndc.*`. `~/.n8n/custom` и +`N8N_CUSTOM_EXTENSIONS` запрещены для этого пакета: они создают namespace +`CUSTOM.*`. Ручное копирование или `npm install` в живом container, patch +Engine core и подмена built-in node запрещены. Production использует +проверенный offline tarball, immutable release, atomic current/previous switch, +read-only mount и restart всех n8n execution processes; acceptance завершается +только когда Engine schema и MCP catalog возвращают точные package-qualified +types. + +### 5.2. Realtime delivery contract + +Новый writer использует: + +```text +POST /internal/data-plane/v1/data-products/:dataProductId/publish +Authorization: Bearer +``` + +Body имеет schema `nodedc.data-product.publish/v1` и содержит только batch +identity и canonical facts. Data Plane одной транзакцией: + +1. проверяет idempotency; +2. обновляет current projection только более новым или изменившимся фактом; +3. применяет declarative history policy (`none`, `all`, `sampled`); +4. добавляет changed facts в durable patch outbox; +5. фиксирует монотонный cursor. + +Reader grant получает snapshot `nodedc.data-product.snapshot/v1`, затем +подключается к SSE stream с `after=` или `Last-Event-ID`. +Каждый outbox event имеет schema `nodedc.data-product.patch/v1`. Если cursor +уже удалён retention policy, stream отвечает `409 resync_required`, и consumer +повторяет snapshot. Browser не получает Data Plane token или scope: Foundry BFF +разрешает persisted binding и читает server-only reader grant. + +`snapshot+patch` в v1 является **bounded product contract**. Current projection +одного scoped Data Product не может содержать больше 5000 entity keys +`(sourceId, semanticType)`. Snapshot возвращает целую согласованную projection +и один barrier cursor из одной `REPEATABLE READ` transaction. Query-параметр +`limit` — только защитный потолок, а не размер страницы: если полная projection +не помещается, Data Plane отвечает `413 data_product_snapshot_limit_exceeded` +и consumer не начинает patch stream с неполной базой. + +Наивная pagination current snapshot по `sourceId`, offset или независимо +полученным page cursors запрещена: изменения между страницами могут быть +пропущены или продублированы относительно patch cursor. Product с ожидаемой +cardinality выше 5000 обязан **до включения realtime** выбрать один из двух +отдельно версионируемых контрактов: + +- stable partitioning в несколько Data Products, где partition key является + частью definition, а каждый partition имеет собственный полный snapshot и + независимый patch cursor; +- новый query delivery contract с явно определёнными snapshot barrier, + continuation cursor, query scope и правилами перехода к patch stream. + +Такой query contract не входит в `nodedc.data-product.snapshot/v1` и не может +быть имитирован полем `nextPageCursor`. До его отдельного утверждения runtime и +`NDC Data Product Read` работают только с bounded products до 5000 сущностей. + +### 6. Порядок первой реализации + +1. Версионировать нейтральный intake/read contract и поднять External Data + Plane без provider-specific logic. +2. Пересобрать существующий L2 proof в native-node pipeline: safe read → + split/batch → semantic mapping → generic append → safe summary. Никаких + provider ID filters и provider gateway endpoint. +3. Выполнить manual run, проверить execution/trace, idempotency and current + projection; только затем включить Schedule Trigger. +4. Provision immutable writer binding, one-time place its token into an opaque + Engine credential, then switch the L2 to + `/data-products/:dataProductId/publish` and remove caller scope/headers and + the shared legacy credential. +5. Подключить Foundry к data product current positions. История, геозоны и + новые сущности добавляются отдельными L2 collection profiles and ontology + revisions. + +## Последствия + +- `services/-gateway` не является шаблоном для новых интеграций. + Существующий experimental code не расширяется и не деплоится как часть этого + решения. +- Data Plane может иметь собственную БД/Timescale/PostGIS implementation, но + физическая схема остаётся внутренней деталью neutral service, а не API для + L2 или Foundry. +- Inline raw запрещён в L2 → Data Plane v1: до отдельного raw-vault допускается + только restricted ref/hash либо отсутствие raw envelope. Retention reference + вычисляется по серверному acceptance time, а не по timestamp из batch; Data + Plane выполняет bounded retention sweeps после готовности сервиса и по таймеру. +- Обычный Codex Desktop получает ровно те L2 grants, которые выданы владельцем + connection/workflow, и может безопасно развивать adapter только в этой + границе. +- `POST /internal/data-plane/v1/intake` остаётся временным migration-only route: + он выключен по умолчанию (`EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED=false`), + требует `NODEDC_INTERNAL_ACCESS_TOKEN`, canonical scoped batch и совпадающие + scope headers. Новый или migrated L2 не получает этот shared token и не + использует fallback в legacy route. После миграции всех writers он удаляется. diff --git a/packages/external-provider-contract/README.md b/packages/external-provider-contract/README.md new file mode 100644 index 0000000..ea98ced --- /dev/null +++ b/packages/external-provider-contract/README.md @@ -0,0 +1,302 @@ +# External Provider Contract + +Этот package — единственное общее место для формы внешних интеграций NODE.DC. +Он не содержит provider secrets, customer records, runtime payloads или +исполняемый connector code. `src/index.mjs` даёт dependency-free проверку +минимальных v1 contracts; она запускается через `npm run check`. + +Каждый L2 connector instance должен поставлять совместимые versioned +артефакты: + +```text +provider-manifest +connection-profile +capability-catalog +field-catalog +collection-profile +retention-profile +semantic-mapping-contract +read-model-contract +command-catalog (metadata only until a red command gateway is approved) +``` + +`provider-manifest` — декларативный template-артефакт: provider ID, ontology +revision, L2 template version, capability catalog и data-product IDs. Он не +является tenant connection, не содержит endpoint/URL, credential reference, +secret, ручной scope или executable provider code. Concrete non-secret +connection profile принадлежит и версионируется вместе с конкретным L2 +workflow; Engine только привязывает к нему opaque credential reference и grant. + +## Проверяемые v1 contracts + +- `Connection` — provider instance, tenant scope и *ссылка* на credential в + Engine. Любые token/secret/password-like поля запрещены. +- `Collection Profile` — явная policy сбора. `manual` не может скрыто содержать + polling interval; `realtime` требует interval не чаще одного раза в секунду. +- `Data Product` — нормализованный versioned output с semantic types, полями и + внутренней аудиторией. +- `Intake Batch` — canonical **scoped** record, который External Data Plane + валидирует и сохраняет: source, contract revision, idempotency, restricted + raw envelope и canonical facts. Его `source` содержит `providerId`, + `tenantId` и `connectionId`. Writer-bound L2 request намеренно не является + готовым `Intake Batch`: Data Plane сначала materializes scope и лишь затем + применяет этот contract. В canonical record нет provider field mapping, + entity allowlist, token или renderer data. Inline `raw.payload` в v1 + запрещён: если нужна provenance-ссылка, connector передаёт restricted + `raw.ref` вместе с hash. Отдельный raw-vault может быть добавлен только + отдельным ADR и не становится частью L2 → Data Plane wire boundary. +- `NDC Foundry Binding` — адресует data product только в конкретную цепочку + `Foundry Application → Page → approved slot`; `templateId` можно сохранить + как дополнительную типизацию, но он не заменяет `applicationId` и `pageId`. + В binding запрещены provider transport, endpoint и credential reference. +- `Provider Manifest` — статическое описание L2 connector template, capability + catalog и ontology/data-product contracts. Оно не может содержать tenant, + connection, credential, secret или provider transport. + +## Data Product delivery contracts + +Provider-neutral runtime использует отдельные wire schemas: + +- `nodedc.data-product.publish/v1` — unscoped publish request от + `NDC Data Product Publish`; содержит только batch identity и canonical facts; +- `nodedc.data-product.snapshot/v1` — согласованный current snapshot с cursor; +- `nodedc.data-product.patch/v1` — committed upsert operations из durable + outbox с `previousCursor`/`cursor`. + +Publish request не может задавать provider, tenant, connection, endpoint, +receivedAt, ontology revision, version или persistence policy. Data Plane +materializes эти значения из opaque writer binding и зарегистрированного Data +Product definition. Snapshot/stream читаются только по отдельному opaque reader +binding; shared internal bearer и caller-provided scope headers являются legacy +и не используются новыми nodes/Foundry runtime. + +`snapshot+patch` v1 — bounded contract: один scoped Data Product содержит не +более 5000 current entity keys `(sourceId, semanticType)`. Snapshot обязан быть +полным и привязанным к одному repeatable-read cursor. Параметр `limit` задаёт +защитный ceiling, не page size: превышение возвращает +`413 data_product_snapshot_limit_exceeded`, а reader не имеет права начинать +stream с усечённой базой. `nextPageCursor` зарезервирован для будущего отдельно +версионируемого query contract и текущим bounded runtime не выдаётся. + +Для большей cardinality definition заранее раскладывается по стабильным +partition Data Products с независимыми snapshot/patch cursors либо использует +будущий query contract с единым snapshot barrier и continuation semantics. +Offset/source-ID pagination поверх меняющегося current snapshot запрещена: +между страницами она способна потерять или задублировать изменения относительно +patch cursor. + +Для каждого canonical fact действует одинаковый hard ceiling: сериализованный +`attributes` не больше 64 KiB как на publish/intake входе, так и в +snapshot/patch выходе. GeoJSON `Point` принимает longitude только в +`[-180, 180]`, latitude в `[-90, 90]`; batch sequence ограничен диапазоном +PostgreSQL `integer` `0..2147483647`. Эти ограничения нельзя ослабить опциями +конкретного caller-а. + +Manifest, connection/collection profiles, data-product definition и Foundry +binding используют fail-closed allowed-key schemas. Неизвестные поля, а также +secret-like имена или значения (включая `ndc_edpwb_`/`ndc_edprb_`) отклоняются +на общей границе. + +Все private custom nodes NODE.DC поставляются package +`platform/packages/n8n-nodes-ndc`. Их display name обязан начинаться с `NDC `, +а runtime type — с `n8n-nodes-ndc.`; package называется строго +`n8n-nodes-ndc`. Provider-specific adapters остаются L2 workflow logic и не +становятся custom nodes или ветками Data Plane. Эти инварианты проверяются +package test. + +Control-plane команда `NDC Foundry Binding` имеет отдельную replay-safe schema +`nodedc.foundry.binding-upsert/v1` (`validateFoundryBindingUpsert`). Это не +declarative provider artifact и не runtime transport: команда содержит только +application/page/binding/data-product projection и idempotency key, а право на +операцию извлекается Foundry из отдельного opaque workload grant. + +## Engine opaque credential sink + +`src/engine-credential-sink.mjs` задаёт dependency-free server-to-server v1 +границу для доставки трёх workload capabilities в Engine Credentials: + +- `external-data-plane.writer` → точная нода + `n8n-nodes-ndc.ndcDataProductPublish` / `ndcDataProductWriterApi`; +- `external-data-plane.reader` → точная нода + `n8n-nodes-ndc.ndcDataProductRead` / `ndcDataProductReaderApi`; +- `foundry.binding` → точная нода + `n8n-nodes-ndc.ndcFoundryBinding` / `ndcFoundryBindingApi`. + +Provision request фиксирует `workflowId`, `workflowRevision`, `nodeId`, runtime +node type, credential type, grant ID, expiry и issuer policy hash. Aggregate +`transaction.policyHash` детерминированно считается по всему secret-free +descriptor; подмена любой цели или policy ломает валидацию. Единственное поле, +которое переносит plaintext capability, — `bindings[].material.value`; request +нельзя писать в логи, traces, очередь или audit. + +Provision и rollback envelopes действуют не более 15 минут: sink отклоняет +истёкшие запросы и допускает максимум 60 секунд положительного clock skew. +Receipt повторно проверяет freshness относительно собственного `processedAt`, +чтобы старый запрос нельзя было применить или подтвердить через replay. + +Каждый binding содержит `capabilityDigest = sha256(material.value)`: Engine +самостоятельно хеширует полученный plaintext и сравнивает digest. Aggregate +`policyHash` включает этот digest и issuer identity, а provision transaction +несёт обязательную Ed25519 attestation. Engine принимает её только по +allowlisted `serviceId:keyId`; заменить capability и пересчитать обычный hash +без приватного issuer key невозможно. + +Sink обязан выполнять `rollback-all`: сначала проверить весь request и точное +состояние graph, затем создать credentials в staging, атомарно привязать весь +набор и только после commit вернуть opaque `credentialRef`. При любой ошибке +новые credentials удаляются, а прежние bindings остаются без изменений. +`rollback-failed` означает карантин и ручное восстановление, но никогда не +возвращает частичные credential refs. + +Receipt, rollback request/receipt и audit имеют отдельные strict schemas. Они +не способны вернуть capability material; audit хранит только hash opaque +credential reference. Explicit rollback адресует committed transaction через +`transactionId`, `policyHash` и hash committed receipt, поэтому не может +случайно откатить другой набор. Реализация sink принадлежит Engine и не даёт +Platform/Codex доступа к Engine core, runtime files или plaintext credential +storage. + +## Engine private-extension management + +`src/engine-private-extension.mjs` задаёт строгую Platform-side границу для +Engine-owned активации проверенного `n8n-nodes-ndc` release. Контракт не +устанавливает package и не меняет Engine: он фиксирует async +`plan -> apply receipt -> operation/status` protocol, где `apply` обязан +быстро вернуть `queued` и `operationId`, а долгий recreate/acceptance +отслеживается отдельно. + +Активация и rollback требуют отдельной глобальной capability +`engine.private-extension.manage`. Обычные L1/L2 grants её не дают. Чтение +состояния допускает `engine.private-extension.read` или manage-capability. +Запрос выбирает только allowlisted package, digest-bound `releaseId` и +`packageSha256`; caller не передаёт host path, package bytes, Compose service, +shell command или credential material. Plan живёт не более 15 минут, является +single-use и применяется только с тем же idempotency key и plan hash. + +Runtime transition для n8n 2.3.2 зафиксирован как community-package loader по +`/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc`, а не как +`N8N_CUSTOM_EXTENSIONS` или `CUSTOM.*` loader: + +- `N8N_COMMUNITY_PACKAGES_ENABLED=true`, + `N8N_COMMUNITY_PACKAGES_PREVENT_LOADING=false`, + `N8N_REINSTALL_MISSING_PACKAGES=false`; +- Deploy/Run quiesced and execution queue drained before the version switch; +- read-only mount and atomic current/recovery state; +- force-recreate main, every worker and every webhook instance as one version + barrier; hot reload запрещён; +- acceptance требует единый generation и точный набор трёх package-qualified + node schemas и трёх credential schemas. + +Любая ошибка после switch запускает automatic rollback и повторный +force-recreate/acceptance. Ошибка самого rollback переводит runtime в +`quarantined`. Immutable release и существующие Engine Credentials +сохраняются. Для первой активации предыдущим проверенным состоянием является +`n8n-nodes-ndc.inactive/v1`: rollback в этот baseline удаляет package из +loader surface, но не удаляет credentials. + +Public Ops gateway уже умеет прозрачно передавать эти операции через +`/engine/mcp`, если Engine реализует соответствующие MCP tools. Так как +gateway имеет 30-second upstream timeout, side effect остаётся асинхронным; +отдельный public REST proxy для management boundary не требуется. + +Пример `examples/gelios-positions-current.v1.mjs` — provider-specific fixture +без customer, tenant identity или credential material. + +## Ownership + +- `platform/packages/external-provider-contract` — общий контракт и schemas. +- NDC Agent L2 — provider API adapter: fetch, pagination, batching, + semantic mapping, collection profile и ссылка на credential в Engine. +- Platform External Data Plane — provider-neutral intake, raw retention, + canonical facts, current/history projections и scoped read products. Он не + знает provider fields, customer/business filters или renderer rules. +- `platform/services/ontology-core/catalog/domain-packages/` — + семантика, отношения и guardrails, но не runtime data. +- Foundry/interface bindings — consumers scoped data product; они не получают + provider transport, endpoint или credential reference. + +`services/-gateway` — устаревший experimental path и не является +шаблоном новых integration services. Канон описан в +`docs/ADR_L2_OWNED_EXTERNAL_CONNECTORS.md`. + +## Mandatory connection boundary + +`connection` принадлежит одному tenant/client context и содержит только +ссылку на секрет, утверждённый capability scope, collection profile, field +policy и retention policy. Во всех runtime records Data Plane сохраняет минимум +`tenant_id`, `connection_id`, `provider_id`, `observed_at`, `received_at` и +provenance/version там, где это применимо. + +Writer token — отдельный EDP runtime credential, а не поле `Connection`, +profile, provider manifest или L2 graph. + +## Scoped writer binding + +Writer binding — EDP-owned runtime security state, а не versioned artifact +provider manifest или connection profile. Он фиксирует `tenantId`, +`connectionId`, `providerId`, `allowedDataProductIds`, active/revoked state и +`expiresAt`. L2 не может редактировать binding или задавать его scope; +изменение любого scope-поля либо TTL создаёт новый binding, а прежний binding +можно только rotate/revoke. + +Новый L2 вызывает `POST /internal/data-plane/v1/intake/writer-bound` с +`Authorization: Bearer ` и unscoped envelope. В `source` +разрешён только `providerId`; `tenantId`, `connectionId` и +`x-nodedc-*-id` headers запрещены. EDP проверяет token, binding, provider и +`contract.dataProductId`, затем materializes immutable canonical scope и +сохраняет обычный `Intake Batch`. Caller-provided scope отклоняется, а не +доверяется и не объединяется с binding. + +Plaintext writer token возвращается только trusted provisioner при создании +или rotation. Provisioner помещает его непосредственно в opaque Engine +credential, доступный назначенному L2. EDP хранит только hash token и binding +metadata; token запрещён в connection/profile/manifest, L2 graph, logs/traces, +raw payload, Foundry и UI. + +Provisioning routes принимают только отдельный secret file, созданный +root-owned deploy runner: +`/volume1/docker/nodedc-platform/secrets/external-data-plane-provisioner/token`. +Он read-only монтируется только в EDP и позднее — в dedicated Engine +provisioner, работающий под выделенным UID/GID `11006`; это не `.env` value, +не `NODEDC_INTERNAL_ACCESS_TOKEN` и не provider credential. Пока generic Engine +provisioner не развёрнут, `EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED=false`, а +create/rotate/revoke routes отвечают `503` и не делают fallback к shared +internal bearer. + +## Legacy intake migration + +`POST /internal/data-plane/v1/intake` — временный compatibility route для +existing writers. Он принимает только canonical scoped `Intake Batch` под +`NODEDC_INTERNAL_ACCESS_TOKEN` и требует совпадения body scope с +`x-nodedc-tenant-id` и `x-nodedc-connection-id`. Writer token этот route не +заменяет. + +Для миграции: создать writer binding, один раз записать выданный token в Engine +credential, переключить L2 на `/intake/writer-bound`, удалить tenant/connection +из body и headers, проверить успешный intake, затем удалить у L2 legacy +credential. Новый или migrated writer не должен fallback-иться на legacy route +после ошибки writer-bound intake. Когда migrated все writers, legacy route и +его shared-token access удаляются. + +## Intake boundary + +L2 отправляет в общий Data Plane `Intake Batch`, а не SQL-запрос в общие +таблицы. Platform проверяет boundary и сохраняет raw/history/current +projections; semantic mapping остаётся в L2. Список конкретных source IDs не +является частью connection scope: scope выбирает read-capability API, а +visibility решает data-product consumer. + +Inline `raw.payload` запрещён: L2 не может записать в Data Plane полный ответ +provider-а или произвольную строку. Пока отдельный raw-vault не утверждён, +batch либо не содержит `raw`, либо содержит только restricted `raw.ref` и hash. +Secret-like keys и распознаваемые bearer/JWT/writer-token values в любом участке +canonical batch, включая `facts[].attributes`, отклоняются. Retention raw +reference вычисляется от server acceptance time, а не от `batch.receivedAt`; +service делает sweep expired envelopes при старте и по расписанию. + +## Safety classification + +Capabilities классифицируются как `read`, `metadata`, `write`, `destructive` +или `unknown`. Только `read` и согласованные `metadata` могут попасть в +collector. `write` и `destructive` остаются каталогизированными, но не имеют +transport route в read adapter. diff --git a/packages/external-provider-contract/examples/gelios-positions-current.v1.mjs b/packages/external-provider-contract/examples/gelios-positions-current.v1.mjs new file mode 100644 index 0000000..9828dec --- /dev/null +++ b/packages/external-provider-contract/examples/gelios-positions-current.v1.mjs @@ -0,0 +1,76 @@ +import { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "../src/index.mjs"; + +export const geliosPositionsCurrentExample = Object.freeze({ + providerManifest: { + schemaVersion: EXTERNAL_PROVIDER_CONTRACT_VERSION, + id: "gelios.positions.current", + providerId: "gelios", + version: "1.0.0", + ontology: { + packageId: "gelios", + revision: "gelios.positions.v1", + }, + l2Template: { + id: "gelios.positions.current", + version: "1.0.0", + }, + capabilities: [ + { id: "gelios.units.current.read", classification: "read" }, + { id: "gelios.units.command.write", classification: "write" }, + ], + dataProductIds: ["fleet.positions.current.v1"], + }, + connection: { + schemaVersion: EXTERNAL_PROVIDER_CONTRACT_VERSION, + id: "gelios.sample-fleet", + providerId: "gelios", + tenantId: "sample-tenant", + credentialRef: { owner: "engine", reference: "engine-credential-reference" }, + scope: { + capabilityIds: ["gelios.units.current.read"], + fieldPolicyId: "fleet.position.display.v1", + }, + }, + collectionProfile: { + schemaVersion: EXTERNAL_PROVIDER_CONTRACT_VERSION, + id: "gelios.positions.realtime", + connectionId: "gelios.sample-fleet", + mode: "realtime", + schedule: { intervalMs: 15000 }, + capabilityIds: ["gelios.units.current.read"], + dataProductId: "fleet.positions.current.v1", + }, + dataProduct: { + schemaVersion: EXTERNAL_PROVIDER_CONTRACT_VERSION, + id: "fleet.positions.current.v1", + version: "1.0.0", + delivery: { mode: "snapshot+patch" }, + semanticTypes: ["map.moving_object"], + fields: [ + "course_degrees", + "display_name", + "elevation_meters", + "geometry", + "hdop", + "horizontal_accuracy_meters", + "object_kind", + "operational_status", + "position_source", + "position_valid", + "quality_flags", + "satellite_count", + "speed_kph", + ], + access: { audience: "internal" }, + }, + foundryBinding: { + schemaVersion: EXTERNAL_PROVIDER_CONTRACT_VERSION, + id: "fleet.operations-map.moving-objects", + dataProductId: "fleet.positions.current.v1", + applicationId: "11111111-1111-4111-8111-111111111111", + pageId: "map", + templateId: "map", + slotId: "points", + semanticType: "map.moving_object", + }, +}); diff --git a/packages/external-provider-contract/package.json b/packages/external-provider-contract/package.json new file mode 100644 index 0000000..0493ee7 --- /dev/null +++ b/packages/external-provider-contract/package.json @@ -0,0 +1,10 @@ +{ + "name": "@nodedc/external-provider-contract", + "version": "0.1.0", + "private": true, + "type": "module", + "exports": "./src/index.mjs", + "scripts": { + "check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/engine-credential-sink.test.mjs && node test/engine-private-extension.test.mjs" + } +} diff --git a/packages/external-provider-contract/src/data-product.mjs b/packages/external-provider-contract/src/data-product.mjs new file mode 100644 index 0000000..e9bfd05 --- /dev/null +++ b/packages/external-provider-contract/src/data-product.mjs @@ -0,0 +1,197 @@ +export const DATA_PRODUCT_PUBLISH_SCHEMA_VERSION = "nodedc.data-product.publish/v1"; +export const DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION = "nodedc.data-product.snapshot/v1"; +export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1"; + +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; +const CURSOR = /^(?:0|[1-9]\d*)$/; +const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i; +const SECRET_LIKE_VALUE = /(?:ndc_edp(?:wb|rb)_[A-Za-z0-9_-]*|[?&](?:token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)=|(?:bearer|basic)\s+\S+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/i; +const MAX_BATCH_SEQUENCE = 2_147_483_647; +const MAX_FACT_ATTRIBUTES_BYTES = 64 * 1024; + +/** + * Wire form accepted from an NDC Data Product Publish node. + * + * Scope, provider identity, product version, ontology revision and storage + * policy are deliberately absent: the Data Plane materializes them from the + * opaque writer grant and its product registry. + */ +export function validateDataProductPublish(value, { maxFacts = 5000, maxAttributesBytes = 64 * 1024 } = {}) { + const errors = []; + const attributesCeiling = Number.isInteger(maxAttributesBytes) && maxAttributesBytes > 0 + ? Math.min(maxAttributesBytes, MAX_FACT_ATTRIBUTES_BYTES) + : MAX_FACT_ATTRIBUTES_BYTES; + if (!isPlainObject(value)) return result(["publish_must_be_object"]); + if (value.schemaVersion !== DATA_PRODUCT_PUBLISH_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, new Set(["schemaVersion", "batch", "facts"]), "publish", errors); + + if (!isPlainObject(value.batch)) { + errors.push("batch_must_be_object"); + } else { + rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey"]), "batch", errors); + requiredIdentifier(value.batch.runId, "batch.runId", errors); + requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors); + if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) { + errors.push("batch.sequence_must_be_integer_0_to_2147483647"); + } + } + + if (!Array.isArray(value.facts) || value.facts.length === 0) { + errors.push("facts_must_be_nonempty_array"); + } else if (value.facts.length > maxFacts) { + errors.push("facts_limit_exceeded"); + } else { + const entityKeys = new Set(); + value.facts.forEach((fact, index) => { + validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling }); + if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return; + const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`; + if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key"); + entityKeys.add(entityKey); + }); + } + + if (containsSecretLikeMaterial(value)) errors.push("publish_must_not_contain_secret_material"); + return result(errors); +} + +export function validateDataProductSnapshot(value) { + const errors = envelopeErrors(value, DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, "snapshot"); + rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "cursor", "facts", "nextPageCursor"]), "snapshot", errors); + requiredCursor(value?.cursor, "cursor", errors); + requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors); + if (!Array.isArray(value?.facts)) { + errors.push("facts_must_be_array"); + } else { + value.facts.forEach((fact, index) => validateCanonicalFact(fact, `facts[${index}]`, errors)); + } + if (value?.nextPageCursor !== undefined) requiredString(value.nextPageCursor, "nextPageCursor", errors); + if (containsSecretLikeMaterial(value)) errors.push("snapshot_must_not_contain_secret_material"); + return result(errors); +} + +export function validateDataProductPatch(value) { + const errors = envelopeErrors(value, DATA_PRODUCT_PATCH_SCHEMA_VERSION, "patch"); + rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "cursor", "previousCursor", "emittedAt", "operations"]), "patch", errors); + requiredCursor(value?.cursor, "cursor", errors); + requiredCursor(value?.previousCursor, "previousCursor", errors); + requiredIsoTimestamp(value?.emittedAt, "emittedAt", errors); + if (!Array.isArray(value?.operations) || value.operations.length === 0) { + errors.push("operations_must_be_nonempty_array"); + } else { + value.operations.forEach((operation, index) => { + if (!isPlainObject(operation) || operation.op !== "upsert") { + errors.push(`operations[${index}].op_must_be_upsert`); + return; + } + rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors); + validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors); + }); + } + if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material"); + return result(errors); +} + +function envelopeErrors(value, schemaVersion, label) { + const errors = []; + if (!isPlainObject(value)) return [`${label}_must_be_object`]; + if (value.schemaVersion !== schemaVersion) errors.push("schemaVersion_mismatch"); + if (!isPlainObject(value.dataProduct)) { + errors.push("dataProduct_must_be_object"); + } else { + rejectUnknownKeys(value.dataProduct, new Set(["id", "version"]), "dataProduct", errors); + requiredIdentifier(value.dataProduct.id, "dataProduct.id", errors); + requiredString(value.dataProduct.version, "dataProduct.version", errors); + if (value.dataProduct.version && !SEMVER.test(value.dataProduct.version)) errors.push("dataProduct.version_must_be_semver"); + } + return errors; +} + +function validateFact(value, path, errors, { maxAttributesBytes, canonical = false }) { + if (!isPlainObject(value)) { + errors.push(`${path}_must_be_object`); + return; + } + const allowedKeys = new Set(["sourceId", "semanticType", "observedAt", "attributes", "geometry"]); + if (canonical) allowedKeys.add("receivedAt"); + rejectUnknownKeys(value, allowedKeys, path, errors); + requiredIdentifier(value.sourceId, `${path}.sourceId`, errors); + requiredIdentifier(value.semanticType, `${path}.semanticType`, errors); + requiredIsoTimestamp(value.observedAt, `${path}.observedAt`, errors); + if (value.attributes !== undefined) { + if (!isPlainObject(value.attributes)) { + errors.push(`${path}.attributes_must_be_object`); + } else if (serializedByteLength(value.attributes) > maxAttributesBytes) { + errors.push(`${path}.attributes_size_exceeded`); + } + } + if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors); +} + +function validateCanonicalFact(value, path, errors) { + validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true }); + if (!isPlainObject(value)) return; + requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors); +} + +function validatePointGeometry(value, path, errors) { + if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) { + errors.push(`${path}_must_be_geojson_point`); + return; + } + rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors); + if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) { + errors.push(`${path}_coordinates_must_be_finite_numbers`); + return; + } + const [longitude, latitude] = value.coordinates; + if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`); + if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`); +} + +function rejectUnknownKeys(value, allowed, path, errors) { + if (!isPlainObject(value)) return; + for (const key of Object.keys(value)) { + if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`); + } +} + +function requiredIdentifier(value, path, errors) { + if (typeof value !== "string" || !IDENTIFIER.test(value)) errors.push(`${path}_invalid`); +} + +function requiredString(value, path, errors) { + if (typeof value !== "string" || !value.trim()) errors.push(`${path}_required`); +} + +function requiredCursor(value, path, errors) { + if (typeof value !== "string" || !CURSOR.test(value)) errors.push(`${path}_invalid`); +} + +function requiredIsoTimestamp(value, path, errors) { + if (typeof value !== "string" || Number.isNaN(Date.parse(value))) errors.push(`${path}_invalid_timestamp`); +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function serializedByteLength(value) { + try { + return Buffer.byteLength(JSON.stringify(value)); + } catch { + return Number.POSITIVE_INFINITY; + } +} + +function containsSecretLikeMaterial(value) { + if (typeof value === "string") return SECRET_LIKE_VALUE.test(value); + if (Array.isArray(value)) return value.some(containsSecretLikeMaterial); + if (!isPlainObject(value)) return false; + return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeMaterial(child)); +} + +function result(errors) { + return Object.freeze({ ok: errors.length === 0, errors: Object.freeze([...new Set(errors)]) }); +} diff --git a/packages/external-provider-contract/src/engine-credential-sink.mjs b/packages/external-provider-contract/src/engine-credential-sink.mjs new file mode 100644 index 0000000..c924b9a --- /dev/null +++ b/packages/external-provider-contract/src/engine-credential-sink.mjs @@ -0,0 +1,662 @@ +import { createHash, verify as verifySignature } from "node:crypto"; + +export const ENGINE_CREDENTIAL_SINK_PROVISION_SCHEMA_VERSION = "nodedc.engine.credential-sink.provision/v1"; +export const ENGINE_CREDENTIAL_SINK_RECEIPT_SCHEMA_VERSION = "nodedc.engine.credential-sink.receipt/v1"; +export const ENGINE_CREDENTIAL_SINK_ROLLBACK_SCHEMA_VERSION = "nodedc.engine.credential-sink.rollback/v1"; +export const ENGINE_CREDENTIAL_SINK_ROLLBACK_RECEIPT_SCHEMA_VERSION = "nodedc.engine.credential-sink.rollback-receipt/v1"; +export const ENGINE_CREDENTIAL_SINK_AUDIT_SCHEMA_VERSION = "nodedc.engine.credential-sink.audit/v1"; + +const HASH = /^sha256:[a-f0-9]{64}$/; +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const OPAQUE_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{2,159}$/; +const NODE_TYPE = /^[a-z][A-Za-z0-9.-]{2,159}$/; +const CREDENTIAL_TYPE = /^[a-z][A-Za-z0-9]{2,127}$/; +const CREDENTIAL_REFERENCE = /^[A-Za-z0-9][A-Za-z0-9_-]{5,159}$/; +const REASON_CODE = /^[a-z][a-z0-9_.:-]{2,127}$/; +const ED25519_SIGNATURE = /^[A-Za-z0-9_-]{86}$/; +const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key|material|value)/i; +const SECRET_LIKE_VALUE = /(?:ndc_(?:edp(?:wb|rb)|fndbg)_[A-Za-z0-9_-]+|(?:bearer|basic)\s+\S+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/i; +const MAX_BINDINGS = 32; +const MAX_REQUEST_LIFETIME_MS = 15 * 60 * 1000; +const MAX_REQUEST_CLOCK_SKEW_MS = 60 * 1000; + +const CAPABILITY_SPECS = Object.freeze({ + "external-data-plane.writer": Object.freeze({ + nodeType: "n8n-nodes-ndc.ndcDataProductPublish", + credentialType: "ndcDataProductWriterApi", + materialPattern: /^ndc_edpwb_[A-Za-z0-9_-]{43}$/, + }), + "external-data-plane.reader": Object.freeze({ + nodeType: "n8n-nodes-ndc.ndcDataProductRead", + credentialType: "ndcDataProductReaderApi", + materialPattern: /^ndc_edprb_[A-Za-z0-9_-]{43}$/, + }), + "foundry.binding": Object.freeze({ + nodeType: "n8n-nodes-ndc.ndcFoundryBinding", + credentialType: "ndcFoundryBindingApi", + materialPattern: /^ndc_fndbg_[A-Za-z0-9_-]{43}$/, + }), +}); + +export const ENGINE_CREDENTIAL_SINK_CAPABILITY_TYPES = Object.freeze(Object.keys(CAPABILITY_SPECS)); + +const PROVISION_KEYS = new Set(["schemaVersion", "transaction", "bindings"]); +const TRANSACTION_KEYS = new Set([ + "id", + "idempotencyKey", + "requestedAt", + "requestExpiresAt", + "policyHash", + "failureMode", + "issuer", + "attestation", +]); +const ISSUER_KEYS = new Set(["serviceId", "keyId"]); +const ATTESTATION_KEYS = new Set(["algorithm", "signature"]); +const BINDING_KEYS = new Set([ + "bindingId", + "capabilityType", + "grantId", + "target", + "expiresAt", + "policyHash", + "capabilityDigest", + "material", +]); +const TARGET_KEYS = new Set([ + "workflowId", + "workflowRevision", + "nodeId", + "nodeType", + "credentialType", +]); +const MATERIAL_KEYS = new Set(["format", "value"]); +const RECEIPT_KEYS = new Set([ + "schemaVersion", + "transactionId", + "idempotencyKey", + "outcome", + "policyHash", + "processedAt", + "credentials", + "rollback", + "errorCode", +]); +const RECEIPT_CREDENTIAL_KEYS = new Set([ + "bindingId", + "capabilityType", + "grantId", + "target", + "credentialRef", + "expiresAt", + "policyHash", + "capabilityDigest", + "disposition", +]); +const RECEIPT_ROLLBACK_KEYS = new Set(["status", "completedAt"]); +const ROLLBACK_REQUEST_KEYS = new Set(["schemaVersion", "rollback"]); +const ROLLBACK_KEYS = new Set([ + "id", + "idempotencyKey", + "transactionId", + "requestedAt", + "requestExpiresAt", + "policyHash", + "committedReceiptHash", + "reasonCode", +]); +const ROLLBACK_RECEIPT_KEYS = new Set([ + "schemaVersion", + "rollbackId", + "transactionId", + "outcome", + "policyHash", + "committedReceiptHash", + "processedAt", + "errorCode", +]); +const AUDIT_KEYS = new Set([ + "schemaVersion", + "eventId", + "transactionId", + "operationId", + "operation", + "outcome", + "occurredAt", + "policyHash", + "principal", + "targets", + "reasonCode", +]); +const PRINCIPAL_KEYS = new Set(["serviceId", "fingerprint"]); +const AUDIT_TARGET_KEYS = new Set([ + "bindingId", + "capabilityType", + "grantId", + "target", + "expiresAt", + "policyHash", + "capabilityDigest", + "credentialRefHash", +]); + +/** + * Validates the only request allowed to carry plaintext workload capability + * material across the trusted Platform -> Engine server boundary. Callers and + * receivers must never log, trace, persist or return this request body. + */ +export function validateEngineCredentialSinkProvision(value, { now = Date.now(), issuerPublicKeys } = {}) { + const errors = []; + const nowMs = normalizeNow(now, errors); + if (!isPlainObject(value)) return result(["credentialSinkProvision_must_be_object"]); + if (value.schemaVersion !== ENGINE_CREDENTIAL_SINK_PROVISION_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, PROVISION_KEYS, "credentialSinkProvision", errors); + + if (!isPlainObject(value.transaction)) { + errors.push("transaction_must_be_object"); + } else { + rejectUnknownKeys(value.transaction, TRANSACTION_KEYS, "transaction", errors); + requiredOpaqueId(value.transaction.id, "transaction.id", errors); + requiredOpaqueId(value.transaction.idempotencyKey, "transaction.idempotencyKey", errors); + requiredTimestamp(value.transaction.requestedAt, "transaction.requestedAt", errors); + requiredTimestamp(value.transaction.requestExpiresAt, "transaction.requestExpiresAt", errors); + requiredHash(value.transaction.policyHash, "transaction.policyHash", errors); + if (value.transaction.failureMode !== "rollback-all") errors.push("transaction.failureMode_must_be_rollback-all"); + validateIssuer(value.transaction.issuer, "transaction.issuer", errors); + validateAttestation(value.transaction.attestation, "transaction.attestation", errors); + validateRequestWindow(value.transaction.requestedAt, value.transaction.requestExpiresAt, nowMs, errors); + } + + if (!Array.isArray(value.bindings) || value.bindings.length === 0) { + errors.push("bindings_must_be_nonempty_array"); + } else if (value.bindings.length > MAX_BINDINGS) { + errors.push("bindings_limit_exceeded"); + } else { + const bindingIds = new Set(); + const targets = new Set(); + value.bindings.forEach((binding, index) => { + validateProvisionBinding(binding, index, value.transaction?.requestedAt, errors); + if (!isPlainObject(binding)) return; + if (bindingIds.has(binding.bindingId)) errors.push("bindings_bindingId_must_be_unique"); + bindingIds.add(binding.bindingId); + const targetKey = targetIdentity(binding.target); + if (targetKey && targets.has(targetKey)) errors.push("bindings_target_credential_must_be_unique"); + if (targetKey) targets.add(targetKey); + }); + } + + if (isPlainObject(value.transaction) && HASH.test(String(value.transaction.policyHash || ""))) { + const expectedHash = computeEngineCredentialSinkPolicyHash(value); + if (value.transaction.policyHash !== expectedHash) errors.push("transaction.policyHash_mismatch"); + verifyProvisionAttestation(value.transaction, issuerPublicKeys, errors); + } + return result(errors); +} + +/** + * Computes the aggregate policy digest over the complete secret-free request + * descriptor. Plaintext `material` is excluded by construction, while its + * high-entropy capability digest is included; changing a target, grant, + * capability, expiry, individual policy hash or transaction envelope changes + * the aggregate digest and invalidates the issuer attestation. + */ +export function computeEngineCredentialSinkPolicyHash(value) { + const descriptor = { + schemaVersion: value?.schemaVersion, + transaction: isPlainObject(value?.transaction) ? { + id: value.transaction.id, + idempotencyKey: value.transaction.idempotencyKey, + requestedAt: value.transaction.requestedAt, + requestExpiresAt: value.transaction.requestExpiresAt, + failureMode: value.transaction.failureMode, + issuer: value.transaction.issuer, + } : value?.transaction, + bindings: Array.isArray(value?.bindings) + ? value.bindings.map(secretFreeBindingDescriptor) + : value?.bindings, + }; + return `sha256:${createHash("sha256").update(stableJson(descriptor), "utf8").digest("hex")}`; +} + +/** + * Receipt is intentionally incapable of carrying credential material. When a + * request is supplied, the validator also proves the sink committed the exact + * requested workflow/node/type set without target substitution. + */ +export function validateEngineCredentialSinkReceipt(value, { request, issuerPublicKeys } = {}) { + const errors = []; + if (!isPlainObject(value)) return result(["credentialSinkReceipt_must_be_object"]); + if (value.schemaVersion !== ENGINE_CREDENTIAL_SINK_RECEIPT_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, RECEIPT_KEYS, "credentialSinkReceipt", errors); + requiredOpaqueId(value.transactionId, "transactionId", errors); + requiredOpaqueId(value.idempotencyKey, "idempotencyKey", errors); + requiredHash(value.policyHash, "policyHash", errors); + requiredTimestamp(value.processedAt, "processedAt", errors); + const outcomes = new Set(["committed", "rolled-back", "rejected", "rollback-failed"]); + if (!outcomes.has(value.outcome)) errors.push("outcome_invalid"); + + if (!Array.isArray(value.credentials)) { + errors.push("credentials_must_be_array"); + } else { + const bindingIds = new Set(); + const refs = new Set(); + value.credentials.forEach((credential, index) => { + validateReceiptCredential(credential, index, errors); + if (!isPlainObject(credential)) return; + if (bindingIds.has(credential.bindingId)) errors.push("credentials_bindingId_must_be_unique"); + bindingIds.add(credential.bindingId); + if (refs.has(credential.credentialRef)) errors.push("credentials_credentialRef_must_be_unique"); + refs.add(credential.credentialRef); + }); + } + validateReceiptOutcome(value, errors); + if (containsSecretLikeMaterial(value)) errors.push("receipt_must_not_contain_secret_material"); + if (request !== undefined) compareReceiptToProvisionRequest(value, request, issuerPublicKeys, errors); + return result(errors); +} + +export function computeEngineCredentialSinkReceiptHash(value) { + return `sha256:${createHash("sha256").update(stableJson(value), "utf8").digest("hex")}`; +} + +export function validateEngineCredentialSinkRollback(value, { now = Date.now() } = {}) { + const errors = []; + const nowMs = normalizeNow(now, errors); + if (!isPlainObject(value)) return result(["credentialSinkRollback_must_be_object"]); + if (value.schemaVersion !== ENGINE_CREDENTIAL_SINK_ROLLBACK_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, ROLLBACK_REQUEST_KEYS, "credentialSinkRollback", errors); + if (!isPlainObject(value.rollback)) { + errors.push("rollback_must_be_object"); + return result(errors); + } + rejectUnknownKeys(value.rollback, ROLLBACK_KEYS, "rollback", errors); + requiredOpaqueId(value.rollback.id, "rollback.id", errors); + requiredOpaqueId(value.rollback.idempotencyKey, "rollback.idempotencyKey", errors); + requiredOpaqueId(value.rollback.transactionId, "rollback.transactionId", errors); + requiredTimestamp(value.rollback.requestedAt, "rollback.requestedAt", errors); + requiredTimestamp(value.rollback.requestExpiresAt, "rollback.requestExpiresAt", errors); + requiredHash(value.rollback.policyHash, "rollback.policyHash", errors); + requiredHash(value.rollback.committedReceiptHash, "rollback.committedReceiptHash", errors); + requiredReason(value.rollback.reasonCode, "rollback.reasonCode", errors); + validateRequestWindow(value.rollback.requestedAt, value.rollback.requestExpiresAt, nowMs, errors); + if (containsSecretLikeMaterial(value)) errors.push("rollback_must_not_contain_secret_material"); + return result(errors); +} + +export function validateEngineCredentialSinkRollbackReceipt(value, { request } = {}) { + const errors = []; + if (!isPlainObject(value)) return result(["credentialSinkRollbackReceipt_must_be_object"]); + if (value.schemaVersion !== ENGINE_CREDENTIAL_SINK_ROLLBACK_RECEIPT_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, ROLLBACK_RECEIPT_KEYS, "credentialSinkRollbackReceipt", errors); + requiredOpaqueId(value.rollbackId, "rollbackId", errors); + requiredOpaqueId(value.transactionId, "transactionId", errors); + requiredHash(value.policyHash, "policyHash", errors); + requiredHash(value.committedReceiptHash, "committedReceiptHash", errors); + requiredTimestamp(value.processedAt, "processedAt", errors); + if (!new Set(["rolled-back", "rejected", "rollback-failed"]).has(value.outcome)) errors.push("outcome_invalid"); + if (value.outcome === "rolled-back") { + if (value.errorCode !== undefined) errors.push("errorCode_forbidden_for_success"); + } else { + requiredReason(value.errorCode, "errorCode", errors); + } + if (containsSecretLikeMaterial(value)) errors.push("rollbackReceipt_must_not_contain_secret_material"); + if (request !== undefined && isPlainObject(request?.rollback)) { + if (!validateEngineCredentialSinkRollback(request, { now: value.processedAt }).ok) { + errors.push("request_invalid_for_rollback_receipt_comparison"); + } + if (value.rollbackId !== request.rollback.id) errors.push("rollbackId_request_mismatch"); + if (value.transactionId !== request.rollback.transactionId) errors.push("transactionId_request_mismatch"); + if (value.policyHash !== request.rollback.policyHash) errors.push("policyHash_request_mismatch"); + if (value.committedReceiptHash !== request.rollback.committedReceiptHash) { + errors.push("committedReceiptHash_request_mismatch"); + } + } + return result(errors); +} + +export function validateEngineCredentialSinkAudit(value) { + const errors = []; + if (!isPlainObject(value)) return result(["credentialSinkAudit_must_be_object"]); + if (value.schemaVersion !== ENGINE_CREDENTIAL_SINK_AUDIT_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, AUDIT_KEYS, "credentialSinkAudit", errors); + requiredOpaqueId(value.eventId, "eventId", errors); + requiredOpaqueId(value.transactionId, "transactionId", errors); + requiredOpaqueId(value.operationId, "operationId", errors); + if (!new Set(["provision", "rollback"]).has(value.operation)) errors.push("operation_invalid"); + if (!new Set(["committed", "rolled-back", "rejected", "rollback-failed"]).has(value.outcome)) errors.push("outcome_invalid"); + if (value.operation === "provision" && value.outcome === "rolled-back" && !value.reasonCode) { + errors.push("reasonCode_required"); + } + if (value.operation === "rollback" && value.outcome === "committed") errors.push("rollback_outcome_invalid"); + requiredTimestamp(value.occurredAt, "occurredAt", errors); + requiredHash(value.policyHash, "policyHash", errors); + if (!isPlainObject(value.principal)) { + errors.push("principal_must_be_object"); + } else { + rejectUnknownKeys(value.principal, PRINCIPAL_KEYS, "principal", errors); + requiredIdentifier(value.principal.serviceId, "principal.serviceId", errors); + requiredHash(value.principal.fingerprint, "principal.fingerprint", errors); + } + if (!Array.isArray(value.targets)) { + errors.push("targets_must_be_array"); + } else { + value.targets.forEach((target, index) => validateAuditTarget(target, index, errors)); + } + if (value.reasonCode !== undefined) requiredReason(value.reasonCode, "reasonCode", errors); + if (new Set(["rejected", "rollback-failed"]).has(value.outcome) && value.reasonCode === undefined) { + errors.push("reasonCode_required"); + } + if (containsSecretLikeMaterial(value)) errors.push("audit_must_not_contain_secret_material"); + return result(errors); +} + +/** Returns audit-safe target descriptors; material and credential refs cannot escape. */ +export function engineCredentialSinkAuditTargets(request, receipt) { + const refByBinding = new Map( + Array.isArray(receipt?.credentials) + ? receipt.credentials.map((item) => [item.bindingId, item.credentialRef]) + : [], + ); + return Array.isArray(request?.bindings) ? request.bindings.map((binding) => { + const descriptor = secretFreeBindingDescriptor(binding); + const credentialRef = refByBinding.get(binding.bindingId); + return credentialRef + ? { ...descriptor, credentialRefHash: sha256Value(credentialRef) } + : descriptor; + }) : []; +} + +function validateProvisionBinding(value, index, requestedAt, errors) { + const path = `bindings[${index}]`; + if (!isPlainObject(value)) { + errors.push(`${path}_must_be_object`); + return; + } + rejectUnknownKeys(value, BINDING_KEYS, path, errors); + requiredIdentifier(value.bindingId, `${path}.bindingId`, errors); + const spec = CAPABILITY_SPECS[value.capabilityType]; + if (!spec) errors.push(`${path}.capabilityType_invalid`); + requiredOpaqueId(value.grantId, `${path}.grantId`, errors); + validateTarget(value.target, path, errors); + requiredTimestamp(value.expiresAt, `${path}.expiresAt`, errors); + requiredHash(value.policyHash, `${path}.policyHash`, errors); + requiredHash(value.capabilityDigest, `${path}.capabilityDigest`, errors); + if (isTimestamp(requestedAt) && isTimestamp(value.expiresAt) && Date.parse(value.expiresAt) <= Date.parse(requestedAt)) { + errors.push(`${path}.expiresAt_must_be_after_requestedAt`); + } + if (spec && isPlainObject(value.target)) { + if (value.target.nodeType !== spec.nodeType) errors.push(`${path}.target.nodeType_capability_mismatch`); + if (value.target.credentialType !== spec.credentialType) errors.push(`${path}.target.credentialType_capability_mismatch`); + } + if (!isPlainObject(value.material)) { + errors.push(`${path}.material_must_be_object`); + } else { + rejectUnknownKeys(value.material, MATERIAL_KEYS, `${path}.material`, errors); + if (value.material.format !== "opaque-bearer") errors.push(`${path}.material.format_must_be_opaque-bearer`); + if (!spec || typeof value.material.value !== "string" || !spec.materialPattern.test(value.material.value)) { + errors.push(`${path}.material.value_invalid_for_capability`); + } else if (value.capabilityDigest !== computeEngineCredentialCapabilityDigest(value.material.value)) { + errors.push(`${path}.capabilityDigest_material_mismatch`); + } + } +} + +function validateTarget(value, path, errors) { + if (!isPlainObject(value)) { + errors.push(`${path}.target_must_be_object`); + return; + } + rejectUnknownKeys(value, TARGET_KEYS, `${path}.target`, errors); + requiredOpaqueId(value.workflowId, `${path}.target.workflowId`, errors); + requiredOpaqueId(value.workflowRevision, `${path}.target.workflowRevision`, errors); + requiredOpaqueId(value.nodeId, `${path}.target.nodeId`, errors); + if (typeof value.nodeType !== "string" || !NODE_TYPE.test(value.nodeType)) errors.push(`${path}.target.nodeType_invalid`); + if (typeof value.credentialType !== "string" || !CREDENTIAL_TYPE.test(value.credentialType)) { + errors.push(`${path}.target.credentialType_invalid`); + } +} + +function validateReceiptCredential(value, index, errors) { + const path = `credentials[${index}]`; + if (!isPlainObject(value)) { + errors.push(`${path}_must_be_object`); + return; + } + rejectUnknownKeys(value, RECEIPT_CREDENTIAL_KEYS, path, errors); + requiredIdentifier(value.bindingId, `${path}.bindingId`, errors); + if (!CAPABILITY_SPECS[value.capabilityType]) errors.push(`${path}.capabilityType_invalid`); + requiredOpaqueId(value.grantId, `${path}.grantId`, errors); + validateTarget(value.target, path, errors); + if (typeof value.credentialRef !== "string" || !CREDENTIAL_REFERENCE.test(value.credentialRef)) { + errors.push(`${path}.credentialRef_invalid`); + } + requiredTimestamp(value.expiresAt, `${path}.expiresAt`, errors); + requiredHash(value.policyHash, `${path}.policyHash`, errors); + requiredHash(value.capabilityDigest, `${path}.capabilityDigest`, errors); + if (!new Set(["created", "reused", "rotated"]).has(value.disposition)) errors.push(`${path}.disposition_invalid`); +} + +function validateReceiptOutcome(value, errors) { + if (!isPlainObject(value.rollback)) { + errors.push("rollback_must_be_object"); + return; + } + rejectUnknownKeys(value.rollback, RECEIPT_ROLLBACK_KEYS, "rollback", errors); + const expectedRollback = { + committed: "not-required", + "rolled-back": "complete", + rejected: "not-started", + "rollback-failed": "incomplete", + }[value.outcome]; + if (expectedRollback && value.rollback.status !== expectedRollback) errors.push("rollback.status_outcome_mismatch"); + if (new Set(["complete", "incomplete"]).has(value.rollback.status)) { + requiredTimestamp(value.rollback.completedAt, "rollback.completedAt", errors); + } else if (value.rollback.completedAt !== undefined) { + errors.push("rollback.completedAt_not_allowed"); + } + if (value.outcome === "committed") { + if (!Array.isArray(value.credentials) || value.credentials.length === 0) errors.push("committed_credentials_required"); + if (value.errorCode !== undefined) errors.push("errorCode_forbidden_for_success"); + } else { + if (Array.isArray(value.credentials) && value.credentials.length !== 0) errors.push("noncommitted_credentials_must_be_empty"); + requiredReason(value.errorCode, "errorCode", errors); + } +} + +function compareReceiptToProvisionRequest(receipt, request, issuerPublicKeys, errors) { + const requestValidation = validateEngineCredentialSinkProvision(request, { + now: receipt.processedAt, + issuerPublicKeys, + }); + if (!requestValidation.ok) { + errors.push("request_invalid_for_receipt_comparison"); + return; + } + if (receipt.transactionId !== request.transaction.id) errors.push("transactionId_request_mismatch"); + if (receipt.idempotencyKey !== request.transaction.idempotencyKey) errors.push("idempotencyKey_request_mismatch"); + if (receipt.policyHash !== request.transaction.policyHash) errors.push("policyHash_request_mismatch"); + if (receipt.outcome !== "committed") return; + if (receipt.credentials.length !== request.bindings.length) errors.push("credentials_request_count_mismatch"); + const requested = new Map(request.bindings.map((binding) => [binding.bindingId, secretFreeBindingDescriptor(binding)])); + for (const credential of receipt.credentials) { + const expected = requested.get(credential.bindingId); + if (!expected || stableJson({ + bindingId: credential.bindingId, + capabilityType: credential.capabilityType, + grantId: credential.grantId, + target: credential.target, + expiresAt: credential.expiresAt, + policyHash: credential.policyHash, + capabilityDigest: credential.capabilityDigest, + }) !== stableJson(expected)) { + errors.push("credentials_request_target_mismatch"); + } + } +} + +function validateAuditTarget(value, index, errors) { + const path = `targets[${index}]`; + if (!isPlainObject(value)) { + errors.push(`${path}_must_be_object`); + return; + } + rejectUnknownKeys(value, AUDIT_TARGET_KEYS, path, errors); + requiredIdentifier(value.bindingId, `${path}.bindingId`, errors); + if (!CAPABILITY_SPECS[value.capabilityType]) errors.push(`${path}.capabilityType_invalid`); + requiredOpaqueId(value.grantId, `${path}.grantId`, errors); + validateTarget(value.target, path, errors); + requiredTimestamp(value.expiresAt, `${path}.expiresAt`, errors); + requiredHash(value.policyHash, `${path}.policyHash`, errors); + requiredHash(value.capabilityDigest, `${path}.capabilityDigest`, errors); + if (value.credentialRefHash !== undefined) requiredHash(value.credentialRefHash, `${path}.credentialRefHash`, errors); +} + +function secretFreeBindingDescriptor(value) { + return { + bindingId: value?.bindingId, + capabilityType: value?.capabilityType, + grantId: value?.grantId, + target: value?.target, + expiresAt: value?.expiresAt, + policyHash: value?.policyHash, + capabilityDigest: value?.capabilityDigest, + }; +} + +export function computeEngineCredentialCapabilityDigest(value) { + return sha256Value(value); +} + +function validateIssuer(value, path, errors) { + if (!isPlainObject(value)) { + errors.push(`${path}_must_be_object`); + return; + } + rejectUnknownKeys(value, ISSUER_KEYS, path, errors); + requiredIdentifier(value.serviceId, `${path}.serviceId`, errors); + requiredOpaqueId(value.keyId, `${path}.keyId`, errors); +} + +function validateAttestation(value, path, errors) { + if (!isPlainObject(value)) { + errors.push(`${path}_must_be_object`); + return; + } + rejectUnknownKeys(value, ATTESTATION_KEYS, path, errors); + if (value.algorithm !== "Ed25519") errors.push(`${path}.algorithm_must_be_Ed25519`); + if (typeof value.signature !== "string" || !ED25519_SIGNATURE.test(value.signature)) { + errors.push(`${path}.signature_invalid`); + } +} + +function verifyProvisionAttestation(transaction, issuerPublicKeys, errors) { + if (!isPlainObject(transaction?.issuer) || !isPlainObject(transaction?.attestation)) return; + if (transaction.attestation.algorithm !== "Ed25519" || !ED25519_SIGNATURE.test(String(transaction.attestation.signature || ""))) return; + const keyIdentity = `${transaction.issuer.serviceId}:${transaction.issuer.keyId}`; + const publicKey = isPlainObject(issuerPublicKeys) && Object.hasOwn(issuerPublicKeys, keyIdentity) + ? issuerPublicKeys[keyIdentity] + : undefined; + if (!publicKey) { + errors.push("transaction.issuer_public_key_required"); + return; + } + let valid = false; + try { + valid = verifySignature( + null, + Buffer.from(String(transaction.policyHash), "utf8"), + publicKey, + Buffer.from(transaction.attestation.signature, "base64url"), + ); + } catch { + valid = false; + } + if (!valid) errors.push("transaction.attestation_invalid"); +} + +function validateRequestWindow(requestedAt, requestExpiresAt, nowMs, errors) { + if (!isTimestamp(requestedAt) || !isTimestamp(requestExpiresAt)) return; + const requested = Date.parse(requestedAt); + const expires = Date.parse(requestExpiresAt); + if (expires <= requested) errors.push("requestExpiresAt_must_be_after_requestedAt"); + if (expires - requested > MAX_REQUEST_LIFETIME_MS) errors.push("request_lifetime_exceeds_15_minutes"); + if (Number.isFinite(nowMs)) { + if (requested > nowMs + MAX_REQUEST_CLOCK_SKEW_MS) errors.push("requestedAt_exceeds_clock_skew"); + if (expires <= nowMs) errors.push("request_expired"); + } +} + +function normalizeNow(value, errors) { + const normalized = value instanceof Date ? value.getTime() : typeof value === "string" ? Date.parse(value) : Number(value); + if (!Number.isFinite(normalized)) { + errors.push("validation_now_invalid"); + return Number.NaN; + } + return normalized; +} + +function targetIdentity(value) { + if (!isPlainObject(value)) return ""; + return [value.workflowId, value.workflowRevision, value.nodeId, value.nodeType, value.credentialType].join("\u0000"); +} + +function requiredIdentifier(value, path, errors) { + if (typeof value !== "string" || !IDENTIFIER.test(value)) errors.push(`${path}_invalid`); +} + +function requiredOpaqueId(value, path, errors) { + if (typeof value !== "string" || !OPAQUE_ID.test(value)) errors.push(`${path}_invalid`); +} + +function requiredHash(value, path, errors) { + if (typeof value !== "string" || !HASH.test(value)) errors.push(`${path}_invalid`); +} + +function requiredReason(value, path, errors) { + if (typeof value !== "string" || !REASON_CODE.test(value)) errors.push(`${path}_invalid`); +} + +function requiredTimestamp(value, path, errors) { + if (!isTimestamp(value)) errors.push(`${path}_invalid_timestamp`); +} + +function isTimestamp(value) { + return typeof value === "string" && !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value; +} + +function sha256Value(value) { + return `sha256:${createHash("sha256").update(String(value), "utf8").digest("hex")}`; +} + +function stableJson(value) { + return JSON.stringify(sortValue(value)); +} + +function sortValue(value) { + if (Array.isArray(value)) return value.map(sortValue); + if (!isPlainObject(value)) return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sortValue(value[key])])); +} + +function containsSecretLikeMaterial(value) { + if (typeof value === "string") return SECRET_LIKE_VALUE.test(value); + if (Array.isArray(value)) return value.some(containsSecretLikeMaterial); + if (!isPlainObject(value)) return false; + return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeMaterial(child)); +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function rejectUnknownKeys(value, allowed, path, errors) { + if (!isPlainObject(value)) return; + for (const key of Object.keys(value)) { + if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`); + } +} + +function result(errors) { + const uniqueErrors = [...new Set(errors)]; + return Object.freeze({ ok: uniqueErrors.length === 0, errors: Object.freeze(uniqueErrors) }); +} diff --git a/packages/external-provider-contract/src/engine-private-extension.mjs b/packages/external-provider-contract/src/engine-private-extension.mjs new file mode 100644 index 0000000..4598e92 --- /dev/null +++ b/packages/external-provider-contract/src/engine-private-extension.mjs @@ -0,0 +1,721 @@ +import { createHash } from "node:crypto"; + +export const ENGINE_PRIVATE_EXTENSION_PLAN_REQUEST_SCHEMA_VERSION = + "nodedc.engine.private-extension.plan-request/v1"; +export const ENGINE_PRIVATE_EXTENSION_PLAN_SCHEMA_VERSION = + "nodedc.engine.private-extension.plan/v1"; +export const ENGINE_PRIVATE_EXTENSION_APPLY_REQUEST_SCHEMA_VERSION = + "nodedc.engine.private-extension.apply-request/v1"; +export const ENGINE_PRIVATE_EXTENSION_APPLY_RECEIPT_SCHEMA_VERSION = + "nodedc.engine.private-extension.apply-receipt/v1"; +export const ENGINE_PRIVATE_EXTENSION_OPERATION_SCHEMA_VERSION = + "nodedc.engine.private-extension.operation/v1"; +export const ENGINE_PRIVATE_EXTENSION_STATUS_SCHEMA_VERSION = + "nodedc.engine.private-extension.status/v1"; + +export const ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY = "engine.private-extension.manage"; +export const ENGINE_PRIVATE_EXTENSION_READ_CAPABILITY = "engine.private-extension.read"; +export const ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME = "n8n-nodes-ndc"; +export const ENGINE_PRIVATE_EXTENSION_INACTIVE_BASELINE = "n8n-nodes-ndc.inactive/v1"; + +export const ENGINE_PRIVATE_EXTENSION_NODE_TYPES = Object.freeze([ + "n8n-nodes-ndc.ndcDataProductPublish", + "n8n-nodes-ndc.ndcDataProductRead", + "n8n-nodes-ndc.ndcFoundryBinding", +]); + +export const ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES = Object.freeze([ + "ndcDataProductWriterApi", + "ndcDataProductReaderApi", + "ndcFoundryBindingApi", +]); +export const ENGINE_PRIVATE_EXTENSION_CREDENTIAL_SCHEMAS = ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES; + +const PLAN_REQUEST_KEYS = new Set([ + "schemaVersion", + "requestId", + "idempotencyKey", + "action", + "requestedAt", + "requestExpiresAt", + "expectedCurrentGeneration", + "target", +]); +const RELEASE_STATE_KEYS = new Set(["kind", "packageName", "releaseId", "packageSha256"]); +const INACTIVE_STATE_KEYS = new Set(["kind", "packageName", "baselineId"]); +const PREVIOUS_STATE_TARGET_KEYS = new Set(["kind", "packageName"]); +const PLAN_KEYS = new Set([ + "schemaVersion", + "planId", + "planHash", + "requestId", + "idempotencyKey", + "action", + "createdAt", + "expiresAt", + "singleUse", + "requiredCapability", + "expectedCurrentGeneration", + "nextGeneration", + "currentState", + "targetState", + "recoveryState", + "actions", + "transition", + "acceptance", + "failurePolicy", +]); +const TRANSITION_KEYS = new Set([ + "mountMode", + "loaderMode", + "loaderPath", + "loaderEnvironment", + "quiesceMode", + "stateSwitch", + "runtimeAction", + "scope", + "requireUniformGeneration", + "hotReload", + "preserveCredentials", +]); +const LOADER_ENVIRONMENT_KEYS = new Set([ + "N8N_COMMUNITY_PACKAGES_ENABLED", + "N8N_COMMUNITY_PACKAGES_PREVENT_LOADING", + "N8N_REINSTALL_MISSING_PACKAGES", +]); +const ACCEPTANCE_SPEC_KEYS = new Set([ + "mode", + "nodeTypes", + "credentialSchemas", + "requireUniformGeneration", +]); +const FAILURE_POLICY_KEYS = new Set([ + "mode", + "rollbackFailureOutcome", + "preserveImmutableRelease", + "preserveCredentials", +]); +const APPLY_REQUEST_KEYS = new Set([ + "schemaVersion", + "planId", + "planHash", + "idempotencyKey", + "confirmedAt", +]); +const APPLY_RECEIPT_KEYS = new Set([ + "schemaVersion", + "operationId", + "planId", + "planHash", + "action", + "acceptedAt", + "state", +]); +const OPERATION_KEYS = new Set([ + "schemaVersion", + "operationId", + "planId", + "planHash", + "action", + "state", + "outcome", + "phase", + "expectedCurrentGeneration", + "nextGeneration", + "targetState", + "recoveryState", + "effectiveState", + "runtime", + "acceptance", + "updatedAt", + "errorCode", +]); +const RUNTIME_KEYS = new Set(["mode", "expectedInstances", "readyInstances", "generation"]); +const ACCEPTANCE_REPORT_KEYS = new Set([ + "state", + "nodeTypes", + "credentialSchemas", + "uniformGeneration", +]); +const OBSERVATION_KEYS = new Set(["expected", "observed"]); +const STATUS_KEYS = new Set([ + "schemaVersion", + "packageName", + "generation", + "health", + "currentState", + "previousState", + "activeOperationId", + "runtime", + "acceptance", + "updatedAt", + "errorCode", +]); + +const RELEASE_ID = /^\d+\.\d+\.\d+-[a-f0-9]{16}$/; +const SHA256 = /^[a-f0-9]{64}$/; +const HASH = /^sha256:[a-f0-9]{64}$/; +const OPAQUE_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{2,159}$/; +const REASON_CODE = /^[a-z][a-z0-9_.:-]{2,127}$/; +const MAX_REQUEST_LIFETIME_MS = 15 * 60 * 1000; +const MAX_CLOCK_SKEW_MS = 60 * 1000; + +const ACTIVATE_ACTIONS = Object.freeze([ + "verify_staged_immutable_release", + "prepare_sealed_package_tree", + "verify_community_package_loader_policy", + "quiesce_deploy_run_and_drain_queue", + "record_recovery_state", + "atomic_switch_current", + "force_recreate_main_workers_webhooks_as_version_barrier", + "verify_exact_runtime_acceptance", + "commit_active_state", + "resume_deploy_run", +]); +const ROLLBACK_ACTIONS = Object.freeze([ + "verify_previous_activation_state", + "verify_community_package_loader_policy", + "quiesce_deploy_run_and_drain_queue", + "record_recovery_state", + "atomic_switch_current_to_previous", + "force_recreate_main_workers_webhooks_as_version_barrier", + "verify_exact_runtime_acceptance", + "commit_rolled_back_state", + "resume_deploy_run", +]); + +const NON_TERMINAL_STATES = new Set([ + "queued", + "preparing", + "switching", + "recreating", + "accepting", + "rolling-back", +]); +const TERMINAL_STATE_OUTCOMES = Object.freeze({ + active: "committed", + rejected: "rejected", + quarantined: "quarantined", +}); + +export function authorizeEnginePrivateExtensionOperation(operation, grantedCapabilities) { + const capabilities = new Set(Array.isArray(grantedCapabilities) ? grantedCapabilities : []); + const requiredCapability = operation === "status" + ? ENGINE_PRIVATE_EXTENSION_READ_CAPABILITY + : ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY; + const authorized = operation === "status" + ? capabilities.has(ENGINE_PRIVATE_EXTENSION_READ_CAPABILITY) + || capabilities.has(ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY) + : capabilities.has(ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY); + return Object.freeze({ + ok: authorized, + requiredCapability, + errors: Object.freeze(authorized ? [] : ["engine_private_extension_capability_required"]), + }); +} + +export function validateEnginePrivateExtensionPlanRequest(value, options = {}) { + const errors = []; + validatePlanRequestBody(value, options.now, errors); + addAuthorizationErrors("plan", options.grantedCapabilities, errors); + return result(errors); +} + +export function computeEnginePrivateExtensionPlanHash(value) { + if (!isPlainObject(value)) return ""; + const descriptor = { ...value }; + delete descriptor.planHash; + return "sha256:" + createHash("sha256").update(stableJson(descriptor), "utf8").digest("hex"); +} + +export function validateEnginePrivateExtensionPlan(value, { request } = {}) { + const errors = []; + if (!isPlainObject(value)) return result(["enginePrivateExtensionPlan_must_be_object"]); + if (value.schemaVersion !== ENGINE_PRIVATE_EXTENSION_PLAN_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, PLAN_KEYS, "enginePrivateExtensionPlan", errors); + requiredOpaqueId(value.planId, "planId", errors); + requiredHash(value.planHash, "planHash", errors); + requiredOpaqueId(value.requestId, "requestId", errors); + requiredOpaqueId(value.idempotencyKey, "idempotencyKey", errors); + if (!new Set(["activate", "rollback"]).has(value.action)) errors.push("action_invalid"); + requiredTimestamp(value.createdAt, "createdAt", errors); + requiredTimestamp(value.expiresAt, "expiresAt", errors); + validateWindow(value.createdAt, value.expiresAt, Date.parse(value.createdAt), errors, "plan"); + if (value.singleUse !== true) errors.push("singleUse_must_be_true"); + if (value.requiredCapability !== ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY) { + errors.push("requiredCapability_mismatch"); + } + requiredGeneration(value.expectedCurrentGeneration, "expectedCurrentGeneration", errors); + requiredGeneration(value.nextGeneration, "nextGeneration", errors); + if (Number.isInteger(value.expectedCurrentGeneration) + && value.nextGeneration !== value.expectedCurrentGeneration + 1) { + errors.push("nextGeneration_must_increment_current_generation"); + } + validateActivationState(value.currentState, "currentState", errors); + validateActivationState(value.targetState, "targetState", errors); + validateActivationState(value.recoveryState, "recoveryState", errors); + if (!sameValue(value.currentState, value.recoveryState)) errors.push("recoveryState_must_equal_currentState"); + if (value.action === "activate" && value.targetState?.kind !== "release") { + errors.push("activate_targetState_must_be_release"); + } + if (sameValue(value.currentState, value.targetState)) errors.push("targetState_must_differ_from_currentState"); + validateExactArray( + value.actions, + value.action === "rollback" ? ROLLBACK_ACTIONS : ACTIVATE_ACTIONS, + "actions", + errors, + ); + validateTransition(value.transition, errors); + validateAcceptanceSpec(value.acceptance, value.targetState, errors); + validateFailurePolicy(value.failurePolicy, errors); + if (HASH.test(String(value.planHash || "")) && value.planHash !== computeEnginePrivateExtensionPlanHash(value)) { + errors.push("planHash_mismatch"); + } + if (request !== undefined) comparePlanToRequest(value, request, errors); + return result(errors); +} + +export function validateEnginePrivateExtensionApplyRequest(value, { plan, now = Date.now(), grantedCapabilities } = {}) { + const errors = []; + if (!isPlainObject(value)) return result(["enginePrivateExtensionApplyRequest_must_be_object"]); + if (value.schemaVersion !== ENGINE_PRIVATE_EXTENSION_APPLY_REQUEST_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, APPLY_REQUEST_KEYS, "enginePrivateExtensionApplyRequest", errors); + requiredOpaqueId(value.planId, "planId", errors); + requiredHash(value.planHash, "planHash", errors); + requiredOpaqueId(value.idempotencyKey, "idempotencyKey", errors); + requiredTimestamp(value.confirmedAt, "confirmedAt", errors); + addAuthorizationErrors("apply", grantedCapabilities, errors); + const nowMs = normalizeNow(now, errors); + if (isTimestamp(value.confirmedAt) && Date.parse(value.confirmedAt) > nowMs + MAX_CLOCK_SKEW_MS) { + errors.push("confirmedAt_exceeds_clock_skew"); + } + if (plan !== undefined) { + const planValidation = validateEnginePrivateExtensionPlan(plan); + if (!planValidation.ok) errors.push("plan_invalid_for_apply"); + if (value.planId !== plan?.planId) errors.push("planId_plan_mismatch"); + if (value.planHash !== plan?.planHash) errors.push("planHash_plan_mismatch"); + if (value.idempotencyKey !== plan?.idempotencyKey) errors.push("idempotencyKey_plan_mismatch"); + if (isTimestamp(plan?.expiresAt) && nowMs > Date.parse(plan.expiresAt)) errors.push("plan_expired"); + if (isTimestamp(value.confirmedAt) && isTimestamp(plan?.expiresAt) + && Date.parse(value.confirmedAt) > Date.parse(plan.expiresAt)) { + errors.push("confirmedAt_after_plan_expiresAt"); + } + if (isTimestamp(value.confirmedAt) && isTimestamp(plan?.createdAt) + && Date.parse(value.confirmedAt) < Date.parse(plan.createdAt)) { + errors.push("confirmedAt_before_plan_createdAt"); + } + } + return result(errors); +} + +export function validateEnginePrivateExtensionApplyReceipt(value, { plan } = {}) { + const errors = []; + if (!isPlainObject(value)) return result(["enginePrivateExtensionApplyReceipt_must_be_object"]); + if (value.schemaVersion !== ENGINE_PRIVATE_EXTENSION_APPLY_RECEIPT_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, APPLY_RECEIPT_KEYS, "enginePrivateExtensionApplyReceipt", errors); + requiredOpaqueId(value.operationId, "operationId", errors); + requiredOpaqueId(value.planId, "planId", errors); + requiredHash(value.planHash, "planHash", errors); + if (!new Set(["activate", "rollback"]).has(value.action)) errors.push("action_invalid"); + requiredTimestamp(value.acceptedAt, "acceptedAt", errors); + if (value.state !== "queued") errors.push("apply_receipt_state_must_be_queued"); + if (plan !== undefined) { + if (value.planId !== plan?.planId) errors.push("planId_plan_mismatch"); + if (value.planHash !== plan?.planHash) errors.push("planHash_plan_mismatch"); + if (value.action !== plan?.action) errors.push("action_plan_mismatch"); + if (isTimestamp(value.acceptedAt) && isTimestamp(plan?.expiresAt) + && Date.parse(value.acceptedAt) > Date.parse(plan.expiresAt)) { + errors.push("acceptedAt_after_plan_expiresAt"); + } + } + return result(errors); +} + +export function validateEnginePrivateExtensionOperation(value) { + const errors = []; + if (!isPlainObject(value)) return result(["enginePrivateExtensionOperation_must_be_object"]); + if (value.schemaVersion !== ENGINE_PRIVATE_EXTENSION_OPERATION_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, OPERATION_KEYS, "enginePrivateExtensionOperation", errors); + requiredOpaqueId(value.operationId, "operationId", errors); + requiredOpaqueId(value.planId, "planId", errors); + requiredHash(value.planHash, "planHash", errors); + if (!new Set(["activate", "rollback"]).has(value.action)) errors.push("action_invalid"); + if (!new Set([...NON_TERMINAL_STATES, "active", "rolled-back", "rejected", "quarantined"]).has(value.state)) { + errors.push("state_invalid"); + } + if (!new Set([ + "pending", + "committed", + "automatically-rolled-back", + "explicitly-rolled-back", + "rejected", + "quarantined", + ]).has(value.outcome)) errors.push("outcome_invalid"); + if (!new Set([ + "queued", + "prepare", + "switch", + "force-recreate", + "acceptance", + "rollback", + "complete", + ]).has(value.phase)) errors.push("phase_invalid"); + requiredGeneration(value.expectedCurrentGeneration, "expectedCurrentGeneration", errors); + requiredGeneration(value.nextGeneration, "nextGeneration", errors); + if (Number.isInteger(value.expectedCurrentGeneration) + && value.nextGeneration !== value.expectedCurrentGeneration + 1) { + errors.push("nextGeneration_must_increment_current_generation"); + } + validateActivationState(value.targetState, "targetState", errors); + validateActivationState(value.recoveryState, "recoveryState", errors); + validateActivationState(value.effectiveState, "effectiveState", errors); + if (sameValue(value.targetState, value.recoveryState)) errors.push("targetState_must_differ_from_recoveryState"); + validateRuntimeReport(value.runtime, errors); + validateAcceptanceReport(value.acceptance, value.effectiveState, errors); + requiredTimestamp(value.updatedAt, "updatedAt", errors); + validateOperationOutcome(value, errors); + return result(errors); +} + +export function validateEnginePrivateExtensionStatus(value) { + const errors = []; + if (!isPlainObject(value)) return result(["enginePrivateExtensionStatus_must_be_object"]); + if (value.schemaVersion !== ENGINE_PRIVATE_EXTENSION_STATUS_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, STATUS_KEYS, "enginePrivateExtensionStatus", errors); + if (value.packageName !== ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME) errors.push("packageName_mismatch"); + requiredGeneration(value.generation, "generation", errors); + if (!new Set(["ready", "transitioning", "quarantined"]).has(value.health)) errors.push("health_invalid"); + validateActivationState(value.currentState, "currentState", errors); + validateActivationState(value.previousState, "previousState", errors); + if (value.activeOperationId !== undefined) requiredOpaqueId(value.activeOperationId, "activeOperationId", errors); + validateRuntimeReport(value.runtime, errors); + validateAcceptanceReport(value.acceptance, value.currentState, errors); + requiredTimestamp(value.updatedAt, "updatedAt", errors); + if (value.runtime?.generation !== value.generation) errors.push("runtime_generation_mismatch"); + if (value.health === "ready") { + if (value.activeOperationId !== undefined) errors.push("ready_status_must_not_have_active_operation"); + if (value.acceptance?.state !== "accepted") errors.push("ready_status_requires_acceptance"); + if (value.errorCode !== undefined) errors.push("ready_status_must_not_have_errorCode"); + } else if (value.health === "transitioning") { + if (value.activeOperationId === undefined) errors.push("transitioning_status_requires_active_operation"); + } else { + requiredReason(value.errorCode, "errorCode", errors); + } + return result(errors); +} + +function validatePlanRequestBody(value, now, errors) { + if (!isPlainObject(value)) { + errors.push("enginePrivateExtensionPlanRequest_must_be_object"); + return; + } + if (value.schemaVersion !== ENGINE_PRIVATE_EXTENSION_PLAN_REQUEST_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, PLAN_REQUEST_KEYS, "enginePrivateExtensionPlanRequest", errors); + requiredOpaqueId(value.requestId, "requestId", errors); + requiredOpaqueId(value.idempotencyKey, "idempotencyKey", errors); + if (!new Set(["activate", "rollback"]).has(value.action)) errors.push("action_invalid"); + requiredTimestamp(value.requestedAt, "requestedAt", errors); + requiredTimestamp(value.requestExpiresAt, "requestExpiresAt", errors); + requiredGeneration(value.expectedCurrentGeneration, "expectedCurrentGeneration", errors); + validateWindow(value.requestedAt, value.requestExpiresAt, normalizeNow(now, errors), errors, "request"); + validateRequestTarget(value.target, value.action, errors); +} + +function validateRequestTarget(value, action, errors) { + if (!isPlainObject(value)) { + errors.push("target_must_be_object"); + return; + } + if (action === "activate") { + validateActivationState(value, "target", errors); + if (value.kind !== "release") errors.push("activate_target_must_be_release"); + return; + } + rejectUnknownKeys(value, PREVIOUS_STATE_TARGET_KEYS, "target", errors); + if (value.kind !== "previous-state") errors.push("rollback_target_must_be_previous-state"); + if (value.packageName !== ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME) errors.push("target.packageName_mismatch"); +} + +function validateActivationState(value, path, errors) { + if (!isPlainObject(value)) { + errors.push(path + "_must_be_object"); + return; + } + if (value.kind === "release") { + rejectUnknownKeys(value, RELEASE_STATE_KEYS, path, errors); + if (value.packageName !== ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME) errors.push(path + ".packageName_mismatch"); + if (typeof value.releaseId !== "string" || !RELEASE_ID.test(value.releaseId)) { + errors.push(path + ".releaseId_invalid"); + } + if (typeof value.packageSha256 !== "string" || !SHA256.test(value.packageSha256)) { + errors.push(path + ".packageSha256_invalid"); + } else if (typeof value.releaseId === "string" + && RELEASE_ID.test(value.releaseId) + && !value.releaseId.endsWith("-" + value.packageSha256.slice(0, 16))) { + errors.push(path + ".releaseId_digest_mismatch"); + } + return; + } + if (value.kind === "inactive-baseline") { + rejectUnknownKeys(value, INACTIVE_STATE_KEYS, path, errors); + if (value.packageName !== ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME) errors.push(path + ".packageName_mismatch"); + if (value.baselineId !== ENGINE_PRIVATE_EXTENSION_INACTIVE_BASELINE) errors.push(path + ".baselineId_mismatch"); + return; + } + errors.push(path + ".kind_invalid"); +} + +function validateTransition(value, errors) { + if (!isPlainObject(value)) { + errors.push("transition_must_be_object"); + return; + } + rejectUnknownKeys(value, TRANSITION_KEYS, "transition", errors); + const expected = { + mountMode: "read-only", + loaderMode: "community-package", + loaderPath: "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc", + loaderEnvironment: { + N8N_COMMUNITY_PACKAGES_ENABLED: "true", + N8N_COMMUNITY_PACKAGES_PREVENT_LOADING: "false", + N8N_REINSTALL_MISSING_PACKAGES: "false", + }, + quiesceMode: "block-deploy-run-and-drain-queue", + stateSwitch: "atomic-current-previous", + runtimeAction: "force-recreate", + scope: "main-workers-webhooks", + requireUniformGeneration: true, + hotReload: false, + preserveCredentials: true, + }; + if (isPlainObject(value.loaderEnvironment)) { + rejectUnknownKeys(value.loaderEnvironment, LOADER_ENVIRONMENT_KEYS, "transition.loaderEnvironment", errors); + } + if (!sameValue(value, expected)) errors.push("transition_policy_mismatch"); +} + +function validateAcceptanceSpec(value, targetState, errors) { + if (!isPlainObject(value)) { + errors.push("acceptance_must_be_object"); + return; + } + rejectUnknownKeys(value, ACCEPTANCE_SPEC_KEYS, "acceptance", errors); + if (value.mode !== "exact") errors.push("acceptance.mode_must_be_exact"); + const expectedNodes = expectedTypes(targetState, ENGINE_PRIVATE_EXTENSION_NODE_TYPES); + const expectedCredentials = expectedTypes(targetState, ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES); + validateExactArray(value.nodeTypes, expectedNodes, "acceptance.nodeTypes", errors); + validateExactArray(value.credentialSchemas, expectedCredentials, "acceptance.credentialSchemas", errors); + if (value.requireUniformGeneration !== true) errors.push("acceptance.requireUniformGeneration_must_be_true"); +} + +function validateFailurePolicy(value, errors) { + if (!isPlainObject(value)) { + errors.push("failurePolicy_must_be_object"); + return; + } + rejectUnknownKeys(value, FAILURE_POLICY_KEYS, "failurePolicy", errors); + const expected = { + mode: "automatic-rollback", + rollbackFailureOutcome: "quarantined", + preserveImmutableRelease: true, + preserveCredentials: true, + }; + if (!sameValue(value, expected)) errors.push("failurePolicy_mismatch"); +} + +function validateRuntimeReport(value, errors) { + if (!isPlainObject(value)) { + errors.push("runtime_must_be_object"); + return; + } + rejectUnknownKeys(value, RUNTIME_KEYS, "runtime", errors); + if (value.mode !== "force-recreate") errors.push("runtime.mode_must_be_force-recreate"); + requiredPositiveInteger(value.expectedInstances, "runtime.expectedInstances", errors); + requiredNonnegativeInteger(value.readyInstances, "runtime.readyInstances", errors); + if (Number.isInteger(value.expectedInstances) && Number.isInteger(value.readyInstances) + && value.readyInstances > value.expectedInstances) errors.push("runtime.readyInstances_exceeds_expectedInstances"); + requiredGeneration(value.generation, "runtime.generation", errors); +} + +function validateAcceptanceReport(value, effectiveState, errors) { + if (!isPlainObject(value)) { + errors.push("acceptance_must_be_object"); + return; + } + rejectUnknownKeys(value, ACCEPTANCE_REPORT_KEYS, "acceptance", errors); + if (!new Set(["pending", "accepted", "rejected"]).has(value.state)) errors.push("acceptance.state_invalid"); + const expectedNodes = expectedTypes(effectiveState, ENGINE_PRIVATE_EXTENSION_NODE_TYPES); + const expectedCredentials = expectedTypes(effectiveState, ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES); + validateObservation(value.nodeTypes, expectedNodes, "acceptance.nodeTypes", value.state, errors); + validateObservation(value.credentialSchemas, expectedCredentials, "acceptance.credentialSchemas", value.state, errors); + if (typeof value.uniformGeneration !== "boolean") errors.push("acceptance.uniformGeneration_must_be_boolean"); + if (value.state === "accepted" && value.uniformGeneration !== true) { + errors.push("accepted_runtime_requires_uniform_generation"); + } +} + +function validateObservation(value, expected, path, state, errors) { + if (!isPlainObject(value)) { + errors.push(path + "_must_be_object"); + return; + } + rejectUnknownKeys(value, OBSERVATION_KEYS, path, errors); + validateExactArray(value.expected, expected, path + ".expected", errors); + if (!Array.isArray(value.observed) || value.observed.some((item) => typeof item !== "string")) { + errors.push(path + ".observed_must_be_string_array"); + return; + } + if (new Set(value.observed).size !== value.observed.length) errors.push(path + ".observed_must_be_unique"); + if (state === "accepted" && !sameValue(value.observed, expected)) errors.push(path + ".observed_exact_set_required"); +} + +function validateOperationOutcome(value, errors) { + if (NON_TERMINAL_STATES.has(value.state)) { + if (value.outcome !== "pending") errors.push("nonterminal_operation_outcome_must_be_pending"); + return; + } + if (value.state === "rolled-back") { + const expected = value.action === "rollback" ? "explicitly-rolled-back" : "automatically-rolled-back"; + if (value.outcome !== expected) errors.push("rolled_back_outcome_mismatch"); + } else if (TERMINAL_STATE_OUTCOMES[value.state] !== value.outcome) { + errors.push("terminal_operation_outcome_mismatch"); + } + if (value.state === "active" || value.state === "rolled-back") { + if (value.acceptance?.state !== "accepted") errors.push("successful_terminal_state_requires_acceptance"); + if (value.runtime?.readyInstances !== value.runtime?.expectedInstances) { + errors.push("successful_terminal_state_requires_all_instances_ready"); + } + if (value.runtime?.generation !== value.nextGeneration) { + errors.push("successful_terminal_state_runtime_generation_mismatch"); + } + if (value.errorCode !== undefined && value.state === "active") errors.push("active_state_must_not_have_errorCode"); + if (value.state === "active" && value.action !== "activate") errors.push("active_state_action_mismatch"); + const expectedEffective = value.state === "active" + ? value.targetState + : value.action === "rollback" ? value.targetState : value.recoveryState; + if (!sameValue(value.effectiveState, expectedEffective)) errors.push("effectiveState_terminal_mismatch"); + if (value.state === "rolled-back" && value.action === "activate") { + requiredReason(value.errorCode, "errorCode", errors); + } + if (value.state === "rolled-back" && value.action === "rollback" && value.errorCode !== undefined) { + errors.push("explicit_rollback_must_not_have_errorCode"); + } + } else if (value.state === "rejected" || value.state === "quarantined") { + requiredReason(value.errorCode, "errorCode", errors); + } +} + +function comparePlanToRequest(plan, request, errors) { + const requestErrors = []; + validatePlanRequestBody(request, plan.createdAt, requestErrors); + if (requestErrors.length) errors.push("request_invalid_for_plan_comparison"); + if (plan.requestId !== request?.requestId) errors.push("requestId_request_mismatch"); + if (plan.idempotencyKey !== request?.idempotencyKey) errors.push("idempotencyKey_request_mismatch"); + if (plan.action !== request?.action) errors.push("action_request_mismatch"); + if (plan.expectedCurrentGeneration !== request?.expectedCurrentGeneration) { + errors.push("expectedCurrentGeneration_request_mismatch"); + } + if (request?.action === "activate" && !sameValue(plan.targetState, request?.target)) { + errors.push("targetState_request_mismatch"); + } + if (isTimestamp(request?.requestExpiresAt) && isTimestamp(plan.expiresAt) + && Date.parse(plan.expiresAt) > Date.parse(request.requestExpiresAt)) { + errors.push("plan_expiresAt_exceeds_request"); + } +} + +function addAuthorizationErrors(operation, grantedCapabilities, errors) { + errors.push(...authorizeEnginePrivateExtensionOperation(operation, grantedCapabilities).errors); +} + +function validateWindow(start, end, nowMs, errors, label) { + if (!isTimestamp(start) || !isTimestamp(end) || !Number.isFinite(nowMs)) return; + const startMs = Date.parse(start); + const endMs = Date.parse(end); + if (endMs <= startMs) errors.push(label + "_expiresAt_must_be_after_start"); + if (endMs - startMs > MAX_REQUEST_LIFETIME_MS) errors.push(label + "_lifetime_exceeds_15_minutes"); + if (startMs > nowMs + MAX_CLOCK_SKEW_MS) errors.push(label + "_start_exceeds_clock_skew"); + if (endMs < nowMs) errors.push(label + "_expired"); +} + +function normalizeNow(value, errors) { + const candidate = value === undefined ? Date.now() : value; + const milliseconds = typeof candidate === "number" ? candidate : Date.parse(candidate); + if (!Number.isFinite(milliseconds)) { + errors.push("now_invalid"); + return Number.NaN; + } + return milliseconds; +} + +function requiredGeneration(value, path, errors) { + requiredNonnegativeInteger(value, path, errors); +} + +function requiredPositiveInteger(value, path, errors) { + if (!Number.isInteger(value) || value < 1) errors.push(path + "_must_be_positive_integer"); +} + +function requiredNonnegativeInteger(value, path, errors) { + if (!Number.isInteger(value) || value < 0) errors.push(path + "_must_be_nonnegative_integer"); +} + +function requiredOpaqueId(value, path, errors) { + if (typeof value !== "string" || !OPAQUE_ID.test(value)) errors.push(path + "_invalid"); +} + +function requiredReason(value, path, errors) { + if (typeof value !== "string" || !REASON_CODE.test(value)) errors.push(path + "_invalid"); +} + +function requiredHash(value, path, errors) { + if (typeof value !== "string" || !HASH.test(value)) errors.push(path + "_invalid"); +} + +function requiredTimestamp(value, path, errors) { + if (!isTimestamp(value)) errors.push(path + "_invalid_timestamp"); +} + +function isTimestamp(value) { + return typeof value === "string" && Number.isFinite(Date.parse(value)); +} + +function expectedTypes(state, releaseTypes) { + return state?.kind === "release" ? [...releaseTypes] : []; +} + +function validateExactArray(actual, expected, path, errors) { + if (!Array.isArray(actual)) { + errors.push(path + "_must_be_array"); + return; + } + if (!sameValue(actual, [...expected])) errors.push(path + "_mismatch"); +} + +function rejectUnknownKeys(value, allowedKeys, path, errors) { + if (!isPlainObject(value)) return; + for (const key of Object.keys(value)) { + if (!allowedKeys.has(key)) errors.push(path + "." + key + "_not_allowed"); + } +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function sameValue(left, right) { + return stableJson(left) === stableJson(right); +} + +function stableJson(value) { + if (Array.isArray(value)) return "[" + value.map(stableJson).join(",") + "]"; + if (isPlainObject(value)) { + return "{" + Object.keys(value).sort().map((key) => JSON.stringify(key) + ":" + stableJson(value[key])).join(",") + "}"; + } + return JSON.stringify(value); +} + +function result(errors) { + const unique = [...new Set(errors)]; + return Object.freeze({ ok: unique.length === 0, errors: Object.freeze(unique) }); +} diff --git a/packages/external-provider-contract/src/index.mjs b/packages/external-provider-contract/src/index.mjs new file mode 100644 index 0000000..eb6546b --- /dev/null +++ b/packages/external-provider-contract/src/index.mjs @@ -0,0 +1,481 @@ +export const EXTERNAL_PROVIDER_CONTRACT_VERSION = "nodedc.external-provider-contract/v1"; +export const FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION = "nodedc.foundry.binding-upsert/v1"; + +export { + DATA_PRODUCT_PATCH_SCHEMA_VERSION, + DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, + DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, + validateDataProductPatch, + validateDataProductPublish, + validateDataProductSnapshot, +} from "./data-product.mjs"; + +export { + ENGINE_CREDENTIAL_SINK_AUDIT_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_CAPABILITY_TYPES, + ENGINE_CREDENTIAL_SINK_PROVISION_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_RECEIPT_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_ROLLBACK_RECEIPT_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_ROLLBACK_SCHEMA_VERSION, + computeEngineCredentialCapabilityDigest, + computeEngineCredentialSinkPolicyHash, + computeEngineCredentialSinkReceiptHash, + engineCredentialSinkAuditTargets, + validateEngineCredentialSinkAudit, + validateEngineCredentialSinkProvision, + validateEngineCredentialSinkReceipt, + validateEngineCredentialSinkRollback, + validateEngineCredentialSinkRollbackReceipt, +} from "./engine-credential-sink.mjs"; + +export { + ENGINE_PRIVATE_EXTENSION_APPLY_RECEIPT_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_APPLY_REQUEST_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_CREDENTIAL_SCHEMAS, + ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES, + ENGINE_PRIVATE_EXTENSION_INACTIVE_BASELINE, + ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY, + ENGINE_PRIVATE_EXTENSION_NODE_TYPES, + ENGINE_PRIVATE_EXTENSION_OPERATION_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME, + ENGINE_PRIVATE_EXTENSION_PLAN_REQUEST_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_PLAN_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_READ_CAPABILITY, + ENGINE_PRIVATE_EXTENSION_STATUS_SCHEMA_VERSION, + authorizeEnginePrivateExtensionOperation, + computeEnginePrivateExtensionPlanHash, + validateEnginePrivateExtensionApplyReceipt, + validateEnginePrivateExtensionApplyRequest, + validateEnginePrivateExtensionOperation, + validateEnginePrivateExtensionPlan, + validateEnginePrivateExtensionPlanRequest, + validateEnginePrivateExtensionStatus, +} from "./engine-private-extension.mjs"; + +const COLLECTION_MODES = new Set(["realtime", "manual", "weekly", "history"]); +const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]); +const CAPABILITY_CLASSIFICATIONS = new Set(["read", "metadata", "write", "destructive", "unknown"]); +const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i; +const SECRET_LIKE_REFERENCE = /(?:[?&](?:token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)=|(?:bearer|basic)\s+)/i; +const SECRET_LIKE_VALUE = /(?:ndc_edp(?:wb|rb)_[A-Za-z0-9_-]*|[?&](?:token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)=|(?:bearer|basic)\s+\S+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/i; +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const FOUNDRY_APPLICATION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const FOUNDRY_PAGE_ID = /^[a-z0-9][a-z0-9-]{0,79}$/; +const FOUNDRY_SLOT_ID = /^[A-Za-z0-9][A-Za-z0-9-]{0,79}$/; +const CONTRACT_VERSION = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; +const MAX_BATCH_SEQUENCE = 2_147_483_647; +const MAX_FACT_ATTRIBUTES_BYTES = 64 * 1024; + +const PROVIDER_MANIFEST_KEYS = new Set(["schemaVersion", "id", "providerId", "version", "ontology", "l2Template", "capabilities", "dataProductIds"]); +const CONNECTION_PROFILE_KEYS = new Set(["schemaVersion", "id", "providerId", "tenantId", "credentialRef", "scope"]); +const COLLECTION_PROFILE_KEYS = new Set(["schemaVersion", "id", "connectionId", "dataProductId", "mode", "schedule", "capabilityIds"]); +const DATA_PRODUCT_KEYS = new Set(["schemaVersion", "id", "version", "delivery", "semanticTypes", "fields", "access"]); +const FOUNDRY_BINDING_KEYS = new Set(["schemaVersion", "id", "dataProductId", "applicationId", "pageId", "templateId", "slotId", "semanticType"]); +const FOUNDRY_BINDING_UPSERT_KEYS = new Set(["schemaVersion", "applicationId", "pageId", "idempotencyKey", "binding"]); +const FOUNDRY_BINDING_UPSERT_BINDING_KEYS = new Set(["id", "dataProductId", "slotId", "semanticTypes", "fieldProjection"]); + +/** + * Provider-neutral, versioned description of an L2 connector template. + * + * This is a declarative package artifact, not a tenant connection or an + * executable adapter. It may catalogue write capabilities, but it never + * grants or transports them. Runtime API requests, secrets and connection + * scope remain outside this manifest. + */ +export function validateProviderManifest(value) { + const errors = baseErrors(value, "providerManifest"); + rejectUnknownKeys(value, PROVIDER_MANIFEST_KEYS, "providerManifest", errors); + requiredIdentifier(value?.id, "id", errors); + requiredIdentifier(value?.providerId, "providerId", errors); + requiredString(value?.version, "version", errors); + if (value?.version && !CONTRACT_VERSION.test(value.version)) { + errors.push("version_must_be_semver"); + } + + if (!isPlainObject(value?.ontology)) { + errors.push("ontology_must_be_object"); + } else { + rejectUnknownKeys(value.ontology, new Set(["packageId", "revision"]), "ontology", errors); + } + requiredIdentifier(value?.ontology?.packageId, "ontology.packageId", errors); + requiredIdentifier(value?.ontology?.revision, "ontology.revision", errors); + if (!isPlainObject(value?.l2Template)) { + errors.push("l2Template_must_be_object"); + } else { + rejectUnknownKeys(value.l2Template, new Set(["id", "version"]), "l2Template", errors); + } + requiredIdentifier(value?.l2Template?.id, "l2Template.id", errors); + requiredString(value?.l2Template?.version, "l2Template.version", errors); + if (value?.l2Template?.version && !CONTRACT_VERSION.test(value.l2Template.version)) { + errors.push("l2Template.version_must_be_semver"); + } + + if (!Array.isArray(value?.capabilities) || value.capabilities.length === 0) { + errors.push("capabilities_must_be_nonempty_array"); + } else { + value.capabilities.forEach((capability, index) => { + if (!isPlainObject(capability)) { + errors.push(`capabilities[${index}]_must_be_object`); + return; + } + rejectUnknownKeys(capability, new Set(["id", "classification"]), `capabilities[${index}]`, errors); + requiredIdentifier(capability?.id, `capabilities[${index}].id`, errors); + if (!CAPABILITY_CLASSIFICATIONS.has(capability?.classification)) { + errors.push(`capabilities[${index}].classification_invalid`); + } + }); + } + + if (!Array.isArray(value?.dataProductIds) || value.dataProductIds.length === 0) { + errors.push("dataProductIds_must_be_nonempty_array"); + } else { + value.dataProductIds.forEach((dataProductId, index) => { + requiredIdentifier(dataProductId, `dataProductIds[${index}]`, errors); + }); + } + + if (value?.tenantId !== undefined || value?.connectionId !== undefined || value?.credentialRef !== undefined) { + errors.push("manifest_must_not_contain_connection_runtime_state"); + } + if (value?.endpoint !== undefined || value?.url !== undefined || value?.host !== undefined) { + errors.push("manifest_must_not_contain_provider_transport"); + } + if (containsSecretLikeMaterial(value)) errors.push("manifest_must_not_contain_secret_material"); + + return result(errors); +} + +export function validateConnectionProfile(value) { + const errors = baseErrors(value, "connection"); + rejectUnknownKeys(value, CONNECTION_PROFILE_KEYS, "connection", errors); + requiredIdentifier(value?.id, "id", errors); + requiredIdentifier(value?.providerId, "providerId", errors); + requiredIdentifier(value?.tenantId, "tenantId", errors); + if (!isPlainObject(value?.credentialRef)) { + errors.push("credentialRef_must_be_object"); + } else { + rejectUnknownKeys(value.credentialRef, new Set(["owner", "reference"]), "credentialRef", errors); + } + requiredString(value?.credentialRef?.owner, "credentialRef.owner", errors); + requiredString(value?.credentialRef?.reference, "credentialRef.reference", errors); + + if (value?.credentialRef?.owner && value.credentialRef.owner !== "engine") { + errors.push("credentialRef.owner_must_be_engine"); + } + if (containsSecretLikeMaterial(value)) errors.push("profile_must_not_contain_secret_material"); + if (value?.scope !== undefined) { + if (!isPlainObject(value.scope)) { + errors.push("scope_must_be_object"); + } else { + rejectUnknownKeys(value.scope, new Set(["capabilityIds", "fieldPolicyId", "retentionPolicyId", "collectionProfileIds"]), "scope", errors); + validateOptionalIdentifierArray(value.scope.capabilityIds, "scope.capabilityIds", errors); + validateOptionalIdentifierArray(value.scope.collectionProfileIds, "scope.collectionProfileIds", errors); + if (value.scope.fieldPolicyId !== undefined) requiredIdentifier(value.scope.fieldPolicyId, "scope.fieldPolicyId", errors); + if (value.scope.retentionPolicyId !== undefined) requiredIdentifier(value.scope.retentionPolicyId, "scope.retentionPolicyId", errors); + } + } + + return result(errors); +} + +export function validateCollectionProfile(value) { + const errors = baseErrors(value, "collectionProfile"); + rejectUnknownKeys(value, COLLECTION_PROFILE_KEYS, "collectionProfile", errors); + requiredIdentifier(value?.id, "id", errors); + requiredIdentifier(value?.connectionId, "connectionId", errors); + requiredIdentifier(value?.dataProductId, "dataProductId", errors); + + if (!COLLECTION_MODES.has(value?.mode)) errors.push("mode_must_be_realtime_manual_weekly_or_history"); + if (!Array.isArray(value?.capabilityIds) || value.capabilityIds.length === 0) { + errors.push("capabilityIds_must_be_nonempty_array"); + } else { + value.capabilityIds.forEach((capabilityId, index) => requiredIdentifier(capabilityId, `capabilityIds[${index}]`, errors)); + } + + const intervalMs = value?.schedule?.intervalMs; + if (value?.schedule !== undefined) { + if (!isPlainObject(value.schedule)) { + errors.push("schedule_must_be_object"); + } else { + rejectUnknownKeys(value.schedule, new Set(["intervalMs"]), "schedule", errors); + } + } + if (value?.mode === "realtime") { + if (!Number.isInteger(intervalMs) || intervalMs < 1000) errors.push("realtime_schedule_intervalMs_must_be_integer_gte_1000"); + } else if (value?.mode === "manual") { + if (intervalMs !== undefined) errors.push("manual_profile_must_not_define_intervalMs"); + } + + if (containsSecretLikeMaterial(value)) errors.push("collectionProfile_must_not_contain_secret_material"); + + return result(errors); +} + +export function validateDataProduct(value) { + const errors = baseErrors(value, "dataProduct"); + rejectUnknownKeys(value, DATA_PRODUCT_KEYS, "dataProduct", errors); + requiredIdentifier(value?.id, "id", errors); + requiredString(value?.version, "version", errors); + + if (!isPlainObject(value?.delivery)) { + errors.push("delivery_must_be_object"); + } else { + rejectUnknownKeys(value.delivery, new Set(["mode"]), "delivery", errors); + } + if (!DELIVERY_MODES.has(value?.delivery?.mode)) errors.push("delivery.mode_must_be_snapshot_snapshot+patch_or_query"); + if (!Array.isArray(value?.semanticTypes) || value.semanticTypes.length === 0) { + errors.push("semanticTypes_must_be_nonempty_array"); + } else { + value.semanticTypes.forEach((semanticType, index) => requiredIdentifier(semanticType, `semanticTypes[${index}]`, errors)); + } + if (!Array.isArray(value?.fields) || value.fields.length === 0) { + errors.push("fields_must_be_nonempty_array"); + } else { + value.fields.forEach((field, index) => requiredIdentifier(field, `fields[${index}]`, errors)); + } + if (!isPlainObject(value?.access)) { + errors.push("access_must_be_object"); + } else { + rejectUnknownKeys(value.access, new Set(["audience"]), "access", errors); + } + if (value?.access?.audience !== "internal") errors.push("access.audience_must_be_internal"); + if (containsSecretLikeMaterial(value)) errors.push("dataProduct_must_not_contain_secret_material"); + + return result(errors); +} + +export function validateFoundryBinding(value) { + const errors = baseErrors(value, "foundryBinding"); + rejectUnknownKeys(value, FOUNDRY_BINDING_KEYS, "foundryBinding", errors); + requiredIdentifier(value?.id, "id", errors); + requiredIdentifier(value?.dataProductId, "dataProductId", errors); + if (typeof value?.applicationId !== "string" || !FOUNDRY_APPLICATION_ID.test(value.applicationId)) { + errors.push("applicationId_invalid"); + } + if (typeof value?.pageId !== "string" || !FOUNDRY_PAGE_ID.test(value.pageId)) errors.push("pageId_invalid"); + if (value?.templateId !== undefined) requiredIdentifier(value.templateId, "templateId", errors); + if (typeof value?.slotId !== "string" || !FOUNDRY_SLOT_ID.test(value.slotId)) errors.push("slotId_invalid"); + requiredIdentifier(value?.semanticType, "semanticType", errors); + + if (containsSecretLikeMaterial(value)) errors.push("binding_must_not_contain_secret_material"); + if (value?.providerId !== undefined || value?.credentialRef !== undefined || value?.endpoint !== undefined) { + errors.push("binding_must_reference_data_product_not_provider_transport"); + } + + return result(errors); +} + +/** + * Replay-safe control-plane command emitted by `NDC Foundry Binding`. + * + * This is deliberately separate from the declarative Foundry binding artifact + * above: the command carries an idempotency key and can express a safe + * semantic/field projection, while authorization is materialized exclusively + * from the opaque workload grant at the receiving service. + */ +export function validateFoundryBindingUpsert(value) { + const errors = []; + if (!isPlainObject(value)) return result(["foundryBindingUpsert_must_be_object"]); + if (value.schemaVersion !== FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); + rejectUnknownKeys(value, FOUNDRY_BINDING_UPSERT_KEYS, "foundryBindingUpsert", errors); + if (typeof value.applicationId !== "string" || !FOUNDRY_APPLICATION_ID.test(value.applicationId)) { + errors.push("applicationId_invalid"); + } + if (typeof value.pageId !== "string" || !FOUNDRY_PAGE_ID.test(value.pageId)) errors.push("pageId_invalid"); + requiredIdentifier(value.idempotencyKey, "idempotencyKey", errors); + + if (!isPlainObject(value.binding)) { + errors.push("binding_must_be_object"); + } else { + rejectUnknownKeys(value.binding, FOUNDRY_BINDING_UPSERT_BINDING_KEYS, "binding", errors); + requiredIdentifier(value.binding.id, "binding.id", errors); + requiredIdentifier(value.binding.dataProductId, "binding.dataProductId", errors); + if (typeof value.binding.slotId !== "string" || !FOUNDRY_SLOT_ID.test(value.binding.slotId)) { + errors.push("binding.slotId_invalid"); + } + validateRequiredUniqueIdentifierArray(value.binding.semanticTypes, "binding.semanticTypes", errors); + validateUniqueIdentifierArray(value.binding.fieldProjection, "binding.fieldProjection", errors); + } + + if (containsSecretLikeMaterial(value)) errors.push("binding_must_not_contain_secret_material"); + return result(errors); +} + +/** + * Provider-neutral batch written by an L2 connector to External Data Plane. + * The payload deliberately describes source facts rather than any provider + * field names, customer entities or renderer representation. + */ +export function validateIntakeBatch(value) { + const errors = baseErrors(value, "intakeBatch"); + rejectUnknownKeys(value, new Set(["schemaVersion", "source", "contract", "batch", "raw", "facts"]), "intakeBatch", errors); + rejectUnknownKeys(value?.source, new Set(["providerId", "tenantId", "connectionId"]), "source", errors); + rejectUnknownKeys(value?.contract, new Set(["dataProductId", "ontologyRevision", "version"]), "contract", errors); + rejectUnknownKeys(value?.batch, new Set(["runId", "sequence", "idempotencyKey", "receivedAt"]), "batch", errors); + requiredIdentifier(value?.source?.providerId, "source.providerId", errors); + requiredIdentifier(value?.source?.tenantId, "source.tenantId", errors); + requiredIdentifier(value?.source?.connectionId, "source.connectionId", errors); + requiredIdentifier(value?.contract?.dataProductId, "contract.dataProductId", errors); + requiredIdentifier(value?.contract?.ontologyRevision, "contract.ontologyRevision", errors); + requiredString(value?.contract?.version, "contract.version", errors); + if (value?.contract?.version && !CONTRACT_VERSION.test(value.contract.version)) { + errors.push("contract.version_must_be_semver"); + } + requiredIdentifier(value?.batch?.runId, "batch.runId", errors); + requiredIdentifier(value?.batch?.idempotencyKey, "batch.idempotencyKey", errors); + if (!Number.isInteger(value?.batch?.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) { + errors.push("batch.sequence_must_be_integer_0_to_2147483647"); + } + requiredIsoTimestamp(value?.batch?.receivedAt, "batch.receivedAt", errors); + + if (!Array.isArray(value?.facts) || value.facts.length === 0) { + errors.push("facts_must_be_nonempty_array"); + } else { + value.facts.forEach((fact, index) => validateFact(fact, `facts[${index}]`, errors)); + } + + if (value?.raw !== undefined) validateRawEnvelope(value.raw, errors); + if (containsSecretLikeMaterial(value)) errors.push("intake_must_not_contain_secret_material"); + + return result(errors); +} + +export function assertValid(validator, value) { + const validation = validator(value); + if (!validation.ok) throw new Error(`external_provider_contract_invalid:${validation.errors.join(",")}`); + return value; +} + +function baseErrors(value, label) { + const errors = []; + if (!isPlainObject(value)) return [`${label}_must_be_object`]; + if (value.schemaVersion !== EXTERNAL_PROVIDER_CONTRACT_VERSION) { + errors.push("schemaVersion_mismatch"); + } + return errors; +} + +function result(errors) { + const uniqueErrors = [...new Set(errors)]; + return Object.freeze({ ok: uniqueErrors.length === 0, errors: Object.freeze(uniqueErrors) }); +} + +function requiredString(value, path, errors) { + if (typeof value !== "string" || !value.trim()) errors.push(`${path}_required`); +} + +function requiredIdentifier(value, path, errors) { + if (typeof value !== "string" || !IDENTIFIER.test(value)) errors.push(`${path}_invalid`); +} + +function validateOptionalIdentifierArray(value, path, errors) { + if (value === undefined) return; + if (!Array.isArray(value)) { + errors.push(`${path}_must_be_array`); + return; + } + value.forEach((item, index) => requiredIdentifier(item, `${path}[${index}]`, errors)); +} + +function validateRequiredUniqueIdentifierArray(value, path, errors) { + if (!Array.isArray(value) || value.length === 0) { + errors.push(`${path}_must_be_nonempty_array`); + return; + } + validateUniqueIdentifierArray(value, path, errors); +} + +function validateUniqueIdentifierArray(value, path, errors) { + if (!Array.isArray(value)) { + errors.push(`${path}_must_be_array`); + return; + } + value.forEach((item, index) => requiredIdentifier(item, `${path}[${index}]`, errors)); + if (new Set(value).size !== value.length) errors.push(`${path}_must_not_contain_duplicates`); +} + +function requiredIsoTimestamp(value, path, errors) { + if (typeof value !== "string" || Number.isNaN(Date.parse(value))) errors.push(`${path}_invalid_timestamp`); +} + +function validateFact(value, path, errors) { + if (!isPlainObject(value)) { + errors.push(`${path}_must_be_object`); + return; + } + rejectUnknownKeys(value, new Set(["sourceId", "semanticType", "observedAt", "attributes", "geometry"]), path, errors); + requiredIdentifier(value.sourceId, `${path}.sourceId`, errors); + requiredIdentifier(value.semanticType, `${path}.semanticType`, errors); + requiredIsoTimestamp(value.observedAt, `${path}.observedAt`, errors); + if (value.attributes !== undefined) { + if (!isPlainObject(value.attributes)) { + errors.push(`${path}.attributes_must_be_object`); + } else if (serializedByteLength(value.attributes) > MAX_FACT_ATTRIBUTES_BYTES) { + errors.push(`${path}.attributes_size_exceeded`); + } + } + if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors); +} + +function validatePointGeometry(value, path, errors) { + if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) { + errors.push(`${path}_must_be_geojson_point`); + return; + } + rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors); + if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) { + errors.push(`${path}_coordinates_must_be_finite_numbers`); + return; + } + const [longitude, latitude] = value.coordinates; + if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`); + if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`); +} + +function validateRawEnvelope(value, errors) { + if (!isPlainObject(value)) { + errors.push("raw_must_be_object"); + return; + } + rejectUnknownKeys(value, new Set(["contentType", "payload", "hash", "ref", "retentionDays"]), "raw", errors); + requiredString(value.contentType, "raw.contentType", errors); + if (value.payload === undefined && value.ref === undefined) errors.push("raw_requires_payload_or_ref"); + if (value.payload !== undefined) errors.push("raw.inline_payload_not_supported"); + if (value.payload === undefined) requiredString(value.hash, "raw.hash", errors); + if (value.hash !== undefined) requiredString(value.hash, "raw.hash", errors); + if (value.ref !== undefined) { + requiredString(value.ref, "raw.ref", errors); + if (typeof value.ref === "string" && SECRET_LIKE_REFERENCE.test(value.ref)) { + errors.push("raw.ref_must_not_contain_secret_material"); + } + } + if (value.retentionDays !== undefined && (!Number.isInteger(value.retentionDays) || value.retentionDays < 1)) { + errors.push("raw.retentionDays_must_be_positive_integer"); + } +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function rejectUnknownKeys(value, allowedKeys, path, errors) { + if (!isPlainObject(value)) return; + for (const key of Object.keys(value)) { + if (!allowedKeys.has(key)) errors.push(`${path}.${key}_not_allowed`); + } +} + +function serializedByteLength(value) { + try { + return Buffer.byteLength(JSON.stringify(value)); + } catch { + return Number.POSITIVE_INFINITY; + } +} + +function containsSecretLikeMaterial(value) { + if (typeof value === "string") return SECRET_LIKE_VALUE.test(value); + if (Array.isArray(value)) return value.some(containsSecretLikeMaterial); + if (!isPlainObject(value)) return false; + return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeMaterial(child)); +} diff --git a/packages/external-provider-contract/test/contract.test.mjs b/packages/external-provider-contract/test/contract.test.mjs new file mode 100644 index 0000000..8e9ffec --- /dev/null +++ b/packages/external-provider-contract/test/contract.test.mjs @@ -0,0 +1,267 @@ +import assert from "node:assert/strict"; +import { + EXTERNAL_PROVIDER_CONTRACT_VERSION, + FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION, + assertValid, + validateCollectionProfile, + validateConnectionProfile, + validateDataProduct, + validateFoundryBinding, + validateFoundryBindingUpsert, + validateIntakeBatch, + validateProviderManifest, +} from "../src/index.mjs"; +import { geliosPositionsCurrentExample } from "../examples/gelios-positions-current.v1.mjs"; + +assert.equal(validateProviderManifest(geliosPositionsCurrentExample.providerManifest).ok, true); +assert.equal(validateConnectionProfile(geliosPositionsCurrentExample.connection).ok, true); +assert.equal(validateCollectionProfile(geliosPositionsCurrentExample.collectionProfile).ok, true); +assert.equal(validateDataProduct(geliosPositionsCurrentExample.dataProduct).ok, true); +assert.equal(validateFoundryBinding(geliosPositionsCurrentExample.foundryBinding).ok, true); + +const foundryBindingUpsert = { + schemaVersion: FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION, + applicationId: "11111111-1111-4111-8111-111111111111", + pageId: "map", + idempotencyKey: "foundry-binding-0123456789abcdef0123456789abcdef", + binding: { + id: "fleet-live-points", + dataProductId: "fleet.positions.current.v1", + slotId: "points", + semanticTypes: ["map.moving_object"], + fieldProjection: ["name", "speed", "course"], + }, +}; +assert.equal(validateFoundryBindingUpsert(foundryBindingUpsert).ok, true); +assert.equal(validateFoundryBindingUpsert({ + ...foundryBindingUpsert, + binding: { ...foundryBindingUpsert.binding, semanticTypes: ["map.moving_object", "map.moving_object"] }, +}).errors.includes("binding.semanticTypes_must_not_contain_duplicates"), true); +assert.equal(validateFoundryBindingUpsert({ + ...foundryBindingUpsert, + binding: { ...foundryBindingUpsert.binding, accessToken: "forbidden" }, +}).errors.includes("binding.accessToken_not_allowed"), true); +assert.equal(validateFoundryBindingUpsert({ + ...foundryBindingUpsert, + binding: { ...foundryBindingUpsert.binding, fieldProjection: ["ndc_edprb_forbidden-reader-token"] }, +}).errors.includes("binding_must_not_contain_secret_material"), true); + +const attributesWithSerializedSize = (size) => ({ + blob: "x".repeat(size - Buffer.byteLength(JSON.stringify({ blob: "" }))), +}); + +const neutralIntakeBatch = { + schemaVersion: EXTERNAL_PROVIDER_CONTRACT_VERSION, + source: { + providerId: "example-provider", + tenantId: "sample-tenant", + connectionId: "sample-connection", + }, + contract: { + dataProductId: "fleet.positions.current.v1", + ontologyRevision: "ontology.example-fleet.v1", + version: "1.0.0", + }, + batch: { + runId: "run-20260714-001", + sequence: 0, + idempotencyKey: "run-20260714-001.batch-0", + receivedAt: "2026-07-14T18:00:00.000Z", + }, + raw: { + contentType: "application/json", + hash: "sha256:example", + ref: "restricted://raw/run-20260714-001", + retentionDays: 14, + }, + facts: [{ + sourceId: "source-object-42", + semanticType: "map.moving_object", + observedAt: "2026-07-14T17:59:58.000Z", + geometry: { type: "Point", coordinates: [37.6173, 55.7558] }, + attributes: { status: "active" }, + }], +}; + +assert.equal(validateIntakeBatch(neutralIntakeBatch).ok, true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + raw: { contentType: "application/json", payload: { safe: "fixture" } }, +}).errors.includes("raw.inline_payload_not_supported"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + raw: { contentType: "application/json", payload: "unsafe-unstructured-inline-raw" }, +}).errors.includes("raw.inline_payload_not_supported"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + raw: { contentType: "application/json", payload: { providerAccessToken: "must-never-be-stored" } }, +}).errors.includes("intake_must_not_contain_secret_material"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], attributes: { authorization: "Bearer must-never-be-stored" } }], +}).errors.includes("intake_must_not_contain_secret_material"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], attributes: { metadata: "ndc_edpwb_abcdefghijklmnopqrstuvwxyz0123456789ABCDE" } }], +}).errors.includes("intake_must_not_contain_secret_material"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], attributes: { metadata: "ndc_edprb_abcdefghijklmnopqrstuvwxyz0123456789ABCDE" } }], +}).errors.includes("intake_must_not_contain_secret_material"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + raw: { contentType: "application/json", hash: "prefix ndc_edpwb_abcdefghijklmnopqrstuvwxyz0123456789ABCDE suffix", ref: "restricted://raw/fixture" }, +}).errors.includes("intake_must_not_contain_secret_material"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], attributes: { metadata: "Bearer opaque-secret-material" } }], +}).errors.includes("intake_must_not_contain_secret_material"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + raw: { contentType: "application/json", hash: "sha256:fixture", ref: "restricted://raw?access_token=must-never-be-stored" }, +}).errors.includes("raw.ref_must_not_contain_secret_material"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], sourceId: "" }], +}).errors.includes("facts[0].sourceId_invalid"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + batch: { ...neutralIntakeBatch.batch, sequence: 2_147_483_647 }, +}).ok, true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + batch: { ...neutralIntakeBatch.batch, sequence: 2_147_483_648 }, +}).errors.includes("batch.sequence_must_be_integer_0_to_2147483647"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], attributes: attributesWithSerializedSize(64 * 1024) }], +}).ok, true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], attributes: attributesWithSerializedSize((64 * 1024) + 1) }], +}).errors.includes("facts[0].attributes_size_exceeded"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], geometry: { type: "Point", coordinates: [180.0001, 55.7558] } }], +}).errors.includes("facts[0].geometry.longitude_out_of_range"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + facts: [{ ...neutralIntakeBatch.facts[0], geometry: { type: "Point", coordinates: [37.6173, -90.0001] } }], +}).errors.includes("facts[0].geometry.latitude_out_of_range"), true); +assert.equal(validateIntakeBatch({ + ...neutralIntakeBatch, + debug: true, +}).errors.includes("intakeBatch.debug_not_allowed"), true); + +assert.equal(validateConnectionProfile({ + ...geliosPositionsCurrentExample.connection, + apiToken: "must-never-be-here", +}).ok, false); +assert.equal(validateConnectionProfile({ + ...geliosPositionsCurrentExample.connection, + credentialRef: { + ...geliosPositionsCurrentExample.connection.credentialRef, + reference: "ndc_edpwb_not-allowed-even-in-a-non-secret-field", + }, +}).errors.includes("profile_must_not_contain_secret_material"), true); +assert.equal(validateConnectionProfile({ + ...geliosPositionsCurrentExample.connection, + credentialRef: { + ...geliosPositionsCurrentExample.connection.credentialRef, + reference: "opaque-reference?access_token=must-not-cross-the-boundary", + }, +}).errors.includes("profile_must_not_contain_secret_material"), true); +assert.equal(validateConnectionProfile({ + ...geliosPositionsCurrentExample.connection, + credentialRef: { ...geliosPositionsCurrentExample.connection.credentialRef, namespace: "unexpected" }, +}).errors.includes("credentialRef.namespace_not_allowed"), true); +assert.equal(validateConnectionProfile({ + ...geliosPositionsCurrentExample.connection, + scope: { ...geliosPositionsCurrentExample.connection.scope, providerSelector: "unexpected" }, +}).errors.includes("scope.providerSelector_not_allowed"), true); +assert.equal(validateCollectionProfile({ + ...geliosPositionsCurrentExample.collectionProfile, + mode: "manual", +}).errors.includes("manual_profile_must_not_define_intervalMs"), true); +assert.equal(validateCollectionProfile({ + ...geliosPositionsCurrentExample.collectionProfile, + schedule: { intervalMs: 3000, jitterMs: 100 }, +}).errors.includes("schedule.jitterMs_not_allowed"), true); +assert.equal(validateCollectionProfile({ + ...geliosPositionsCurrentExample.collectionProfile, + capabilityIds: ["ndc_edprb_forbidden-reader-token"], +}).errors.includes("collectionProfile_must_not_contain_secret_material"), true); +assert.equal(validateFoundryBinding({ + ...geliosPositionsCurrentExample.foundryBinding, + providerId: "gelios", +}).errors.includes("binding_must_reference_data_product_not_provider_transport"), true); +assert.equal(validateFoundryBinding({ + ...geliosPositionsCurrentExample.foundryBinding, + applicationId: undefined, +}).errors.includes("applicationId_invalid"), true); +assert.equal(validateFoundryBinding({ + ...geliosPositionsCurrentExample.foundryBinding, + pageId: undefined, +}).errors.includes("pageId_invalid"), true); +assert.equal(validateFoundryBinding({ + ...geliosPositionsCurrentExample.foundryBinding, + templateId: undefined, +}).ok, true); +assert.equal(validateFoundryBinding({ + ...geliosPositionsCurrentExample.foundryBinding, + rendererOptions: {}, +}).errors.includes("foundryBinding.rendererOptions_not_allowed"), true); +assert.equal(validateFoundryBinding({ + ...geliosPositionsCurrentExample.foundryBinding, + semanticType: "ndc_edprb_forbidden-reader-token", +}).errors.includes("binding_must_not_contain_secret_material"), true); + +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + tenantId: "must-not-live-in-provider-manifest", +}).errors.includes("manifest_must_not_contain_connection_runtime_state"), true); +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + apiToken: "must-never-be-here", +}).errors.includes("manifest_must_not_contain_secret_material"), true); +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + endpoint: "https://must-live-in-l2-template.invalid", +}).errors.includes("manifest_must_not_contain_provider_transport"), true); +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + capabilities: [{ id: "gelios.units.current.read", classification: "not-a-class" }], +}).errors.includes("capabilities[0].classification_invalid"), true); +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + metadata: {}, +}).errors.includes("providerManifest.metadata_not_allowed"), true); +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + ontology: { ...geliosPositionsCurrentExample.providerManifest.ontology, transport: "unexpected" }, +}).errors.includes("ontology.transport_not_allowed"), true); +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + capabilities: [{ id: "gelios.units.current.read", classification: "read", endpoint: "/units" }], +}).errors.includes("capabilities[0].endpoint_not_allowed"), true); +assert.equal(validateProviderManifest({ + ...geliosPositionsCurrentExample.providerManifest, + dataProductIds: ["ndc_edpwb_forbidden-writer-token"], +}).errors.includes("manifest_must_not_contain_secret_material"), true); + +assert.equal(validateDataProduct({ + ...geliosPositionsCurrentExample.dataProduct, + delivery: { mode: "snapshot+patch", transport: "sse" }, +}).errors.includes("delivery.transport_not_allowed"), true); + +assert.throws(() => assertValid(validateDataProduct, { + schemaVersion: EXTERNAL_PROVIDER_CONTRACT_VERSION, + id: "fleet.positions.current.v1", + version: "1.0.0", + delivery: { mode: "snapshot" }, + semanticTypes: [], + fields: [], + access: { audience: "public" }, +})); + +console.log("external-provider-contract: ok"); diff --git a/packages/external-provider-contract/test/data-product.test.mjs b/packages/external-provider-contract/test/data-product.test.mjs new file mode 100644 index 0000000..28e32bb --- /dev/null +++ b/packages/external-provider-contract/test/data-product.test.mjs @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { + DATA_PRODUCT_PATCH_SCHEMA_VERSION, + DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, + DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, + validateDataProductPatch, + validateDataProductPublish, + validateDataProductSnapshot, +} from "../src/index.mjs"; + +const fact = { + sourceId: "unit-42", + semanticType: "map.moving_object", + observedAt: "2026-07-15T10:00:00.000Z", + attributes: { status: "online" }, + geometry: { type: "Point", coordinates: [37.6173, 55.7558] }, +}; + +const attributesWithSerializedSize = (size) => ({ + blob: "x".repeat(size - Buffer.byteLength(JSON.stringify({ blob: "" }))), +}); + +const publish = { + schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, + batch: { runId: "execution-42", sequence: 0, idempotencyKey: "execution-42.node-01.chunk-0" }, + facts: [fact], +}; +assert.equal(validateDataProductPublish(publish).ok, true); +assert.equal(validateDataProductPublish({ + ...publish, + source: { tenantId: "forged" }, +}).errors.includes("publish.source_not_allowed"), true); +assert.equal(validateDataProductPublish({ + ...publish, + facts: [{ ...fact, attributes: { accessToken: "must-never-cross-the-boundary" } }], +}).errors.includes("publish_must_not_contain_secret_material"), true); +assert.equal(validateDataProductPublish({ + ...publish, + facts: [fact, { ...fact, observedAt: "2026-07-15T10:00:01.000Z" }], +}).errors.includes("facts_duplicate_entity_key"), true); +assert.equal(validateDataProductPublish({ + ...publish, + batch: { ...publish.batch, sequence: 2_147_483_647 }, +}).ok, true); +assert.equal(validateDataProductPublish({ + ...publish, + batch: { ...publish.batch, sequence: 2_147_483_648 }, +}).errors.includes("batch.sequence_must_be_integer_0_to_2147483647"), true); +assert.equal(validateDataProductPublish({ + ...publish, + facts: [{ ...fact, attributes: attributesWithSerializedSize(64 * 1024) }], +}).ok, true); +assert.equal(validateDataProductPublish({ + ...publish, + facts: [{ ...fact, attributes: attributesWithSerializedSize((64 * 1024) + 1) }], +}).errors.includes("facts[0].attributes_size_exceeded"), true); +assert.equal(validateDataProductPublish({ + ...publish, + facts: [{ ...fact, attributes: attributesWithSerializedSize((64 * 1024) + 1) }], +}, { maxAttributesBytes: 1024 * 1024 }).errors.includes("facts[0].attributes_size_exceeded"), true); +assert.equal(validateDataProductPublish({ + ...publish, + facts: [{ ...fact, geometry: { type: "Point", coordinates: [-180.0001, 55.7558] } }], +}).errors.includes("facts[0].geometry.longitude_out_of_range"), true); +assert.equal(validateDataProductPublish({ + ...publish, + facts: [{ ...fact, geometry: { type: "Point", coordinates: [37.6173, 90.0001] } }], +}).errors.includes("facts[0].geometry.latitude_out_of_range"), true); + +const canonicalFact = { ...fact, receivedAt: "2026-07-15T10:00:01.000Z" }; +const snapshot = { + schemaVersion: DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, + dataProduct: { id: "fleet.positions.current.v1", version: "1.0.0" }, + generatedAt: "2026-07-15T10:00:01.000Z", + cursor: "12", + facts: [canonicalFact], +}; +assert.equal(validateDataProductSnapshot(snapshot).ok, true); +assert.equal(validateDataProductSnapshot({ ...snapshot, transport: "sse" }).errors.includes("snapshot.transport_not_allowed"), true); +assert.equal(validateDataProductSnapshot({ + ...snapshot, + facts: [{ ...canonicalFact, attributes: attributesWithSerializedSize((64 * 1024) + 1) }], +}).errors.includes("facts[0].attributes_size_exceeded"), true); +assert.equal(validateDataProductSnapshot({ + ...snapshot, + facts: [{ ...canonicalFact, attributes: { status: "ndc_edprb_forbidden-reader-token" } }], +}).errors.includes("snapshot_must_not_contain_secret_material"), true); + +const patch = { + schemaVersion: DATA_PRODUCT_PATCH_SCHEMA_VERSION, + dataProduct: { id: "fleet.positions.current.v1", version: "1.0.0" }, + cursor: "13", + previousCursor: "12", + emittedAt: "2026-07-15T10:00:02.000Z", + operations: [{ op: "upsert", fact: canonicalFact }], +}; +assert.equal(validateDataProductPatch(patch).ok, true); +assert.equal(validateDataProductPatch({ + ...patch, + operations: [{ op: "delete", fact: canonicalFact }], +}).errors.includes("operations[0].op_must_be_upsert"), true); +assert.equal(validateDataProductPatch({ + ...patch, + operations: [{ op: "upsert", fact: { ...canonicalFact, attributes: attributesWithSerializedSize((64 * 1024) + 1) } }], +}).errors.includes("operations[0].fact.attributes_size_exceeded"), true); +assert.equal(validateDataProductPatch({ + ...patch, + operations: [{ op: "upsert", fact: { ...canonicalFact, attributes: { status: "ndc_edpwb_forbidden-writer-token" } } }], +}).errors.includes("patch_must_not_contain_secret_material"), true); + +console.log("data-product-contract: ok"); diff --git a/packages/external-provider-contract/test/engine-credential-sink.test.mjs b/packages/external-provider-contract/test/engine-credential-sink.test.mjs new file mode 100644 index 0000000..9d933c6 --- /dev/null +++ b/packages/external-provider-contract/test/engine-credential-sink.test.mjs @@ -0,0 +1,242 @@ +import assert from "node:assert/strict"; +import { generateKeyPairSync, sign } from "node:crypto"; + +import { + ENGINE_CREDENTIAL_SINK_AUDIT_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_PROVISION_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_RECEIPT_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_ROLLBACK_RECEIPT_SCHEMA_VERSION, + ENGINE_CREDENTIAL_SINK_ROLLBACK_SCHEMA_VERSION, + computeEngineCredentialCapabilityDigest, + computeEngineCredentialSinkPolicyHash, + computeEngineCredentialSinkReceiptHash, + engineCredentialSinkAuditTargets, + validateEngineCredentialSinkAudit, + validateEngineCredentialSinkProvision, + validateEngineCredentialSinkReceipt, + validateEngineCredentialSinkRollback, + validateEngineCredentialSinkRollbackReceipt, +} from "../src/index.mjs"; + +const hash = (character) => `sha256:${character.repeat(64)}`; +const issuer = { serviceId: "platform.external-data-plane", keyId: "edp-issuer-20260715-001" }; +const { publicKey: issuerPublicKey, privateKey: issuerPrivateKey } = generateKeyPairSync("ed25519"); +const issuerPublicKeys = { [`${issuer.serviceId}:${issuer.keyId}`]: issuerPublicKey }; +const target = (nodeId, nodeType, credentialType) => ({ + workflowId: "WCb62yGL8v", + workflowRevision: "revision-20260715-001", + nodeId, + nodeType, + credentialType, +}); + +const request = { + schemaVersion: ENGINE_CREDENTIAL_SINK_PROVISION_SCHEMA_VERSION, + transaction: { + id: "credential-transaction-20260715-001", + idempotencyKey: "credential-transaction-20260715-001", + requestedAt: "2026-07-15T18:00:00.000Z", + requestExpiresAt: "2026-07-15T18:10:00.000Z", + policyHash: hash("0"), + failureMode: "rollback-all", + issuer, + attestation: { algorithm: "Ed25519", signature: "A".repeat(86) }, + }, + bindings: [ + { + bindingId: "positions-writer", + capabilityType: "external-data-plane.writer", + grantId: "writer-grant-001", + target: target( + "publish-node-001", + "n8n-nodes-ndc.ndcDataProductPublish", + "ndcDataProductWriterApi", + ), + expiresAt: "2026-08-15T18:00:00.000Z", + policyHash: hash("1"), + material: { format: "opaque-bearer", value: `ndc_edpwb_${"A".repeat(43)}` }, + }, + { + bindingId: "positions-reader", + capabilityType: "external-data-plane.reader", + grantId: "reader-grant-001", + target: target( + "read-node-001", + "n8n-nodes-ndc.ndcDataProductRead", + "ndcDataProductReaderApi", + ), + expiresAt: "2026-08-15T18:00:00.000Z", + policyHash: hash("2"), + material: { format: "opaque-bearer", value: `ndc_edprb_${"B".repeat(43)}` }, + }, + { + bindingId: "positions-foundry", + capabilityType: "foundry.binding", + grantId: "foundry-grant-001", + target: target( + "foundry-node-001", + "n8n-nodes-ndc.ndcFoundryBinding", + "ndcFoundryBindingApi", + ), + expiresAt: "2026-08-15T18:00:00.000Z", + policyHash: hash("3"), + material: { format: "opaque-bearer", value: `ndc_fndbg_${"C".repeat(43)}` }, + }, + ], +}; +for (const binding of request.bindings) { + binding.capabilityDigest = computeEngineCredentialCapabilityDigest(binding.material.value); +} +attest(request); +const provisionValidationNow = "2026-07-15T18:00:01.000Z"; + +assert.equal(validateEngineCredentialSinkProvision(request, { now: provisionValidationNow, issuerPublicKeys }).ok, true); +assert.equal(validateEngineCredentialSinkProvision(request, { + now: provisionValidationNow, +}).errors.includes("transaction.issuer_public_key_required"), true); +assert.equal(validateEngineCredentialSinkProvision({ + ...request, + transaction: { ...request.transaction, policyHash: hash("f") }, +}, { now: provisionValidationNow, issuerPublicKeys }).errors.includes("transaction.policyHash_mismatch"), true); +assert.equal(validateEngineCredentialSinkProvision({ + ...request, + bindings: [{ + ...request.bindings[0], + target: { ...request.bindings[0].target, nodeType: "n8n-nodes-ndc.ndcDataProductRead" }, + }], +}, { now: provisionValidationNow, issuerPublicKeys }).errors.includes("bindings[0].target.nodeType_capability_mismatch"), true); +assert.equal(validateEngineCredentialSinkProvision({ + ...request, + bindings: [{ + ...request.bindings[0], + material: { format: "opaque-bearer", value: request.bindings[1].material.value }, + }], +}, { now: provisionValidationNow, issuerPublicKeys }).errors.includes("bindings[0].material.value_invalid_for_capability"), true); + +const swappedCapabilityRequest = structuredClone(request); +swappedCapabilityRequest.bindings[0].material.value = `ndc_edpwb_${"D".repeat(43)}`; +assert.equal(validateEngineCredentialSinkProvision(swappedCapabilityRequest, { + now: provisionValidationNow, + issuerPublicKeys, +}).errors.includes("bindings[0].capabilityDigest_material_mismatch"), true); + +const resignedByAttackerRequest = structuredClone(swappedCapabilityRequest); +resignedByAttackerRequest.bindings[0].capabilityDigest = computeEngineCredentialCapabilityDigest( + resignedByAttackerRequest.bindings[0].material.value, +); +resignedByAttackerRequest.transaction.policyHash = computeEngineCredentialSinkPolicyHash(resignedByAttackerRequest); +assert.equal(validateEngineCredentialSinkProvision(resignedByAttackerRequest, { + now: provisionValidationNow, + issuerPublicKeys, +}).errors.includes("transaction.attestation_invalid"), true); + +const staleRequest = structuredClone(request); +staleRequest.transaction.requestedAt = "2020-01-01T00:00:00.000Z"; +staleRequest.transaction.requestExpiresAt = "2020-01-01T00:10:00.000Z"; +attest(staleRequest); +assert.equal(validateEngineCredentialSinkProvision(staleRequest, { now: provisionValidationNow, issuerPublicKeys }).errors.includes("request_expired"), true); + +const futureRequest = structuredClone(request); +futureRequest.transaction.requestedAt = "2026-07-15T18:02:00.000Z"; +futureRequest.transaction.requestExpiresAt = "2026-07-15T18:12:00.000Z"; +attest(futureRequest); +assert.equal(validateEngineCredentialSinkProvision(futureRequest, { now: provisionValidationNow, issuerPublicKeys }).errors.includes("requestedAt_exceeds_clock_skew"), true); + +const credentials = request.bindings.map((binding, index) => ({ + bindingId: binding.bindingId, + capabilityType: binding.capabilityType, + grantId: binding.grantId, + target: binding.target, + credentialRef: `engcred_${index + 1}_opaque_reference`, + expiresAt: binding.expiresAt, + policyHash: binding.policyHash, + capabilityDigest: binding.capabilityDigest, + disposition: "created", +})); +const receipt = { + schemaVersion: ENGINE_CREDENTIAL_SINK_RECEIPT_SCHEMA_VERSION, + transactionId: request.transaction.id, + idempotencyKey: request.transaction.idempotencyKey, + outcome: "committed", + policyHash: request.transaction.policyHash, + processedAt: "2026-07-15T18:00:02.000Z", + credentials, + rollback: { status: "not-required" }, +}; +assert.equal(validateEngineCredentialSinkReceipt(receipt, { request, issuerPublicKeys }).ok, true); +assert.equal(validateEngineCredentialSinkReceipt({ + ...receipt, + credentials: [{ ...credentials[0], target: { ...credentials[0].target, nodeId: "wrong-node" } }, ...credentials.slice(1)], +}, { request, issuerPublicKeys }).errors.includes("credentials_request_target_mismatch"), true); +assert.equal(validateEngineCredentialSinkReceipt({ + ...receipt, + capability: request.bindings[0].material.value, +}).errors.includes("credentialSinkReceipt.capability_not_allowed"), true); +assert.equal(validateEngineCredentialSinkReceipt({ + ...receipt, + outcome: "rolled-back", + credentials: credentials.slice(0, 1), + rollback: { status: "complete", completedAt: "2026-07-15T18:00:02.000Z" }, + errorCode: "engine_binding_failed", +}).errors.includes("noncommitted_credentials_must_be_empty"), true); + +const committedReceiptHash = computeEngineCredentialSinkReceiptHash(receipt); +const rollbackRequest = { + schemaVersion: ENGINE_CREDENTIAL_SINK_ROLLBACK_SCHEMA_VERSION, + rollback: { + id: "credential-rollback-20260715-001", + idempotencyKey: "credential-rollback-20260715-001", + transactionId: request.transaction.id, + requestedAt: "2026-07-15T18:05:00.000Z", + requestExpiresAt: "2026-07-15T18:10:00.000Z", + policyHash: request.transaction.policyHash, + committedReceiptHash, + reasonCode: "operator_requested", + }, +}; +assert.equal(validateEngineCredentialSinkRollback(rollbackRequest, { now: "2026-07-15T18:05:01.000Z" }).ok, true); +const staleRollbackRequest = structuredClone(rollbackRequest); +staleRollbackRequest.rollback.requestedAt = "2020-01-01T00:00:00.000Z"; +staleRollbackRequest.rollback.requestExpiresAt = "2020-01-01T00:10:00.000Z"; +assert.equal(validateEngineCredentialSinkRollback(staleRollbackRequest, { now: "2026-07-15T18:05:01.000Z" }).errors.includes("request_expired"), true); +const rollbackReceipt = { + schemaVersion: ENGINE_CREDENTIAL_SINK_ROLLBACK_RECEIPT_SCHEMA_VERSION, + rollbackId: rollbackRequest.rollback.id, + transactionId: rollbackRequest.rollback.transactionId, + outcome: "rolled-back", + policyHash: rollbackRequest.rollback.policyHash, + committedReceiptHash, + processedAt: "2026-07-15T18:05:01.000Z", +}; +assert.equal(validateEngineCredentialSinkRollbackReceipt(rollbackReceipt, { request: rollbackRequest }).ok, true); + +const audit = { + schemaVersion: ENGINE_CREDENTIAL_SINK_AUDIT_SCHEMA_VERSION, + eventId: "credential-audit-20260715-001", + transactionId: request.transaction.id, + operationId: request.transaction.id, + operation: "provision", + outcome: "committed", + occurredAt: receipt.processedAt, + policyHash: request.transaction.policyHash, + principal: { serviceId: "platform.credential-provisioner", fingerprint: hash("a") }, + targets: engineCredentialSinkAuditTargets(request, receipt), +}; +assert.equal(validateEngineCredentialSinkAudit(audit).ok, true); +assert.equal(JSON.stringify(audit).includes("ndc_edpwb_"), false); +assert.equal(JSON.stringify(audit).includes("engcred_1_opaque_reference"), false); +assert.equal(validateEngineCredentialSinkAudit({ + ...audit, + authorization: `Bearer ${request.bindings[0].material.value}`, +}).errors.includes("audit_must_not_contain_secret_material"), true); + +function attest(value) { + value.transaction.policyHash = computeEngineCredentialSinkPolicyHash(value); + value.transaction.attestation.signature = sign( + null, + Buffer.from(value.transaction.policyHash, "utf8"), + issuerPrivateKey, + ).toString("base64url"); +} + +console.log("external-provider credential sink contract: ok"); diff --git a/packages/external-provider-contract/test/engine-private-extension.test.mjs b/packages/external-provider-contract/test/engine-private-extension.test.mjs new file mode 100644 index 0000000..0231750 --- /dev/null +++ b/packages/external-provider-contract/test/engine-private-extension.test.mjs @@ -0,0 +1,370 @@ +import assert from "node:assert/strict"; + +import { + ENGINE_PRIVATE_EXTENSION_APPLY_RECEIPT_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_APPLY_REQUEST_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES, + ENGINE_PRIVATE_EXTENSION_INACTIVE_BASELINE, + ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY, + ENGINE_PRIVATE_EXTENSION_NODE_TYPES, + ENGINE_PRIVATE_EXTENSION_OPERATION_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME, + ENGINE_PRIVATE_EXTENSION_PLAN_REQUEST_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_PLAN_SCHEMA_VERSION, + ENGINE_PRIVATE_EXTENSION_READ_CAPABILITY, + ENGINE_PRIVATE_EXTENSION_STATUS_SCHEMA_VERSION, + authorizeEnginePrivateExtensionOperation, + computeEnginePrivateExtensionPlanHash, + validateEnginePrivateExtensionApplyReceipt, + validateEnginePrivateExtensionApplyRequest, + validateEnginePrivateExtensionOperation, + validateEnginePrivateExtensionPlan, + validateEnginePrivateExtensionPlanRequest, + validateEnginePrivateExtensionStatus, +} from "../src/index.mjs"; + +const digest = "03413c6f3c706a1f6a9597a1cf1730504e9719228d0c77fdfb93bbdcc57a4cf3"; +const release = Object.freeze({ + kind: "release", + packageName: ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME, + releaseId: "0.1.0-03413c6f3c706a1f", + packageSha256: digest, +}); +const inactive = Object.freeze({ + kind: "inactive-baseline", + packageName: ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME, + baselineId: ENGINE_PRIVATE_EXTENSION_INACTIVE_BASELINE, +}); +const manage = [ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY]; + +const activateRequest = { + schemaVersion: ENGINE_PRIVATE_EXTENSION_PLAN_REQUEST_SCHEMA_VERSION, + requestId: "extension-request-20260715-001", + idempotencyKey: "extension-request-20260715-001", + action: "activate", + requestedAt: "2026-07-15T18:00:00.000Z", + requestExpiresAt: "2026-07-15T18:10:00.000Z", + expectedCurrentGeneration: 0, + target: release, +}; + +assert.deepEqual( + validateEnginePrivateExtensionPlanRequest(activateRequest, { + now: "2026-07-15T18:00:01.000Z", + grantedCapabilities: manage, + }), + { ok: true, errors: [] }, +); +assert.equal( + validateEnginePrivateExtensionPlanRequest(activateRequest, { + now: "2026-07-15T18:00:01.000Z", + grantedCapabilities: ["engine.l2.deploy"], + }).errors.includes("engine_private_extension_capability_required"), + true, +); +assert.equal(authorizeEnginePrivateExtensionOperation("status", [ENGINE_PRIVATE_EXTENSION_READ_CAPABILITY]).ok, true); +assert.equal(authorizeEnginePrivateExtensionOperation("apply", [ENGINE_PRIVATE_EXTENSION_READ_CAPABILITY]).ok, false); + +const activatePlan = withPlanHash({ + schemaVersion: ENGINE_PRIVATE_EXTENSION_PLAN_SCHEMA_VERSION, + planId: "extension-plan-20260715-001", + planHash: hash("0"), + requestId: activateRequest.requestId, + idempotencyKey: activateRequest.idempotencyKey, + action: "activate", + createdAt: "2026-07-15T18:00:01.000Z", + expiresAt: "2026-07-15T18:09:00.000Z", + singleUse: true, + requiredCapability: ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY, + expectedCurrentGeneration: 0, + nextGeneration: 1, + currentState: inactive, + targetState: release, + recoveryState: inactive, + actions: [ + "verify_staged_immutable_release", + "prepare_sealed_package_tree", + "verify_community_package_loader_policy", + "quiesce_deploy_run_and_drain_queue", + "record_recovery_state", + "atomic_switch_current", + "force_recreate_main_workers_webhooks_as_version_barrier", + "verify_exact_runtime_acceptance", + "commit_active_state", + "resume_deploy_run", + ], + transition: transition(), + acceptance: acceptanceSpec(release), + failurePolicy: failurePolicy(), +}); + +assert.deepEqual(validateEnginePrivateExtensionPlan(activatePlan, { request: activateRequest }), { ok: true, errors: [] }); +assert.equal( + validateEnginePrivateExtensionPlan({ ...activatePlan, nextGeneration: 2 }).errors.includes("nextGeneration_must_increment_current_generation"), + true, +); +assert.equal( + validateEnginePrivateExtensionPlan({ ...activatePlan, transition: { ...transition(), runtimeAction: "restart" } }) + .errors.includes("transition_policy_mismatch"), + true, +); +assert.equal( + validateEnginePrivateExtensionPlan({ + ...activatePlan, + transition: { ...transition(), loaderMode: "custom-extension", loaderPath: "N8N_CUSTOM_EXTENSIONS" }, + }).errors.includes("transition_policy_mismatch"), + true, +); +assert.equal( + validateEnginePrivateExtensionPlan({ + ...activatePlan, + transition: { ...transition(), hotReload: true }, + }).errors.includes("transition_policy_mismatch"), + true, +); +assert.equal( + validateEnginePrivateExtensionPlan({ + ...activatePlan, + transition: { + ...transition(), + loaderEnvironment: { ...transition().loaderEnvironment, N8N_REINSTALL_MISSING_PACKAGES: "true" }, + }, + }).errors.includes("transition_policy_mismatch"), + true, +); + +const applyRequest = { + schemaVersion: ENGINE_PRIVATE_EXTENSION_APPLY_REQUEST_SCHEMA_VERSION, + planId: activatePlan.planId, + planHash: activatePlan.planHash, + idempotencyKey: activatePlan.idempotencyKey, + confirmedAt: "2026-07-15T18:00:02.000Z", +}; +assert.deepEqual(validateEnginePrivateExtensionApplyRequest(applyRequest, { + plan: activatePlan, + now: "2026-07-15T18:00:02.000Z", + grantedCapabilities: manage, +}), { ok: true, errors: [] }); +assert.equal(validateEnginePrivateExtensionApplyRequest(applyRequest, { + plan: activatePlan, + now: "2026-07-15T18:10:00.000Z", + grantedCapabilities: manage, +}).errors.includes("plan_expired"), true); + +const receipt = { + schemaVersion: ENGINE_PRIVATE_EXTENSION_APPLY_RECEIPT_SCHEMA_VERSION, + operationId: "extension-operation-20260715-001", + planId: activatePlan.planId, + planHash: activatePlan.planHash, + action: "activate", + acceptedAt: "2026-07-15T18:00:02.100Z", + state: "queued", +}; +assert.deepEqual(validateEnginePrivateExtensionApplyReceipt(receipt, { plan: activatePlan }), { ok: true, errors: [] }); +assert.equal(validateEnginePrivateExtensionApplyReceipt({ ...receipt, state: "active" }) + .errors.includes("apply_receipt_state_must_be_queued"), true); + +const activeOperation = { + schemaVersion: ENGINE_PRIVATE_EXTENSION_OPERATION_SCHEMA_VERSION, + operationId: receipt.operationId, + planId: activatePlan.planId, + planHash: activatePlan.planHash, + action: "activate", + state: "active", + outcome: "committed", + phase: "complete", + expectedCurrentGeneration: 0, + nextGeneration: 1, + targetState: release, + recoveryState: inactive, + effectiveState: release, + runtime: runtime(1, 2, 2), + acceptance: acceptanceReport(release, "accepted"), + updatedAt: "2026-07-15T18:00:32.000Z", +}; +assert.deepEqual(validateEnginePrivateExtensionOperation(activeOperation), { ok: true, errors: [] }); +const unexpectedSchema = structuredClone(activeOperation); +unexpectedSchema.acceptance.nodeTypes.observed.push("n8n-nodes-ndc.unreviewedNode"); +assert.equal(validateEnginePrivateExtensionOperation(unexpectedSchema).errors + .includes("acceptance.nodeTypes.observed_exact_set_required"), true); + +const automaticRollback = { + ...activeOperation, + state: "rolled-back", + outcome: "automatically-rolled-back", + phase: "complete", + effectiveState: inactive, + acceptance: acceptanceReport(inactive, "accepted"), + errorCode: "runtime_acceptance_failed", +}; +assert.deepEqual(validateEnginePrivateExtensionOperation(automaticRollback), { ok: true, errors: [] }); +assert.equal( + validateEnginePrivateExtensionOperation({ ...automaticRollback, errorCode: undefined }) + .errors.includes("errorCode_invalid"), + true, +); + +const activeStatus = { + schemaVersion: ENGINE_PRIVATE_EXTENSION_STATUS_SCHEMA_VERSION, + packageName: ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME, + generation: 1, + health: "ready", + currentState: release, + previousState: inactive, + runtime: runtime(1, 2, 2), + acceptance: acceptanceReport(release, "accepted"), + updatedAt: "2026-07-15T18:00:33.000Z", +}; +assert.deepEqual(validateEnginePrivateExtensionStatus(activeStatus), { ok: true, errors: [] }); + +const rollbackRequest = { + schemaVersion: ENGINE_PRIVATE_EXTENSION_PLAN_REQUEST_SCHEMA_VERSION, + requestId: "extension-rollback-request-20260715-001", + idempotencyKey: "extension-rollback-request-20260715-001", + action: "rollback", + requestedAt: "2026-07-15T18:05:00.000Z", + requestExpiresAt: "2026-07-15T18:15:00.000Z", + expectedCurrentGeneration: 1, + target: { kind: "previous-state", packageName: ENGINE_PRIVATE_EXTENSION_PACKAGE_NAME }, +}; +assert.deepEqual(validateEnginePrivateExtensionPlanRequest(rollbackRequest, { + now: "2026-07-15T18:05:01.000Z", + grantedCapabilities: manage, +}), { ok: true, errors: [] }); + +const rollbackPlan = withPlanHash({ + schemaVersion: ENGINE_PRIVATE_EXTENSION_PLAN_SCHEMA_VERSION, + planId: "extension-rollback-plan-20260715-001", + planHash: hash("0"), + requestId: rollbackRequest.requestId, + idempotencyKey: rollbackRequest.idempotencyKey, + action: "rollback", + createdAt: "2026-07-15T18:05:01.000Z", + expiresAt: "2026-07-15T18:14:00.000Z", + singleUse: true, + requiredCapability: ENGINE_PRIVATE_EXTENSION_MANAGE_CAPABILITY, + expectedCurrentGeneration: 1, + nextGeneration: 2, + currentState: release, + targetState: inactive, + recoveryState: release, + actions: [ + "verify_previous_activation_state", + "verify_community_package_loader_policy", + "quiesce_deploy_run_and_drain_queue", + "record_recovery_state", + "atomic_switch_current_to_previous", + "force_recreate_main_workers_webhooks_as_version_barrier", + "verify_exact_runtime_acceptance", + "commit_rolled_back_state", + "resume_deploy_run", + ], + transition: transition(), + acceptance: acceptanceSpec(inactive), + failurePolicy: failurePolicy(), +}); +assert.deepEqual(validateEnginePrivateExtensionPlan(rollbackPlan, { request: rollbackRequest }), { ok: true, errors: [] }); + +const explicitRollbackOperation = { + ...activeOperation, + operationId: "extension-operation-rollback-20260715-001", + planId: rollbackPlan.planId, + planHash: rollbackPlan.planHash, + action: "rollback", + state: "rolled-back", + outcome: "explicitly-rolled-back", + expectedCurrentGeneration: 1, + nextGeneration: 2, + targetState: inactive, + recoveryState: release, + effectiveState: inactive, + runtime: runtime(2, 2, 2), + acceptance: acceptanceReport(inactive, "accepted"), +}; +assert.deepEqual(validateEnginePrivateExtensionOperation(explicitRollbackOperation), { ok: true, errors: [] }); + +const quarantinedStatus = { + ...activeStatus, + health: "quarantined", + activeOperationId: "extension-operation-failed-20260715-001", + acceptance: acceptanceReport(release, "rejected"), + errorCode: "automatic_rollback_failed", +}; +assert.deepEqual(validateEnginePrivateExtensionStatus(quarantinedStatus), { ok: true, errors: [] }); + +const pathInjection = structuredClone(activateRequest); +pathInjection.target.releaseId = "../../runtime"; +assert.equal(validateEnginePrivateExtensionPlanRequest(pathInjection, { + now: "2026-07-15T18:00:01.000Z", + grantedCapabilities: manage, +}).errors.includes("target.releaseId_invalid"), true); + +const commandInjection = { ...activateRequest, command: "docker exec runtime npm install" }; +assert.equal(validateEnginePrivateExtensionPlanRequest(commandInjection, { + now: "2026-07-15T18:00:01.000Z", + grantedCapabilities: manage, +}).errors.includes("enginePrivateExtensionPlanRequest.command_not_allowed"), true); + +console.log("external-provider Engine private-extension contract: ok"); + +function transition() { + return { + mountMode: "read-only", + loaderMode: "community-package", + loaderPath: "/home/node/.n8n/nodes/node_modules/n8n-nodes-ndc", + loaderEnvironment: { + N8N_COMMUNITY_PACKAGES_ENABLED: "true", + N8N_COMMUNITY_PACKAGES_PREVENT_LOADING: "false", + N8N_REINSTALL_MISSING_PACKAGES: "false", + }, + quiesceMode: "block-deploy-run-and-drain-queue", + stateSwitch: "atomic-current-previous", + runtimeAction: "force-recreate", + scope: "main-workers-webhooks", + requireUniformGeneration: true, + hotReload: false, + preserveCredentials: true, + }; +} + +function failurePolicy() { + return { + mode: "automatic-rollback", + rollbackFailureOutcome: "quarantined", + preserveImmutableRelease: true, + preserveCredentials: true, + }; +} + +function acceptanceSpec(state) { + return { + mode: "exact", + nodeTypes: state.kind === "release" ? [...ENGINE_PRIVATE_EXTENSION_NODE_TYPES] : [], + credentialSchemas: state.kind === "release" ? [...ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES] : [], + requireUniformGeneration: true, + }; +} + +function acceptanceReport(state, status) { + const nodes = state.kind === "release" ? [...ENGINE_PRIVATE_EXTENSION_NODE_TYPES] : []; + const credentials = state.kind === "release" ? [...ENGINE_PRIVATE_EXTENSION_CREDENTIAL_TYPES] : []; + return { + state: status, + nodeTypes: { expected: nodes, observed: status === "pending" ? [] : nodes }, + credentialSchemas: { expected: credentials, observed: status === "pending" ? [] : credentials }, + uniformGeneration: status === "accepted", + }; +} + +function runtime(generation, expectedInstances, readyInstances) { + return { mode: "force-recreate", generation, expectedInstances, readyInstances }; +} + +function withPlanHash(plan) { + plan.planHash = computeEnginePrivateExtensionPlanHash(plan); + return plan; +} + +function hash(character) { + return "sha256:" + character.repeat(64); +} diff --git a/services/external-data-plane/Dockerfile b/services/external-data-plane/Dockerfile new file mode 100644 index 0000000..0670a12 --- /dev/null +++ b/services/external-data-plane/Dockerfile @@ -0,0 +1,32 @@ +FROM node:20-alpine AS deps + +WORKDIR /workspace + +COPY packages/external-provider-contract ./packages/external-provider-contract +COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/ + +WORKDIR /workspace/services/external-data-plane +RUN npm ci --omit=dev + +FROM node:20-alpine AS runner + +ENV NODE_ENV=production +ENV PORT=18106 + +WORKDIR /app + +COPY --from=deps /workspace/services/external-data-plane/package.json /workspace/services/external-data-plane/package-lock.json ./ +COPY --from=deps /workspace/services/external-data-plane/node_modules ./node_modules +COPY --from=deps /workspace/packages/external-provider-contract /packages/external-provider-contract +COPY services/external-data-plane/src ./src +COPY services/external-data-plane/definitions ./definitions + +# A dedicated numeric identity can read only the EDP provisioning secret mount; +# it is intentionally not the shared Node/Map Gateway uid/gid (1000). +RUN addgroup -S -g 11006 nodedc-edp && adduser -S -D -H -u 11006 -G nodedc-edp nodedc-edp + +USER 11006:11006 + +EXPOSE 18106 + +CMD ["node", "src/server.mjs"] diff --git a/services/external-data-plane/definitions/fleet.positions.current.v1.json b/services/external-data-plane/definitions/fleet.positions.current.v1.json new file mode 100644 index 0000000..1469cb5 --- /dev/null +++ b/services/external-data-plane/definitions/fleet.positions.current.v1.json @@ -0,0 +1,30 @@ +{ + "id": "fleet.positions.current.v1", + "version": "1.0.0", + "ontologyRevision": "ontology.map.moving_object.v1", + "deliveryMode": "snapshot+patch", + "semanticTypes": [ + "map.moving_object" + ], + "fields": [ + "course_degrees", + "display_name", + "elevation_meters", + "geometry", + "hdop", + "horizontal_accuracy_meters", + "object_kind", + "operational_status", + "position_source", + "position_valid", + "quality_flags", + "satellite_count", + "speed_kph" + ], + "history": { + "mode": "sampled", + "intervalMs": 60000, + "strategy": "latest-per-entity-per-bucket", + "retentionDays": 90 + } +} diff --git a/services/external-data-plane/package-lock.json b/services/external-data-plane/package-lock.json new file mode 100644 index 0000000..8f676e3 --- /dev/null +++ b/services/external-data-plane/package-lock.json @@ -0,0 +1,1017 @@ +{ + "name": "@nodedc/external-data-plane", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@nodedc/external-data-plane", + "version": "0.1.0", + "dependencies": { + "@nodedc/external-provider-contract": "file:../../packages/external-provider-contract", + "express": "^5.2.1", + "pg": "^8.18.0" + } + }, + "../../packages/external-provider-contract": { + "name": "@nodedc/external-provider-contract", + "version": "0.1.0" + }, + "node_modules/@nodedc/external-provider-contract": { + "resolved": "../../packages/external-provider-contract", + "link": true + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/services/external-data-plane/package.json b/services/external-data-plane/package.json new file mode 100644 index 0000000..6d6aa36 --- /dev/null +++ b/services/external-data-plane/package.json @@ -0,0 +1,18 @@ +{ + "name": "@nodedc/external-data-plane", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/server.mjs", + "dev": "node --watch src/server.mjs", + "check": "node --check src/server.mjs && node --check src/schema.mjs && node --check src/config.mjs && node --check src/intake-policy.mjs && node --check src/writer-binding.mjs && node --check src/reader-binding.mjs && node --check src/data-product-policy.mjs && node --check src/data-product-delivery.mjs && node --check src/definitions.mjs", + "test": "node test/config.test.mjs && node test/intake-policy.test.mjs && node test/writer-binding.test.mjs && node test/reader-binding.test.mjs && node test/data-product-policy.test.mjs && node test/definitions.test.mjs", + "test:integration": "node test/data-product-delivery.integration.test.mjs && node test/api.integration.test.mjs" + }, + "dependencies": { + "@nodedc/external-provider-contract": "file:../../packages/external-provider-contract", + "express": "^5.2.1", + "pg": "^8.18.0" + } +} diff --git a/services/external-data-plane/src/config.mjs b/services/external-data-plane/src/config.mjs new file mode 100644 index 0000000..7185cc4 --- /dev/null +++ b/services/external-data-plane/src/config.mjs @@ -0,0 +1,75 @@ +import { readFileSync } from "node:fs"; + +export function readConfig(env = process.env) { + const provisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED, false); + const config = { + port: integer(env.PORT, 18106, 1, 65535), + databaseUrl: required(env.EXTERNAL_DATA_PLANE_DATABASE_URL, "EXTERNAL_DATA_PLANE_DATABASE_URL"), + databasePoolSize: integer(env.EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE, 10, 1, 50), + internalAccessToken: optional(env.NODEDC_INTERNAL_ACCESS_TOKEN), + provisionerApiEnabled, + provisionerAccessToken: provisionerApiEnabled ? secretFile(env.EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE) : "", + rawRetentionDays: integer(env.EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS, 14, 1, 3650), + maxBatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES, 5 * 1024 * 1024, 1024, 50 * 1024 * 1024), + maxFactsPerPublish: integer(env.EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH, 5000, 1, 100_000), + maxAttributesBytesPerFact: integer(env.EXTERNAL_DATA_PLANE_MAX_ATTRIBUTES_BYTES_PER_FACT, 64 * 1024, 256, 1024 * 1024), + maxPatchOperations: integer(env.EXTERNAL_DATA_PLANE_MAX_PATCH_OPERATIONS, 500, 1, 5000), + maxPatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_PATCH_BYTES, 256 * 1024, 16 * 1024, 5 * 1024 * 1024), + patchRetentionMs: integer(env.EXTERNAL_DATA_PLANE_PATCH_RETENTION_MS, 60 * 60 * 1000, 60 * 1000, 7 * 24 * 60 * 60 * 1000), + receiptRetentionMs: integer(env.EXTERNAL_DATA_PLANE_RECEIPT_RETENTION_MS, 7 * 24 * 60 * 60 * 1000, 24 * 60 * 60 * 1000, 90 * 24 * 60 * 60 * 1000), + retentionDeleteLimit: integer(env.EXTERNAL_DATA_PLANE_RETENTION_DELETE_LIMIT, 10_000, 100, 100_000), + streamHeartbeatMs: integer(env.EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS, 20_000, 5_000, 60_000), + streamPollMs: integer(env.EXTERNAL_DATA_PLANE_STREAM_POLL_MS, 1_000, 250, 10_000), + maxReaderStreams: integer(env.EXTERNAL_DATA_PLANE_MAX_READER_STREAMS, 10, 1, 1000), + writerBindingMaxTtlDays: integer(env.EXTERNAL_DATA_PLANE_WRITER_BINDING_MAX_TTL_DAYS, 90, 1, 365), + maxFutureSkewSeconds: integer(env.EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS, 300, 0, 86400), + retentionSweepMs: integer(env.EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS, 60 * 60 * 1000, 60 * 1000, 24 * 60 * 60 * 1000), + legacyIntakeEnabled: boolean(env.EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED, false), + }; + if (config.internalAccessToken && config.provisionerAccessToken && config.internalAccessToken === config.provisionerAccessToken) { + throw new Error("provisioner_token_must_differ_from_internal_token"); + } + return config; +} + +function required(value, name) { + const normalized = optional(value); + if (!normalized) throw new Error(`${name}_required`); + return normalized; +} + +function optional(value) { + const normalized = String(value ?? "").trim(); + return normalized || ""; +} + +function integer(value, fallback, min, max) { + const candidate = optional(value); + if (!candidate) return fallback; + const number = Number.parseInt(candidate, 10); + if (!Number.isInteger(number) || number < min || number > max) throw new Error(`invalid_integer:${candidate}`); + return number; +} + +function boolean(value, fallback) { + const normalized = optional(value).toLowerCase(); + if (!normalized) return fallback; + if (normalized === "true") return true; + if (normalized === "false") return false; + throw new Error(`invalid_boolean:${normalized}`); +} + +function secretFile(value) { + const path = optional(value); + if (!path) return ""; + let secret = ""; + try { + secret = readFileSync(path, "utf8").trim(); + } catch { + throw new Error("external_data_plane_provisioner_token_file_unreadable"); + } + if (!/^[A-Za-z0-9_-]{48,256}$/.test(secret)) { + throw new Error("external_data_plane_provisioner_token_file_invalid"); + } + return secret; +} diff --git a/services/external-data-plane/src/data-product-delivery.mjs b/services/external-data-plane/src/data-product-delivery.mjs new file mode 100644 index 0000000..159669c --- /dev/null +++ b/services/external-data-plane/src/data-product-delivery.mjs @@ -0,0 +1,527 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + DATA_PRODUCT_PATCH_SCHEMA_VERSION, + DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, +} from "@nodedc/external-provider-contract"; + +export async function loadDataProductDefinition(db, dataProductId, { activeOnly = true } = {}) { + const result = await db.query( + `select id, version, ontology_revision as "ontologyRevision", + delivery_mode as "deliveryMode", semantic_types as "semanticTypes", + fields, history_policy as "historyPolicy", active, + created_at as "createdAt", updated_at as "updatedAt" + from external_data_plane_products + where id = $1 ${activeOnly ? "and active = true" : ""}`, + [dataProductId], + ); + return result.rows[0] || null; +} + +export async function persistDataProductDefinition(db, definition) { + const result = await db.query( + `insert into external_data_plane_products ( + id, version, ontology_revision, delivery_mode, semantic_types, fields, history_policy + ) values ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7::jsonb) + on conflict (id) do update set + active = true, + updated_at = now() + where external_data_plane_products.version = excluded.version + and external_data_plane_products.ontology_revision = excluded.ontology_revision + and external_data_plane_products.delivery_mode = excluded.delivery_mode + and external_data_plane_products.semantic_types = excluded.semantic_types + and external_data_plane_products.fields = excluded.fields + and external_data_plane_products.history_policy = excluded.history_policy + returning id, version, ontology_revision as "ontologyRevision", + delivery_mode as "deliveryMode", semantic_types as "semanticTypes", + fields, history_policy as "historyPolicy", active, + created_at as "createdAt", updated_at as "updatedAt"`, + [ + definition.id, + definition.version, + definition.ontologyRevision, + definition.deliveryMode, + JSON.stringify(definition.semanticTypes), + JSON.stringify(definition.fields), + JSON.stringify(definition.history), + ], + ); + if (!result.rowCount) throw deliveryError("data_product_version_is_immutable", 409); + return result.rows[0]; +} + +export async function persistDataProductPublish(pool, batch, definition, { + maxPatchOperations = 500, + maxPatchBytes = 256 * 1024, +} = {}) { + assertPublishMatchesDefinition(batch, definition); + const requestFingerprint = publishFingerprint(batch); + const client = await pool.connect(); + try { + await client.query("begin"); + const batchId = randomUUID(); + const insertedBatch = await client.query( + `insert into external_data_plane_batches ( + id, tenant_id, connection_id, provider_id, data_product_id, contract_version, + ontology_revision, run_id, sequence, idempotency_key, received_at, fact_count, + request_fingerprint + ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + on conflict (tenant_id, connection_id, provider_id, data_product_id, idempotency_key) + do nothing + returning id`, + [ + batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + batch.contract.dataProductId, batch.contract.version, batch.contract.ontologyRevision, + batch.batch.runId, batch.batch.sequence, batch.batch.idempotencyKey, + batch.batch.receivedAt, batch.facts.length, requestFingerprint, + ], + ); + if (!insertedBatch.rowCount) { + const existing = await client.query( + `select id as "batchId", fact_count as "publishedFactCount", + current_updated_count as "currentUpdatedCount", + history_inserted_count as "historyInsertedCount", + patch_operation_count as "patchOperationCount", + delivery_cursor::text as cursor, received_at as "acceptedAt", + request_fingerprint as "requestFingerprint" + from external_data_plane_batches + where tenant_id = $1 and connection_id = $2 and provider_id = $3 + and data_product_id = $4 and idempotency_key = $5`, + scopeValues(batch, batch.batch.idempotencyKey), + ); + if (!existing.rowCount || existing.rows[0].requestFingerprint !== requestFingerprint) { + throw deliveryError("idempotency_key_reused", 409); + } + await client.query("commit"); + return { ...normalizeReceipt(existing.rows[0]), idempotent: true }; + } + + const records = batch.facts.map((fact) => ({ + source_id: fact.sourceId, + semantic_type: fact.semanticType, + observed_at: fact.observedAt, + received_at: batch.batch.receivedAt, + attributes: fact.attributes || {}, + geometry: fact.geometry || null, + fingerprint: fingerprint(fact), + })); + + const updated = await client.query( + `insert into external_data_plane_current ( + tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, + observed_at, received_at, attributes, geometry, fingerprint, updated_at + ) + select $1, $2, $3, $4, item.source_id, item.semantic_type, + item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb), + case when item.geometry is null then null + else ST_SetSRID(ST_MakePoint( + (item.geometry->'coordinates'->>0)::double precision, + (item.geometry->'coordinates'->>1)::double precision + ), 4326)::geography end, + item.fingerprint, now() + from jsonb_to_recordset($5::jsonb) as item( + source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz, + attributes jsonb, geometry jsonb, fingerprint text + ) + on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type) + do update set + observed_at = excluded.observed_at, + received_at = excluded.received_at, + attributes = excluded.attributes, + geometry = excluded.geometry, + fingerprint = excluded.fingerprint, + updated_at = now() + where excluded.observed_at > external_data_plane_current.observed_at + or (excluded.observed_at = external_data_plane_current.observed_at + and excluded.fingerprint <> external_data_plane_current.fingerprint) + returning source_id as "sourceId", semantic_type as "semanticType", + observed_at as "observedAt", received_at as "receivedAt", attributes, + case when geometry is null then null + else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( + ST_X(geometry::geometry), ST_Y(geometry::geometry) + )) end as geometry, + fingerprint`, + [ + batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + batch.contract.dataProductId, JSON.stringify(records), + ], + ); + + const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records); + const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })); + const chunks = definition.deliveryMode === "snapshot+patch" + ? chunkOperations(operations, maxPatchOperations, maxPatchBytes) + : []; + const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch); + + await client.query( + `update external_data_plane_batches set + current_updated_count = $2, + history_inserted_count = $3, + patch_operation_count = $4, + delivery_cursor = $5 + where id = $1`, + [batchId, updated.rowCount, historyInsertedCount, operations.length, cursor], + ); + await client.query("commit"); + return normalizeReceipt({ + batchId, + idempotent: false, + publishedFactCount: batch.facts.length, + currentUpdatedCount: updated.rowCount, + historyInsertedCount, + patchOperationCount: operations.length, + cursor: String(cursor), + acceptedAt: batch.batch.receivedAt, + }); + } catch (error) { + await client.query("rollback"); + throw error; + } finally { + client.release(); + } +} + +export async function readDataProductSnapshot(pool, binding, definition, { limit = 5000 } = {}) { + const client = await pool.connect(); + try { + await client.query("begin isolation level repeatable read read only"); + const cursor = await currentCursor(client, binding, definition.id); + const rows = await client.query( + `select source_id as "sourceId", semantic_type as "semanticType", + observed_at as "observedAt", received_at as "receivedAt", attributes, + case when geometry is null then null + else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( + ST_X(geometry::geometry), ST_Y(geometry::geometry) + )) end as geometry + from external_data_plane_current + where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4 + order by source_id asc, semantic_type asc + limit $5`, + [binding.tenantId, binding.connectionId, binding.providerId, definition.id, limit + 1], + ); + if (rows.rowCount > limit) throw deliveryError("data_product_snapshot_limit_exceeded", 413); + await client.query("commit"); + return { + schemaVersion: DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, + dataProduct: { id: definition.id, version: definition.version }, + generatedAt: new Date().toISOString(), + cursor: String(cursor), + facts: rows.rows.map(publicFact), + }; + } catch (error) { + await client.query("rollback"); + throw error; + } finally { + client.release(); + } +} + +export async function readPatchEvents(pool, binding, definition, after, { limit = 100 } = {}) { + const client = await pool.connect(); + try { + await client.query("begin isolation level repeatable read read only"); + const current = await currentCursor(client, binding, definition.id); + if (after > current) throw deliveryError("resync_required", 409); + const floor = await patchFloor(client, binding, definition.id); + if (after < floor - 1n) throw deliveryError("resync_required", 409); + const rows = await client.query( + `select cursor::text, previous_cursor::text as "previousCursor", + operations, emitted_at as "emittedAt" + from external_data_plane_patch_outbox + where tenant_id = $1 and connection_id = $2 and provider_id = $3 + and data_product_id = $4 and cursor > $5 + order by cursor asc limit $6`, + [binding.tenantId, binding.connectionId, binding.providerId, definition.id, after.toString(), limit], + ); + await client.query("commit"); + return rows.rows.map((row) => ({ + schemaVersion: DATA_PRODUCT_PATCH_SCHEMA_VERSION, + dataProduct: { id: definition.id, version: definition.version }, + previousCursor: row.previousCursor, + cursor: row.cursor, + emittedAt: new Date(row.emittedAt).toISOString(), + operations: row.operations, + })); + } catch (error) { + await client.query("rollback"); + throw error; + } finally { + client.release(); + } +} + +export async function prunePatchOutbox(db, { retentionMs, limit = 10_000 }) { + const cutoff = new Date(Date.now() - retentionMs); + const result = await db.query( + `with expired as ( + select tenant_id, connection_id, provider_id, data_product_id, cursor + from external_data_plane_patch_outbox + where emitted_at < $1 + order by emitted_at asc + limit $2 + ) + delete from external_data_plane_patch_outbox as target + using expired + where target.tenant_id = expired.tenant_id + and target.connection_id = expired.connection_id + and target.provider_id = expired.provider_id + and target.data_product_id = expired.data_product_id + and target.cursor = expired.cursor`, + [cutoff, limit], + ); + return result.rowCount; +} + +export async function pruneDataProductHistory(db, { limit = 10_000 } = {}) { + const result = await db.query( + `with expired as ( + select history.tenant_id, history.connection_id, history.provider_id, + history.data_product_id, history.source_id, history.semantic_type, history.bucket_start + from external_data_plane_history as history + join external_data_plane_products as product on product.id = history.data_product_id + where history.bucket_start < now() - ( + greatest(1, coalesce((product.history_policy->>'retentionDays')::integer, 1)) * interval '1 day' + ) + order by history.bucket_start asc + limit $1 + ) + delete from external_data_plane_history as target + using expired + where target.tenant_id = expired.tenant_id + and target.connection_id = expired.connection_id + and target.provider_id = expired.provider_id + and target.data_product_id = expired.data_product_id + and target.source_id = expired.source_id + and target.semantic_type = expired.semantic_type + and target.bucket_start = expired.bucket_start`, + [limit], + ); + return result.rowCount; +} + +export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) { + const cutoff = new Date(Date.now() - retentionMs); + const result = await db.query( + `with expired as ( + select batch.id + from external_data_plane_batches as batch + where batch.created_at < $1 + and not exists (select 1 from external_data_plane_patch_outbox as patch where patch.batch_id = batch.id) + and not exists (select 1 from external_data_plane_raw_envelopes as raw where raw.batch_id = batch.id) + order by batch.created_at asc + limit $2 + ) + delete from external_data_plane_batches as target + using expired + where target.id = expired.id`, + [cutoff, limit], + ); + return result.rowCount; +} + +async function persistHistory(client, batch, policy, facts) { + if (!facts.length || policy?.mode === "none") return 0; + const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1; + const records = facts.map((fact) => ({ + ...fact, + bucket_start: new Date(Math.floor(new Date(fact.observed_at).getTime() / intervalMs) * intervalMs).toISOString(), + })); + const result = await client.query( + `insert into external_data_plane_history ( + tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, + bucket_start, observed_at, received_at, attributes, geometry, fingerprint, updated_at + ) + select $1, $2, $3, $4, item.source_id, item.semantic_type, + item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb), + case when item.geometry is null then null + else ST_SetSRID(ST_MakePoint( + (item.geometry->'coordinates'->>0)::double precision, + (item.geometry->'coordinates'->>1)::double precision + ), 4326)::geography end, + item.fingerprint, now() + from jsonb_to_recordset($5::jsonb) as item( + source_id text, semantic_type text, bucket_start timestamptz, + observed_at timestamptz, received_at timestamptz, + attributes jsonb, geometry jsonb, fingerprint text + ) + on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, bucket_start) + do update set + observed_at = excluded.observed_at, + received_at = excluded.received_at, + attributes = excluded.attributes, + geometry = excluded.geometry, + fingerprint = excluded.fingerprint, + updated_at = now() + where excluded.observed_at >= external_data_plane_history.observed_at`, + [ + batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + batch.contract.dataProductId, + JSON.stringify(records), + ], + ); + return result.rowCount; +} + +async function persistPatchChunks(client, batch, definition, batchId, chunks) { + const endCursorResult = await client.query( + `insert into external_data_plane_delivery_state ( + tenant_id, connection_id, provider_id, data_product_id, current_cursor + ) values ($1, $2, $3, $4, $5) + on conflict (tenant_id, connection_id, provider_id, data_product_id) + do update set + current_cursor = external_data_plane_delivery_state.current_cursor + excluded.current_cursor, + updated_at = now() + returning current_cursor`, + [ + batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + definition.id, chunks.length, + ], + ); + const end = BigInt(endCursorResult.rows[0].current_cursor); + const start = end - BigInt(chunks.length) + 1n; + for (let index = 0; index < chunks.length; index += 1) { + const cursor = start + BigInt(index); + await client.query( + `insert into external_data_plane_patch_outbox ( + tenant_id, connection_id, provider_id, data_product_id, + cursor, previous_cursor, batch_id, operations + ) values ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`, + [ + batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + definition.id, cursor.toString(), (cursor - 1n).toString(), batchId, + JSON.stringify(chunks[index]), + ], + ); + } + return end; +} + +async function currentCursor(db, scope, explicitProductId) { + const dataProductId = explicitProductId || scope.contract.dataProductId; + const result = await db.query( + `select current_cursor from external_data_plane_delivery_state + where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`, + [scope.tenantId || scope.source.tenantId, scope.connectionId || scope.source.connectionId, + scope.providerId || scope.source.providerId, dataProductId], + ); + return result.rowCount ? BigInt(result.rows[0].current_cursor) : 0n; +} + +async function patchFloor(db, binding, dataProductId) { + const result = await db.query( + `select min(cursor) as floor from external_data_plane_patch_outbox + where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`, + [binding.tenantId, binding.connectionId, binding.providerId, dataProductId], + ); + if (result.rows[0]?.floor !== null && result.rows[0]?.floor !== undefined) return BigInt(result.rows[0].floor); + const current = await currentCursor(db, binding, dataProductId); + return current + 1n; +} + +function scopeValues(batch, tail) { + return [ + batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + batch.contract.dataProductId, tail, + ]; +} + +function normalizeReceipt(value) { + return { + batchId: value.batchId, + idempotent: value.idempotent === true, + publishedFactCount: Number(value.publishedFactCount || 0), + currentUpdatedCount: Number(value.currentUpdatedCount || 0), + historyInsertedCount: Number(value.historyInsertedCount || 0), + patchOperationCount: Number(value.patchOperationCount || 0), + cursor: String(value.cursor || "0"), + acceptedAt: new Date(value.acceptedAt).toISOString(), + }; +} + +function publicFact(fact) { + const value = { + sourceId: fact.sourceId, + semanticType: fact.semanticType, + observedAt: new Date(fact.observedAt).toISOString(), + receivedAt: new Date(fact.receivedAt).toISOString(), + attributes: fact.attributes || {}, + }; + if (fact.geometry) value.geometry = fact.geometry; + return value; +} + +function chunkOperations(values, maxOperations, maxBytes) { + const result = []; + let current = []; + let currentBytes = 2; + for (const value of values) { + const serializedBytes = Buffer.byteLength(JSON.stringify(value)); + if (serializedBytes + 2 > maxBytes) throw deliveryError("data_product_patch_operation_size_exceeded", 413); + const separatorBytes = current.length ? 1 : 0; + if (current.length && (current.length >= maxOperations || currentBytes + separatorBytes + serializedBytes > maxBytes)) { + result.push(current); + current = []; + currentBytes = 2; + } + current.push(value); + currentBytes += (current.length > 1 ? 1 : 0) + serializedBytes; + } + if (current.length) result.push(current); + return result; +} + +function fingerprint(value) { + return createHash("sha256").update(stableJson(value)).digest("hex"); +} + +function publishFingerprint(batch) { + return fingerprint({ + schemaVersion: batch.schemaVersion, + source: batch.source, + contract: batch.contract, + batch: { + runId: batch.batch.runId, + sequence: batch.batch.sequence, + idempotencyKey: batch.batch.idempotencyKey, + }, + facts: batch.facts, + }); +} + +export function assertPublishMatchesDefinition(batch, definition) { + const semanticTypes = new Set(Array.isArray(definition?.semanticTypes) ? definition.semanticTypes : []); + const fields = new Set(Array.isArray(definition?.fields) ? definition.fields : []); + const entityKeys = new Set(); + for (const fact of batch?.facts || []) { + if (!semanticTypes.has(fact.semanticType)) throw deliveryError("data_product_semantic_type_forbidden", 422); + const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`; + if (entityKeys.has(entityKey)) throw deliveryError("data_product_duplicate_entity_key", 422); + entityKeys.add(entityKey); + for (const attribute of Object.keys(fact.attributes || {})) { + if (!fields.has(attribute) && !fields.has(`attributes.${attribute}`)) { + throw deliveryError("data_product_field_forbidden", 422); + } + } + if (fact.geometry !== undefined && !geometryDeclared(fields)) { + throw deliveryError("data_product_geometry_forbidden", 422); + } + } +} + +function geometryDeclared(fields) { + return fields.has("geometry") + || fields.has("coordinates") + || (fields.has("longitude") && fields.has("latitude")); +} + +function stableJson(value) { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function deliveryError(code, status) { + return Object.assign(new Error(code), { status, code }); +} diff --git a/services/external-data-plane/src/data-product-policy.mjs b/services/external-data-plane/src/data-product-policy.mjs new file mode 100644 index 0000000..928c0e0 --- /dev/null +++ b/services/external-data-plane/src/data-product-policy.mjs @@ -0,0 +1,118 @@ +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; +const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]); +const HISTORY_MODES = new Set(["none", "all", "sampled"]); +const DEFINITION_KEYS = new Set([ + "id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "history", +]); +const HISTORY_KEYS = new Set(["mode", "intervalMs", "retentionDays", "strategy"]); + +export function normalizeDataProductDefinition(value) { + if (!isPlainObject(value) || !hasOnlyKeys(value, DEFINITION_KEYS)) { + throw policyError("data_product_definition_invalid"); + } + const id = identifier(value.id); + const version = string(value.version); + const ontologyRevision = identifier(value.ontologyRevision); + const deliveryMode = string(value.deliveryMode); + const semanticTypes = identifierSet( + value.semanticTypes, + "data_product_definition_semantic_types_invalid", + "data_product_definition_semantic_types_duplicate", + ); + const fields = identifierSet( + value.fields, + "data_product_definition_fields_invalid", + "data_product_definition_fields_duplicate", + ); + if (!id || !SEMVER.test(version) || !ontologyRevision || !DELIVERY_MODES.has(deliveryMode)) { + throw policyError("data_product_definition_identity_invalid"); + } + if (!semanticTypes.length || !fields.length) throw policyError("data_product_definition_shape_invalid"); + + const history = normalizeHistoryPolicy(value.history); + return Object.freeze({ id, version, ontologyRevision, deliveryMode, semanticTypes, fields, history }); +} + +export function normalizeHistoryPolicy(value = { mode: "none" }) { + if (!isPlainObject(value) || !hasOnlyKeys(value, HISTORY_KEYS)) throw policyError("history_policy_invalid"); + const mode = string(value.mode); + if (!HISTORY_MODES.has(mode)) throw policyError("history_policy_mode_invalid"); + const retentionDays = integer(value.retentionDays, mode === "none" ? 1 : 90, 1, 3650); + if (mode === "none") { + if (value.intervalMs !== undefined || value.strategy !== undefined) throw policyError("history_policy_none_has_sampling_fields"); + return Object.freeze({ mode, retentionDays }); + } + if (mode === "all") { + if (value.intervalMs !== undefined || value.strategy !== undefined) throw policyError("history_policy_all_has_sampling_fields"); + return Object.freeze({ mode, retentionDays }); + } + + const intervalMs = integer(value.intervalMs, 60_000, 1_000, 24 * 60 * 60 * 1000); + const strategy = string(value.strategy || "latest-per-entity-per-bucket"); + if (strategy !== "latest-per-entity-per-bucket") throw policyError("history_policy_strategy_invalid"); + return Object.freeze({ mode, intervalMs, strategy, retentionDays }); +} + +export function safeDataProductDefinition(row) { + return { + id: row.id, + version: row.version, + ontologyRevision: row.ontologyRevision, + deliveryMode: row.deliveryMode, + semanticTypes: array(row.semanticTypes), + fields: array(row.fields), + history: isPlainObject(row.historyPolicy) ? row.historyPolicy : {}, + active: row.active === true, + createdAt: iso(row.createdAt), + updatedAt: iso(row.updatedAt), + }; +} + +function policyError(code) { + return Object.assign(new Error(code), { status: 400, code }); +} + +function identifier(value) { + const normalized = string(value); + return IDENTIFIER.test(normalized) ? normalized : ""; +} + +function identifierSet(value, invalidCode, duplicateCode) { + if (!Array.isArray(value)) throw policyError(invalidCode); + + const normalized = value.map((entry) => { + const result = identifier(entry); + if (!result) throw policyError(invalidCode); + return result; + }); + if (new Set(normalized).size !== normalized.length) throw policyError(duplicateCode); + + return Object.freeze(normalized.sort()); +} + +function integer(value, fallback, min, max) { + const number = value === undefined ? fallback : Number(value); + if (!Number.isInteger(number) || number < min || number > max) throw policyError("history_policy_number_invalid"); + return number; +} + +function string(value) { + return typeof value === "string" ? value.trim() : ""; +} + +function array(value) { + return Array.isArray(value) ? value : []; +} + +function iso(value) { + return value ? new Date(value).toISOString() : undefined; +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function hasOnlyKeys(value, allowed) { + return Object.keys(value).every((key) => allowed.has(key)); +} diff --git a/services/external-data-plane/src/definitions.mjs b/services/external-data-plane/src/definitions.mjs new file mode 100644 index 0000000..ec2654e --- /dev/null +++ b/services/external-data-plane/src/definitions.mjs @@ -0,0 +1,43 @@ +import { lstat, readFile, readdir } from "node:fs/promises"; +import { basename, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { normalizeDataProductDefinition } from "./data-product-policy.mjs"; +import { persistDataProductDefinition } from "./data-product-delivery.mjs"; + +const bundledDefinitionsDir = fileURLToPath(new URL("../definitions/", import.meta.url)); + +export async function loadDataProductDefinitions(directory = bundledDefinitionsDir) { + const root = resolve(directory); + const entries = (await readdir(root, { withFileTypes: true })) + .filter((entry) => entry.name.endsWith(".json")) + .sort((left, right) => left.name.localeCompare(right.name)); + const definitions = []; + for (const entry of entries) { + if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("data_product_definition_file_invalid"); + const path = join(root, entry.name); + const metadata = await lstat(path); + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size < 2 || metadata.size > 256 * 1024) { + throw new Error("data_product_definition_file_invalid"); + } + let value; + try { + value = JSON.parse(await readFile(path, "utf8")); + } catch { + throw new Error("data_product_definition_json_invalid"); + } + const definition = normalizeDataProductDefinition(value); + if (`${definition.id}.json` !== basename(path)) throw new Error("data_product_definition_filename_mismatch"); + definitions.push(definition); + } + if (new Set(definitions.map((definition) => definition.id)).size !== definitions.length) { + throw new Error("data_product_definition_duplicate"); + } + return Object.freeze(definitions); +} + +export async function reconcileDataProductDefinitions(db, directory = bundledDefinitionsDir) { + const definitions = await loadDataProductDefinitions(directory); + for (const definition of definitions) await persistDataProductDefinition(db, definition); + return definitions; +} diff --git a/services/external-data-plane/src/intake-policy.mjs b/services/external-data-plane/src/intake-policy.mjs new file mode 100644 index 0000000..6e17ecb --- /dev/null +++ b/services/external-data-plane/src/intake-policy.mjs @@ -0,0 +1,37 @@ +export function assertBatchTimeBounds(batch, { now = new Date(), maxFutureSkewSeconds = 300 } = {}) { + const acceptedAt = validDate(now, "intake_policy_clock_invalid"); + const maxFutureAt = new Date(acceptedAt.getTime() + Number(maxFutureSkewSeconds) * 1000); + if (!Number.isInteger(maxFutureSkewSeconds) || maxFutureSkewSeconds < 0) { + throw intakePolicyError("intake_policy_future_skew_invalid"); + } + + const receivedAt = validDate(batch?.batch?.receivedAt, "batch_received_at_invalid"); + if (receivedAt > maxFutureAt) throw intakePolicyError("batch_received_at_too_far_in_future"); + + for (const fact of batch?.facts || []) { + const observedAt = validDate(fact?.observedAt, "fact_observed_at_invalid"); + if (observedAt > maxFutureAt) throw intakePolicyError("fact_observed_at_too_far_in_future"); + } +} + +/** + * Retention is based on the server's acceptance time, never on provider or L2 + * timestamps supplied in a batch. + */ +export function rawRetentionExpiry({ now = new Date(), rawRetentionDays }) { + const acceptedAt = validDate(now, "intake_policy_clock_invalid"); + if (!Number.isInteger(rawRetentionDays) || rawRetentionDays < 1) { + throw intakePolicyError("raw_retention_days_invalid"); + } + return new Date(acceptedAt.getTime() + rawRetentionDays * 24 * 60 * 60 * 1000); +} + +export function intakePolicyError(code) { + return Object.assign(new Error(code), { status: 422, code }); +} + +function validDate(value, code) { + const parsed = value instanceof Date ? new Date(value.getTime()) : new Date(String(value ?? "")); + if (Number.isNaN(parsed.getTime())) throw intakePolicyError(code); + return parsed; +} diff --git a/services/external-data-plane/src/reader-binding.mjs b/services/external-data-plane/src/reader-binding.mjs new file mode 100644 index 0000000..7b614b8 --- /dev/null +++ b/services/external-data-plane/src/reader-binding.mjs @@ -0,0 +1,90 @@ +import { createHash, randomBytes } from "node:crypto"; + +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const TOKEN_PREFIX = "ndc_edprb_"; +const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i; +const REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]); +const SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]); + +export function createReaderToken() { + return `${TOKEN_PREFIX}${randomBytes(32).toString("base64url")}`; +} + +export function hashReaderToken(token) { + return createHash("sha256").update(String(token), "utf8").digest("hex"); +} + +export function normalizeReaderBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) { + if (!isPlainObject(value) || !isPlainObject(value.source)) throw readerError("reader_binding_request_invalid"); + if (containsSecretLikeKey(value)) throw readerError("reader_binding_request_secret_material_forbidden"); + if (!hasOnlyKeys(value, REQUEST_KEYS) || !hasOnlyKeys(value.source, SOURCE_KEYS)) { + throw readerError("reader_binding_request_fields_invalid"); + } + const tenantId = identifier(value.source.tenantId); + const connectionId = identifier(value.source.connectionId); + const providerId = identifier(value.source.providerId); + const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds); + if (!tenantId || !connectionId || !providerId || !allowedDataProductIds.length) { + throw readerError("reader_binding_scope_invalid"); + } + const expiresAt = new Date(String(value.expiresAt || "")); + const maxExpiresAt = new Date(now.getTime() + maxTtlDays * 24 * 60 * 60 * 1000); + if (Number.isNaN(expiresAt.getTime()) || expiresAt <= now || expiresAt > maxExpiresAt) { + throw readerError("reader_binding_expiry_invalid"); + } + return Object.freeze({ tenantId, connectionId, providerId, allowedDataProductIds, expiresAt: expiresAt.toISOString() }); +} + +export function assertReaderProduct(binding, dataProductId, now = new Date()) { + if (!isPlainObject(binding) || binding.active !== true || new Date(binding.expiresAt) <= now) { + throw readerError("reader_binding_inactive", 401); + } + const normalized = identifier(dataProductId); + if (!normalized || !uniqueIdentifiers(binding.allowedDataProductIds).includes(normalized)) { + throw readerError("reader_binding_data_product_forbidden", 403); + } + return normalized; +} + +export function safeReaderBinding(binding) { + return { + id: binding.id, + tenantId: binding.tenantId, + connectionId: binding.connectionId, + providerId: binding.providerId, + allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds), + active: binding.active === true, + expiresAt: new Date(binding.expiresAt).toISOString(), + createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined, + rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined, + revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined, + }; +} + +function readerError(code, status = 400) { + return Object.assign(new Error(code), { status, code }); +} + +function identifier(value) { + const normalized = typeof value === "string" ? value.trim() : ""; + return IDENTIFIER.test(normalized) ? normalized : ""; +} + +function uniqueIdentifiers(value) { + if (!Array.isArray(value)) return []; + return [...new Set(value.map(identifier).filter(Boolean))]; +} + +function containsSecretLikeKey(value) { + if (Array.isArray(value)) return value.some(containsSecretLikeKey); + if (!isPlainObject(value)) return false; + return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeKey(child)); +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function hasOnlyKeys(value, allowed) { + return Object.keys(value).every((key) => allowed.has(key)); +} diff --git a/services/external-data-plane/src/schema.mjs b/services/external-data-plane/src/schema.mjs new file mode 100644 index 0000000..adc880b --- /dev/null +++ b/services/external-data-plane/src/schema.mjs @@ -0,0 +1,206 @@ +export async function migrate(pool) { + await pool.query("create extension if not exists timescaledb"); + await pool.query("create extension if not exists postgis"); + + await pool.query(` + create table if not exists external_data_plane_products ( + id text primary key, + version text not null, + ontology_revision text not null, + delivery_mode text not null, + semantic_types jsonb not null, + fields jsonb not null, + history_policy jsonb not null, + active boolean not null default true, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + check (jsonb_typeof(semantic_types) = 'array' and jsonb_array_length(semantic_types) > 0), + check (jsonb_typeof(fields) = 'array' and jsonb_array_length(fields) > 0) + ) + `); + + await pool.query(` + create table if not exists external_data_plane_batches ( + id uuid primary key, + tenant_id text not null, + connection_id text not null, + provider_id text not null, + data_product_id text not null, + contract_version text not null, + ontology_revision text not null, + run_id text not null, + sequence integer not null, + idempotency_key text not null, + received_at timestamptz not null, + fact_count integer not null default 0, + inserted_fact_count integer not null default 0, + created_at timestamptz not null default now(), + unique (tenant_id, connection_id, provider_id, data_product_id, idempotency_key) + ) + `); + await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)"); + await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0"); + await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0"); + await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0"); + await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint"); + await pool.query("alter table external_data_plane_batches add column if not exists request_fingerprint text"); + + await pool.query(` + create table if not exists external_data_plane_raw_envelopes ( + id uuid primary key, + batch_id uuid not null unique references external_data_plane_batches(id) on delete cascade, + payload_hash text not null, + content_type text not null, + payload jsonb, + payload_ref text, + payload_bytes integer, + received_at timestamptz not null, + expires_at timestamptz not null, + check (payload is not null or payload_ref is not null) + ) + `); + await pool.query("create index if not exists external_data_plane_raw_expiry_idx on external_data_plane_raw_envelopes (expires_at)"); + + await pool.query(` + create table if not exists external_data_plane_writer_bindings ( + id uuid primary key, + token_hash text not null unique, + tenant_id text not null, + connection_id text not null, + provider_id text not null, + allowed_data_product_ids jsonb not null, + expires_at timestamptz not null, + active boolean not null default true, + created_at timestamptz not null default now(), + rotated_at timestamptz, + revoked_at timestamptz, + check ( + case when jsonb_typeof(allowed_data_product_ids) = 'array' + then jsonb_array_length(allowed_data_product_ids) > 0 + else false + end + ) + ) + `); + await pool.query("create index if not exists external_data_plane_writer_bindings_active_idx on external_data_plane_writer_bindings (active, expires_at)"); + + await pool.query(` + create table if not exists external_data_plane_reader_bindings ( + id uuid primary key, + token_hash text not null unique, + tenant_id text not null, + connection_id text not null, + provider_id text not null, + allowed_data_product_ids jsonb not null, + expires_at timestamptz not null, + active boolean not null default true, + created_at timestamptz not null default now(), + rotated_at timestamptz, + revoked_at timestamptz, + check ( + case when jsonb_typeof(allowed_data_product_ids) = 'array' + then jsonb_array_length(allowed_data_product_ids) > 0 + else false + end + ) + ) + `); + await pool.query("create index if not exists external_data_plane_reader_bindings_active_idx on external_data_plane_reader_bindings (active, expires_at)"); + + await pool.query(` + create table if not exists external_data_plane_facts ( + id uuid not null, + batch_id uuid not null references external_data_plane_batches(id) on delete cascade, + tenant_id text not null, + connection_id text not null, + provider_id text not null, + data_product_id text not null, + source_id text not null, + semantic_type text not null, + observed_at timestamptz not null, + received_at timestamptz not null, + attributes jsonb not null default '{}'::jsonb, + geometry geography(Point, 4326), + fingerprint text not null, + primary key (id, observed_at), + unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint) + ) + `); + await pool.query("select create_hypertable('external_data_plane_facts', 'observed_at', if_not_exists => true, migrate_data => true)"); + await pool.query("create index if not exists external_data_plane_facts_source_time_idx on external_data_plane_facts (tenant_id, connection_id, data_product_id, source_id, observed_at desc)"); + await pool.query("create index if not exists external_data_plane_facts_geometry_idx on external_data_plane_facts using gist (geometry)"); + + await pool.query(` + create table if not exists external_data_plane_current ( + tenant_id text not null, + connection_id text not null, + provider_id text not null, + data_product_id text not null, + source_id text not null, + semantic_type text not null, + observed_at timestamptz not null, + received_at timestamptz not null, + attributes jsonb not null default '{}'::jsonb, + geometry geography(Point, 4326), + fingerprint text not null, + updated_at timestamptz not null default now(), + primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type) + ) + `); + await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)"); + await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)"); + + await pool.query(` + create table if not exists external_data_plane_delivery_state ( + tenant_id text not null, + connection_id text not null, + provider_id text not null, + data_product_id text not null, + current_cursor bigint not null default 0, + updated_at timestamptz not null default now(), + primary key (tenant_id, connection_id, provider_id, data_product_id) + ) + `); + + await pool.query(` + create table if not exists external_data_plane_patch_outbox ( + tenant_id text not null, + connection_id text not null, + provider_id text not null, + data_product_id text not null, + cursor bigint not null, + previous_cursor bigint not null, + batch_id uuid not null references external_data_plane_batches(id) on delete cascade, + operations jsonb not null, + emitted_at timestamptz not null default now(), + primary key (tenant_id, connection_id, provider_id, data_product_id, cursor), + check (jsonb_typeof(operations) = 'array' and jsonb_array_length(operations) > 0) + ) + `); + await pool.query("create index if not exists external_data_plane_patch_outbox_retention_idx on external_data_plane_patch_outbox (emitted_at)"); + + await pool.query(` + create table if not exists external_data_plane_history ( + tenant_id text not null, + connection_id text not null, + provider_id text not null, + data_product_id text not null, + source_id text not null, + semantic_type text not null, + bucket_start timestamptz not null, + observed_at timestamptz not null, + received_at timestamptz not null, + attributes jsonb not null default '{}'::jsonb, + geometry geography(Point, 4326), + fingerprint text not null, + updated_at timestamptz not null default now(), + primary key ( + tenant_id, connection_id, provider_id, data_product_id, + source_id, semantic_type, bucket_start + ) + ) + `); + await pool.query("select create_hypertable('external_data_plane_history', 'bucket_start', if_not_exists => true, migrate_data => true)"); + await pool.query("create index if not exists external_data_plane_history_source_time_idx on external_data_plane_history (tenant_id, connection_id, data_product_id, source_id, bucket_start desc)"); + await pool.query("create index if not exists external_data_plane_history_geometry_idx on external_data_plane_history using gist (geometry)"); +} diff --git a/services/external-data-plane/src/server.mjs b/services/external-data-plane/src/server.mjs new file mode 100644 index 0000000..ba3b8ef --- /dev/null +++ b/services/external-data-plane/src/server.mjs @@ -0,0 +1,822 @@ +import express from "express"; +import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; +import { createServer } from "node:http"; +import { Pool } from "pg"; +import { validateDataProductPublish, validateIntakeBatch } from "@nodedc/external-provider-contract"; +import { readConfig } from "./config.mjs"; +import { + loadDataProductDefinition, + assertPublishMatchesDefinition, + persistDataProductDefinition, + persistDataProductPublish, + pruneBatchReceipts, + pruneDataProductHistory, + prunePatchOutbox, + readDataProductSnapshot, + readPatchEvents, +} from "./data-product-delivery.mjs"; +import { normalizeDataProductDefinition, safeDataProductDefinition } from "./data-product-policy.mjs"; +import { reconcileDataProductDefinitions } from "./definitions.mjs"; +import { assertBatchTimeBounds, rawRetentionExpiry } from "./intake-policy.mjs"; +import { + assertReaderProduct, + createReaderToken, + hashReaderToken, + normalizeReaderBindingRequest, + safeReaderBinding, +} from "./reader-binding.mjs"; +import { migrate } from "./schema.mjs"; +import { + createWriterToken, + hashWriterToken, + materializeDataProductPublish, + materializeWriterBoundBatch, + normalizeWriterBindingRequest, + safeWriterBinding, +} from "./writer-binding.mjs"; + +const config = readConfig(); +const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize }); +const app = express(); +const httpServer = createServer(app); +let retentionSweepTimer = null; +let lastRetentionSweepAt = null; +const activeReaderStreams = new Map(); +const activeStreamResponses = new Set(); +let shuttingDown = false; +let shutdownPromise = null; + +app.disable("x-powered-by"); +app.use(express.json({ limit: config.maxBatchBytes })); + +app.get("/healthz", asyncRoute(async (_req, res) => { + await pool.query("select 1"); + res.json({ + ok: true, + service: "nodedc-external-data-plane", + database: "ready", + internalApiConfigured: Boolean(config.internalAccessToken), + providerLogic: "absent", + commandTransport: "absent", + writerBindings: "supported", + readerBindings: "supported", + dataProductDelivery: "snapshot+durable-patch", + legacyIntake: config.legacyIntakeEnabled ? "migration-only" : "disabled", + writerBindingProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled", + rawRetentionSweep: { + mode: "server-scheduled", + lastSweepAt: lastRetentionSweepAt, + }, + }); +})); + +app.put("/internal/data-plane/v1/data-products/:dataProductId", requireProvisionerApi, asyncRoute(async (req, res) => { + const definition = normalizeDataProductDefinition({ ...req.body, id: req.params.dataProductId }); + const saved = await persistDataProductDefinition(pool, definition); + res.json({ ok: true, dataProduct: safeDataProductDefinition(saved) }); +})); + +app.post("/internal/data-plane/v1/data-products/:dataProductId/publish", requireWriterBinding, asyncRoute(async (req, res) => { + if (hasScopeHeaders(req)) throw httpError(400, "data_product_publish_scope_headers_forbidden"); + const validation = validateDataProductPublish(req.body, { + maxFacts: config.maxFactsPerPublish, + maxAttributesBytes: config.maxAttributesBytesPerFact, + }); + if (!validation.ok) throw httpError(422, validation.errors[0] || "invalid_data_product_publish"); + const dataProductId = requireIdentifier(req.params.dataProductId, "data_product_id_invalid"); + const definition = await loadDataProductDefinition(pool, dataProductId); + if (!definition) throw httpError(404, "data_product_not_found"); + const batch = materializeDataProductPublish(req.body, req.writerBinding, definition, dataProductId); + const canonicalValidation = validateIntakeBatch(batch); + if (!canonicalValidation.ok) throw httpError(422, "materialized_publish_invalid"); + assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds }); + const receipt = await persistDataProductPublish(pool, batch, definition, { + maxPatchOperations: config.maxPatchOperations, + maxPatchBytes: config.maxPatchBytes, + }); + res.status(receipt.idempotent ? 200 : 201).json({ ok: true, ...receipt }); +})); + +app.get("/internal/data-plane/v1/writer/data-products", requireWriterBinding, asyncRoute(async (req, res) => { + const dataProducts = await listGrantedProducts(req.writerBinding); + res.json({ ok: true, dataProducts }); +})); + +app.get("/internal/data-plane/v1/reader/data-products", requireReaderBinding, asyncRoute(async (req, res) => { + const dataProducts = await listGrantedProducts(req.readerBinding); + res.json({ ok: true, dataProducts }); +})); + +app.post("/internal/data-plane/v1/intake", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => { + const validation = validateIntakeBatch(req.body); + if (!validation.ok) throw httpError(422, "invalid_intake_batch"); + assertBatchTimeBounds(req.body, { maxFutureSkewSeconds: config.maxFutureSkewSeconds }); + await assertLegacyBatchProduct(req.body); + const scope = requireScope(req); + if (scope.tenantId !== req.body.source.tenantId || scope.connectionId !== req.body.source.connectionId) { + throw httpError(403, "scope_mismatch"); + } + + const result = await persistBatch(req.body); + res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result }); +})); + +app.post("/internal/data-plane/v1/intake/writer-bound", requireLegacyIntake, requireWriterBinding, asyncRoute(async (req, res) => { + const batch = materializeWriterBoundBatch(req.body, req.writerBinding, { + hasScopeHeaders: hasScopeHeaders(req), + }); + const validation = validateIntakeBatch(batch); + if (!validation.ok) throw httpError(422, "invalid_intake_batch"); + assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds }); + await assertLegacyBatchProduct(batch); + + const result = await persistBatch(batch); + res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result }); +})); + +app.post("/internal/data-plane/v1/writer-bindings", requireProvisionerApi, asyncRoute(async (req, res) => { + const policy = normalizeWriterBindingRequest(req.body, { + maxTtlDays: config.writerBindingMaxTtlDays, + }); + await assertRegisteredProductIds(policy.allowedDataProductIds); + const token = createWriterToken(); + const bindingId = randomUUID(); + const result = await pool.query( + `insert into external_data_plane_writer_bindings ( + id, token_hash, tenant_id, connection_id, provider_id, + allowed_data_product_ids, expires_at + ) values ($1, $2, $3, $4, $5, $6::jsonb, $7) + returning id, tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [ + bindingId, + hashWriterToken(token), + policy.tenantId, + policy.connectionId, + policy.providerId, + JSON.stringify(policy.allowedDataProductIds), + policy.expiresAt, + ], + ); + // The token is intentionally returned exactly once to a trusted provisioner. + // It must be placed directly into an opaque Engine credential reference and + // must never be logged, stored in L2 graph data or shown to a consumer. + sendOneTimeCapability(res, 201, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token }); +})); + +app.post("/internal/data-plane/v1/writer-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => { + const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid"); + const token = createWriterToken(); + const result = await pool.query( + `update external_data_plane_writer_bindings + set token_hash = $2, rotated_at = now() + where id = $1 and active = true and expires_at > now() + returning id, tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [bindingId, hashWriterToken(token)], + ); + if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive"); + sendOneTimeCapability(res, 200, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token }); +})); + +app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => { + const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid"); + const result = await pool.query( + `update external_data_plane_writer_bindings + set active = false, revoked_at = now() + where id = $1 and active = true + returning id, tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [bindingId], + ); + if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive"); + res.json({ ok: true, writerBinding: safeWriterBinding(result.rows[0]) }); +})); + +app.post("/internal/data-plane/v1/reader-bindings", requireProvisionerApi, asyncRoute(async (req, res) => { + const policy = normalizeReaderBindingRequest(req.body, { + maxTtlDays: config.writerBindingMaxTtlDays, + }); + await assertRegisteredProductIds(policy.allowedDataProductIds); + const token = createReaderToken(); + const bindingId = randomUUID(); + const result = await pool.query( + `insert into external_data_plane_reader_bindings ( + id, token_hash, tenant_id, connection_id, provider_id, + allowed_data_product_ids, expires_at + ) values ($1, $2, $3, $4, $5, $6::jsonb, $7) + returning id, tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [ + bindingId, + hashReaderToken(token), + policy.tenantId, + policy.connectionId, + policy.providerId, + JSON.stringify(policy.allowedDataProductIds), + policy.expiresAt, + ], + ); + sendOneTimeCapability(res, 201, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token }); +})); + +app.post("/internal/data-plane/v1/reader-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => { + const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid"); + const token = createReaderToken(); + const result = await pool.query( + `update external_data_plane_reader_bindings + set token_hash = $2, rotated_at = now() + where id = $1 and active = true and expires_at > now() + returning id, tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [bindingId, hashReaderToken(token)], + ); + if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive"); + sendOneTimeCapability(res, 200, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token }); +})); + +app.post("/internal/data-plane/v1/reader-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => { + const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid"); + const result = await pool.query( + `update external_data_plane_reader_bindings + set active = false, revoked_at = now() + where id = $1 and active = true + returning id, tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [bindingId], + ); + if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive"); + res.json({ ok: true, readerBinding: safeReaderBinding(result.rows[0]) }); +})); + +app.get("/internal/data-plane/v1/data-products/:dataProductId/snapshot", requireReaderBinding, asyncRoute(async (req, res) => { + if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden"); + const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId); + const definition = await loadDataProductDefinition(pool, dataProductId); + if (!definition) throw httpError(404, "data_product_not_found"); + const limit = boundedLimit(req.query.limit, 5000, 1, 5000); + const snapshot = await readDataProductSnapshot(pool, req.readerBinding, definition, { limit }); + res.json(snapshot); +})); + +app.get("/internal/data-plane/v1/data-products/:dataProductId/stream", requireReaderBinding, asyncRoute(async (req, res) => { + if (shuttingDown) throw httpError(503, "service_shutting_down"); + if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden"); + const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId); + const definition = await loadDataProductDefinition(pool, dataProductId); + if (!definition) throw httpError(404, "data_product_not_found"); + if (definition.deliveryMode !== "snapshot+patch") throw httpError(409, "data_product_stream_not_supported"); + const streamCount = activeReaderStreams.get(req.readerBinding.id) || 0; + if (streamCount >= config.maxReaderStreams) throw httpError(429, "reader_stream_limit_exceeded"); + activeReaderStreams.set(req.readerBinding.id, streamCount + 1); + activeStreamResponses.add(res); + + let poll = null; + let heartbeat = null; + let polling = false; + let heartbeatWriting = false; + let cleanedUp = false; + const cleanup = () => { + if (cleanedUp) return; + cleanedUp = true; + if (poll) clearInterval(poll); + if (heartbeat) clearInterval(heartbeat); + activeStreamResponses.delete(res); + const remaining = Math.max(0, (activeReaderStreams.get(req.readerBinding.id) || 1) - 1); + if (remaining) activeReaderStreams.set(req.readerBinding.id, remaining); + else activeReaderStreams.delete(req.readerBinding.id); + }; + req.once("close", cleanup); + res.once("close", cleanup); + + try { + let cursor = parseCursor(req.get("last-event-id") || req.query.after || "0"); + const initialEvents = await readPatchEvents(pool, req.readerBinding, definition, cursor); + if (res.destroyed || cleanedUp) return; + res.status(200); + res.set({ + "Cache-Control": "no-cache, no-transform", + "Content-Type": "text/event-stream", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }); + res.flushHeaders(); + await writeSseFrame(res, ": nodedc-data-product-stream\n\n"); + await writeSseFrame(res, `event: nodedc.data-product.ready.v1\ndata: ${JSON.stringify({ + schemaVersion: "nodedc.data-product.ready/v1", + dataProductId, + cursor: cursor.toString(), + emittedAt: new Date().toISOString(), + })}\n\n`); + for (const event of initialEvents) { + await writePatchEvent(res, event); + cursor = BigInt(event.cursor); + } + + const pump = async () => { + if (polling || res.writableEnded || res.destroyed || shuttingDown) return; + polling = true; + try { + await assertReaderStreamAccess(req.readerBinding, dataProductId); + const events = await readPatchEvents(pool, req.readerBinding, definition, cursor); + for (const event of events) { + await writePatchEvent(res, event); + cursor = BigInt(event.cursor); + } + } catch (error) { + const code = safeErrorCode(error); + if (!res.writableEnded && !res.destroyed) { + await writeSseFrame(res, `event: error\ndata: ${JSON.stringify({ ok: false, error: code })}\n\n`).catch(() => {}); + res.end(); + } + } finally { + polling = false; + } + }; + poll = setInterval(() => { void pump(); }, config.streamPollMs); + poll.unref(); + heartbeat = setInterval(() => { + if (polling || heartbeatWriting || res.writableEnded || res.destroyed) return; + heartbeatWriting = true; + void writeSseFrame(res, `: heartbeat ${Date.now()}\n\n`) + .catch(() => { if (!res.writableEnded) res.end(); }) + .finally(() => { heartbeatWriting = false; }); + }, config.streamHeartbeatMs); + heartbeat.unref(); + } catch (error) { + cleanup(); + if (res.headersSent) { + if (!res.writableEnded) res.end(); + return; + } + throw error; + } +})); + +app.get("/internal/data-plane/v1/data-products/:dataProductId/current", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => { + const scope = requireScope(req); + const dataProductId = String(req.params.dataProductId || ""); + if (!isIdentifier(dataProductId)) throw httpError(400, "data_product_id_invalid"); + const limit = boundedLimit(req.query.limit, 200, 1, 1000); + const rows = await pool.query( + `select + provider_id as "providerId", data_product_id as "dataProductId", + source_id as "sourceId", semantic_type as "semanticType", + observed_at as "observedAt", received_at as "receivedAt", attributes, + case when geometry is null then null + else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( + ST_X(geometry::geometry), ST_Y(geometry::geometry) + )) end as geometry + from external_data_plane_current + where tenant_id = $1 and connection_id = $2 and data_product_id = $3 + order by observed_at desc, source_id asc + limit $4`, + [scope.tenantId, scope.connectionId, dataProductId, limit], + ); + res.json({ + ok: true, + dataProductId, + tenantId: scope.tenantId, + connectionId: scope.connectionId, + facts: rows.rows, + }); +})); + +app.get("/internal/data-plane/v1/status", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => { + const scope = requireScope(req); + const result = await pool.query( + `select count(*)::integer as "currentFactCount", max(received_at) as "lastReceivedAt" + from external_data_plane_current where tenant_id = $1 and connection_id = $2`, + [scope.tenantId, scope.connectionId], + ); + res.json({ ok: true, ...scope, ...result.rows[0], providerLogic: "absent", commandTransport: "absent" }); +})); + +app.use((error, _req, res, _next) => { + const status = Number(error?.status || 500); + const publicStatus = status >= 400 && status < 600 ? status : 500; + console.error(JSON.stringify({ event: "external_data_plane_error", error: safeErrorCode(error), status: publicStatus })); + res.status(publicStatus).json({ ok: false, error: publicStatus >= 500 ? "internal_error" : safeErrorCode(error) }); +}); + +await migrate(pool); +const bundledDataProducts = await reconcileDataProductDefinitions(pool); +retentionSweepTimer = setInterval(() => { + void sweepRetention().catch((error) => { + console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) })); + }); +}, config.retentionSweepMs); +retentionSweepTimer.unref(); + +httpServer.listen(config.port, "0.0.0.0", () => { + console.log(`NODE.DC External Data Plane listening on http://0.0.0.0:${config.port}`); + console.log(JSON.stringify({ event: "external_data_plane_definitions_ready", count: bundledDataProducts.length })); + setImmediate(() => { + void sweepRetention().catch((error) => { + console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) })); + }); + }); +}); + +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); + +async function persistBatch(batch) { + const client = await pool.connect(); + try { + await client.query("begin"); + const existing = await client.query( + `select id, fact_count as "factCount", inserted_fact_count as "insertedFactCount" + from external_data_plane_batches + where tenant_id = $1 and connection_id = $2 and provider_id = $3 + and data_product_id = $4 and idempotency_key = $5 + for update`, + [ + batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + batch.contract.dataProductId, batch.batch.idempotencyKey, + ], + ); + if (existing.rowCount) { + await client.query("commit"); + return { batchId: existing.rows[0].id, idempotent: true, ...existing.rows[0] }; + } + + const batchId = randomUUID(); + await client.query( + `insert into external_data_plane_batches ( + id, tenant_id, connection_id, provider_id, data_product_id, contract_version, + ontology_revision, run_id, sequence, idempotency_key, received_at, fact_count + ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + [ + batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + batch.contract.dataProductId, batch.contract.version, batch.contract.ontologyRevision, + batch.batch.runId, batch.batch.sequence, batch.batch.idempotencyKey, + batch.batch.receivedAt, batch.facts.length, + ], + ); + + if (batch.raw) await persistRawEnvelope(client, batchId, batch); + + const records = batch.facts.map((fact) => ({ + id: randomUUID(), + source_id: fact.sourceId, + semantic_type: fact.semanticType, + observed_at: fact.observedAt, + attributes: fact.attributes || {}, + geometry: fact.geometry || null, + fingerprint: hash(fact), + })); + const inserted = await client.query( + `insert into external_data_plane_facts ( + id, batch_id, tenant_id, connection_id, provider_id, data_product_id, + source_id, semantic_type, observed_at, received_at, attributes, geometry, fingerprint + ) + select item.id::uuid, $1, $2, $3, $4, $5, + item.source_id, item.semantic_type, item.observed_at, $6, + coalesce(item.attributes, '{}'::jsonb), + case when item.geometry is null then null + else ST_SetSRID(ST_MakePoint( + (item.geometry->'coordinates'->>0)::double precision, + (item.geometry->'coordinates'->>1)::double precision + ), 4326)::geography end, + item.fingerprint + from jsonb_to_recordset($7::jsonb) as item( + id text, source_id text, semantic_type text, observed_at timestamptz, + attributes jsonb, geometry jsonb, fingerprint text + ) + on conflict do nothing + returning id`, + [ + batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId, + batch.contract.dataProductId, batch.batch.receivedAt, JSON.stringify(records), + ], + ); + await client.query( + `insert into external_data_plane_current ( + tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, + observed_at, received_at, attributes, geometry, fingerprint, updated_at + ) + select tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, + observed_at, received_at, attributes, geometry, fingerprint, now() + from external_data_plane_facts where batch_id = $1 + on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type) + do update set + observed_at = excluded.observed_at, + received_at = excluded.received_at, + attributes = excluded.attributes, + geometry = excluded.geometry, + fingerprint = excluded.fingerprint, + updated_at = now() + where excluded.observed_at >= external_data_plane_current.observed_at`, + [batchId], + ); + await client.query( + "update external_data_plane_batches set inserted_fact_count = $2 where id = $1", + [batchId, inserted.rowCount], + ); + await client.query("commit"); + return { batchId, idempotent: false, factCount: batch.facts.length, insertedFactCount: inserted.rowCount }; + } catch (error) { + await client.query("rollback"); + throw error; + } finally { + client.release(); + } +} + +async function persistRawEnvelope(client, batchId, batch) { + const serialized = batch.raw.payload === undefined ? null : JSON.stringify(batch.raw.payload); + const expiresAt = rawRetentionExpiry({ rawRetentionDays: config.rawRetentionDays }); + await client.query( + `insert into external_data_plane_raw_envelopes ( + id, batch_id, payload_hash, content_type, payload, payload_ref, payload_bytes, received_at, expires_at + ) values ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9)`, + [ + randomUUID(), batchId, batch.raw.hash || hash(batch.raw.payload), batch.raw.contentType, serialized, batch.raw.ref || null, + serialized ? Buffer.byteLength(serialized) : null, batch.batch.receivedAt, expiresAt, + ], + ); +} + +function requireInternalApi(req, _res, next) { + if (!config.internalAccessToken) return next(httpError(503, "internal_api_not_configured")); + const value = bearerToken(req); + if (!value || !safeEqual(value, config.internalAccessToken)) return next(httpError(401, "unauthorized")); + return next(); +} + +function requireLegacyIntake(_req, _res, next) { + if (!config.legacyIntakeEnabled) return next(httpError(410, "legacy_intake_disabled")); + return next(); +} + +function requireProvisionerApi(req, _res, next) { + if (!config.provisionerApiEnabled) return next(httpError(503, "provisioner_api_disabled")); + if (!config.provisionerAccessToken) return next(httpError(503, "provisioner_api_not_configured")); + const value = bearerToken(req); + if (!value || !safeEqual(value, config.provisionerAccessToken)) return next(httpError(401, "provisioner_unauthorized")); + return next(); +} + +function requireWriterBinding(req, _res, next) { + return resolveWriterBinding(req) + .then((binding) => { + req.writerBinding = binding; + next(); + }) + .catch(next); +} + +function requireReaderBinding(req, _res, next) { + return resolveReaderBinding(req) + .then((binding) => { + req.readerBinding = binding; + next(); + }) + .catch(next); +} + +async function resolveWriterBinding(req) { + const token = bearerToken(req); + if (!token) throw httpError(401, "writer_binding_unauthorized"); + const result = await pool.query( + `select id, tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt" + from external_data_plane_writer_bindings + where token_hash = $1 and active = true and expires_at > now()`, + [hashWriterToken(token)], + ); + if (!result.rowCount) throw httpError(401, "writer_binding_unauthorized"); + return result.rows[0]; +} + +async function resolveReaderBinding(req) { + const token = bearerToken(req); + if (!token) throw httpError(401, "reader_binding_unauthorized"); + const result = await pool.query( + `select id, token_hash as "tokenHash", tenant_id as "tenantId", connection_id as "connectionId", + provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", + expires_at as "expiresAt", active, created_at as "createdAt", + rotated_at as "rotatedAt", revoked_at as "revokedAt" + from external_data_plane_reader_bindings + where token_hash = $1 and active = true and expires_at > now()`, + [hashReaderToken(token)], + ); + if (!result.rowCount) throw httpError(401, "reader_binding_unauthorized"); + return result.rows[0]; +} + +async function assertReaderStreamAccess(binding, dataProductId) { + const result = await pool.query( + `select binding.id + from external_data_plane_reader_bindings as binding + join external_data_plane_products as product + on product.id = $3 and product.active = true + where binding.id = $1 and binding.token_hash = $2 + and binding.active = true and binding.expires_at > now() + and binding.allowed_data_product_ids ? $3`, + [binding.id, binding.tokenHash, dataProductId], + ); + if (!result.rowCount) throw httpError(401, "reader_stream_access_revoked"); +} + +async function assertLegacyBatchProduct(batch) { + const definition = await loadDataProductDefinition(pool, batch.contract.dataProductId); + if (!definition) throw httpError(422, "legacy_data_product_not_registered"); + if (definition.version !== batch.contract.version || definition.ontologyRevision !== batch.contract.ontologyRevision) { + throw httpError(422, "legacy_data_product_contract_mismatch"); + } + assertPublishMatchesDefinition(batch, definition); +} + +async function listGrantedProducts(binding) { + const allowed = Array.isArray(binding.allowedDataProductIds) ? binding.allowedDataProductIds : []; + if (!allowed.length) return []; + const result = await pool.query( + `select id, version, ontology_revision as "ontologyRevision", + delivery_mode as "deliveryMode", semantic_types as "semanticTypes", + fields, history_policy as "historyPolicy", active, + created_at as "createdAt", updated_at as "updatedAt" + from external_data_plane_products + where active = true and id = any($1::text[]) + order by id asc`, + [allowed], + ); + return result.rows.map(safeDataProductDefinition); +} + +async function assertRegisteredProductIds(dataProductIds) { + const result = await pool.query( + "select id from external_data_plane_products where active = true and id = any($1::text[])", + [dataProductIds], + ); + const registered = new Set(result.rows.map((row) => row.id)); + if (dataProductIds.some((id) => !registered.has(id))) throw httpError(422, "binding_data_product_not_registered"); +} + +function requireScope(req) { + const tenantId = String(req.get("x-nodedc-tenant-id") || ""); + const connectionId = String(req.get("x-nodedc-connection-id") || ""); + if (!isIdentifier(tenantId) || !isIdentifier(connectionId)) throw httpError(400, "scope_headers_required"); + return { tenantId, connectionId }; +} + +function hasScopeHeaders(req) { + return Object.hasOwn(req.headers, "x-nodedc-tenant-id") || Object.hasOwn(req.headers, "x-nodedc-connection-id"); +} + +function bearerToken(req) { + return String(req.get("authorization") || "").replace(/^Bearer\s+/i, ""); +} + +function requireUuid(value, code) { + const normalized = String(value || ""); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(normalized)) { + throw httpError(400, code); + } + return normalized; +} + +function requireIdentifier(value, code) { + const normalized = String(value || ""); + if (!isIdentifier(normalized)) throw httpError(400, code); + return normalized; +} + +function parseCursor(value) { + const normalized = String(value ?? ""); + if (!/^(?:0|[1-9]\d*)$/.test(normalized)) throw httpError(400, "data_product_cursor_invalid"); + const cursor = BigInt(normalized); + if (cursor > 9_223_372_036_854_775_807n) throw httpError(400, "data_product_cursor_invalid"); + return cursor; +} + +function writePatchEvent(res, event) { + return writeSseFrame(res, `id: ${event.cursor}\nevent: nodedc.data-product.patch.v1\ndata: ${JSON.stringify(event)}\n\n`); +} + +function writeSseFrame(res, frame) { + if (res.writableEnded || res.destroyed) return Promise.reject(httpError(499, "reader_stream_closed")); + if (res.write(frame)) return Promise.resolve(); + return new Promise((resolve, reject) => { + const cleanup = () => { + res.off("drain", onDrain); + res.off("close", onClose); + res.off("error", onError); + }; + const onDrain = () => { cleanup(); resolve(); }; + const onClose = () => { cleanup(); reject(httpError(499, "reader_stream_closed")); }; + const onError = (error) => { cleanup(); reject(error); }; + res.once("drain", onDrain); + res.once("close", onClose); + res.once("error", onError); + }); +} + +function sendOneTimeCapability(res, status, payload) { + // Create/rotate is the only boundary where a capability exists in + // plaintext. A trusted provisioner must consume the response in memory and + // place it directly into its opaque destination; intermediaries must never + // cache or persist it. + res.set({ + "Cache-Control": "no-store, max-age=0", + Pragma: "no-cache", + Expires: "0", + }); + return res.status(status).json(payload); +} + +function safeEqual(left, right) { + const leftBuffer = Buffer.from(left); + const rightBuffer = Buffer.from(right); + return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer); +} + +function boundedLimit(value, fallback, min, max) { + const candidate = Number.parseInt(String(value ?? ""), 10); + if (!Number.isInteger(candidate)) return fallback; + return Math.max(min, Math.min(max, candidate)); +} + +function hash(value) { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function isIdentifier(value) { + return /^[a-z][a-z0-9._:-]{2,127}$/i.test(value); +} + +function asyncRoute(handler) { + return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next); +} + +function httpError(status, code) { + return Object.assign(new Error(code), { status, code }); +} + +function safeErrorCode(error) { + return String(error?.code || error?.message || "internal_error").replace(/[^a-z0-9_.:-]/gi, "_").slice(0, 120); +} + +async function sweepExpiredRawEnvelopes() { + const result = await pool.query( + `with expired as ( + select id from external_data_plane_raw_envelopes + where expires_at < now() + order by expires_at asc + limit $1 + ) + delete from external_data_plane_raw_envelopes as target + using expired where target.id = expired.id`, + [config.retentionDeleteLimit], + ); + return result.rowCount; +} + +async function sweepRetention() { + const rawEnvelopeCount = await sweepExpiredRawEnvelopes(); + const patchEventCount = await prunePatchOutbox(pool, { + retentionMs: config.patchRetentionMs, + limit: config.retentionDeleteLimit, + }); + const historyFactCount = await pruneDataProductHistory(pool, { limit: config.retentionDeleteLimit }); + const batchReceiptCount = await pruneBatchReceipts(pool, { + retentionMs: config.receiptRetentionMs, + limit: config.retentionDeleteLimit, + }); + lastRetentionSweepAt = new Date().toISOString(); + return { rawEnvelopeCount, patchEventCount, historyFactCount, batchReceiptCount }; +} + +async function shutdown() { + if (shutdownPromise) return shutdownPromise; + shutdownPromise = (async () => { + shuttingDown = true; + if (retentionSweepTimer) clearInterval(retentionSweepTimer); + for (const response of activeStreamResponses) { + if (!response.writableEnded) response.end(); + } + const closed = new Promise((resolve) => httpServer.close(resolve)); + httpServer.closeIdleConnections?.(); + const forceClose = setTimeout(() => httpServer.closeAllConnections?.(), 250); + await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 2_000))]); + clearTimeout(forceClose); + httpServer.closeAllConnections?.(); + await pool.end(); + })(); + return shutdownPromise; +} diff --git a/services/external-data-plane/src/writer-binding.mjs b/services/external-data-plane/src/writer-binding.mjs new file mode 100644 index 0000000..386eb16 --- /dev/null +++ b/services/external-data-plane/src/writer-binding.mjs @@ -0,0 +1,182 @@ +import { createHash, randomBytes } from "node:crypto"; + +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const TOKEN_PREFIX = "ndc_edpwb_"; +const SECRET_LIKE_KEY = /(token|secret|password|access[_-]?token|refresh[_-]?token|api[_-]?key)/i; +const BINDING_REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]); +const BINDING_SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]); + +/** + * Produces an opaque, high-entropy capability. The plaintext is returned only + * to the trusted provisioning caller; persistence uses its SHA-256 digest. + */ +export function createWriterToken() { + return `${TOKEN_PREFIX}${randomBytes(32).toString("base64url")}`; +} + +export function hashWriterToken(token) { + return createHash("sha256").update(String(token), "utf8").digest("hex"); +} + +export function normalizeWriterBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) { + if (!isPlainObject(value) || !isPlainObject(value.source)) { + throw writerBindingError("writer_binding_request_invalid"); + } + if (containsSecretLikeKey(value)) { + throw writerBindingError("writer_binding_request_secret_material_forbidden"); + } + if (value.endpoint !== undefined || value.url !== undefined || value.host !== undefined) { + throw writerBindingError("writer_binding_request_transport_forbidden"); + } + if (!hasOnlyKeys(value, BINDING_REQUEST_KEYS) || !hasOnlyKeys(value.source, BINDING_SOURCE_KEYS)) { + throw writerBindingError("writer_binding_request_fields_invalid"); + } + + const tenantId = normalizeIdentifier(value.source.tenantId); + const connectionId = normalizeIdentifier(value.source.connectionId); + const providerId = normalizeIdentifier(value.source.providerId); + if (!tenantId || !connectionId || !providerId) { + throw writerBindingError("writer_binding_scope_invalid"); + } + + const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds); + if (!allowedDataProductIds.length) { + throw writerBindingError("writer_binding_data_products_invalid"); + } + + const expiresAt = new Date(String(value.expiresAt || "")); + const maxExpiresAt = new Date(now.getTime() + maxTtlDays * 24 * 60 * 60 * 1000); + if (Number.isNaN(expiresAt.getTime()) || expiresAt <= now || expiresAt > maxExpiresAt) { + throw writerBindingError("writer_binding_expiry_invalid"); + } + + return Object.freeze({ + tenantId, + connectionId, + providerId, + allowedDataProductIds, + expiresAt: expiresAt.toISOString(), + }); +} + +/** + * Converts a caller-provided, deliberately unscoped intake envelope into the + * canonical scoped form. Caller scope is rejected, never trusted or merged. + */ +export function materializeWriterBoundBatch(value, binding, { hasScopeHeaders = false, now = new Date() } = {}) { + if (!isPlainObject(value) || !isPlainObject(value.source) || !isPlainObject(binding)) { + throw writerBindingError("writer_bound_intake_invalid"); + } + if (hasScopeHeaders || value.source.tenantId !== undefined || value.source.connectionId !== undefined) { + throw writerBindingError("writer_bound_scope_forbidden"); + } + if (binding.active !== true || !bindingIsCurrent(binding, now)) { + throw writerBindingError("writer_binding_inactive"); + } + + const providerId = normalizeIdentifier(value.source.providerId); + const tenantId = normalizeIdentifier(binding.tenantId); + const connectionId = normalizeIdentifier(binding.connectionId); + if (!providerId || !tenantId || !connectionId || providerId !== binding.providerId) { + throw writerBindingError("writer_binding_provider_forbidden"); + } + + const dataProductId = normalizeIdentifier(value.contract?.dataProductId); + const allowedDataProductIds = uniqueIdentifiers(binding.allowedDataProductIds); + if (!dataProductId || !allowedDataProductIds.includes(dataProductId)) { + throw writerBindingError("writer_binding_data_product_forbidden"); + } + + return { + ...value, + source: { providerId, tenantId, connectionId }, + }; +} + +/** + * Materializes the provider-neutral Data Product publish wire form. Unlike the + * legacy writer-bound intake, the caller cannot send provider identity, + * contract metadata, receivedAt or any scope at all. + */ +export function materializeDataProductPublish(value, binding, definition, dataProductId, { now = new Date() } = {}) { + if (!isPlainObject(value) || !isPlainObject(binding) || !isPlainObject(definition)) { + throw writerBindingError("data_product_publish_invalid"); + } + if (binding.active !== true || !bindingIsCurrent(binding, now)) { + throw writerBindingError("writer_binding_inactive"); + } + const normalizedProductId = normalizeIdentifier(dataProductId); + const allowedDataProductIds = uniqueIdentifiers(binding.allowedDataProductIds); + if (!normalizedProductId || normalizedProductId !== definition.id || !allowedDataProductIds.includes(normalizedProductId)) { + throw writerBindingError("writer_binding_data_product_forbidden"); + } + const tenantId = normalizeIdentifier(binding.tenantId); + const connectionId = normalizeIdentifier(binding.connectionId); + const providerId = normalizeIdentifier(binding.providerId); + if (!tenantId || !connectionId || !providerId) throw writerBindingError("writer_binding_scope_invalid"); + + return { + schemaVersion: "nodedc.external-provider-contract/v1", + source: { tenantId, connectionId, providerId }, + contract: { + dataProductId: normalizedProductId, + ontologyRevision: definition.ontologyRevision, + version: definition.version, + }, + batch: { + runId: value.batch?.runId, + sequence: value.batch?.sequence, + idempotencyKey: value.batch?.idempotencyKey, + receivedAt: now.toISOString(), + }, + facts: value.facts, + }; +} + +export function safeWriterBinding(binding) { + return { + id: binding.id, + tenantId: binding.tenantId, + connectionId: binding.connectionId, + providerId: binding.providerId, + allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds), + active: binding.active === true, + expiresAt: new Date(binding.expiresAt).toISOString(), + createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined, + rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined, + revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined, + }; +} + +export function writerBindingError(code) { + return Object.assign(new Error(code), { status: 400, code }); +} + +function bindingIsCurrent(binding, now) { + const expiresAt = new Date(binding.expiresAt); + return !Number.isNaN(expiresAt.getTime()) && expiresAt > now; +} + +function normalizeIdentifier(value) { + const normalized = typeof value === "string" ? value.trim() : ""; + return IDENTIFIER.test(normalized) ? normalized : ""; +} + +function uniqueIdentifiers(value) { + if (!Array.isArray(value)) return []; + return [...new Set(value.map(normalizeIdentifier).filter(Boolean))]; +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function containsSecretLikeKey(value) { + if (Array.isArray(value)) return value.some(containsSecretLikeKey); + if (!isPlainObject(value)) return false; + return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeKey(child)); +} + +function hasOnlyKeys(value, allowedKeys) { + return Object.keys(value).every((key) => allowedKeys.has(key)); +} diff --git a/services/external-data-plane/test/api.integration.test.mjs b/services/external-data-plane/test/api.integration.test.mjs new file mode 100644 index 0000000..a0e97c8 --- /dev/null +++ b/services/external-data-plane/test/api.integration.test.mjs @@ -0,0 +1,280 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract"; + +const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL; +if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required"); +const port = await freePort(); +const baseUrl = `http://127.0.0.1:${port}`; +const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-api-test-")); +const provisionerSecret = "nodedc_edp_integration_provisioner_secret_0123456789abcdef"; +const secretPath = join(directory, "provisioner-token"); +await writeFile(secretPath, `${provisionerSecret}\n`, { mode: 0o600 }); + +const child = spawn(process.execPath, ["src/server.mjs"], { + cwd: new URL("..", import.meta.url), + env: { + ...process.env, + PORT: String(port), + EXTERNAL_DATA_PLANE_DATABASE_URL: databaseUrl, + EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath, + EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000", + EXTERNAL_DATA_PLANE_STREAM_POLL_MS: "250", + EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS: "5000", + }, + stdio: ["ignore", "pipe", "pipe"], +}); +let childOutput = ""; +child.stdout.on("data", (chunk) => { childOutput += chunk.toString(); }); +child.stderr.on("data", (chunk) => { childOutput += chunk.toString(); }); + +try { + await waitForHealth(); + const productId = "api.test.positions.v1"; + const product = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}`, { + method: "PUT", + token: provisionerSecret, + body: { + version: "1.0.0", + ontologyRevision: "ontology.api.test.v1", + deliveryMode: "snapshot+patch", + semanticTypes: ["map.moving_object"], + fields: ["source_id", "observed_at", "geometry", "status"], + history: { mode: "sampled", intervalMs: 60_000, strategy: "latest-per-entity-per-bucket", retentionDays: 30 }, + }, + }); + assert.equal(product.dataProduct.id, productId); + + const expiry = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(); + const scope = { tenantId: "api-tenant", connectionId: "api-connection", providerId: "api-provider" }; + const writerIssuance = await rawJsonRequest("/internal/data-plane/v1/writer-bindings", { + method: "POST", + token: provisionerSecret, + body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry }, + }); + const readerIssuance = await rawJsonRequest("/internal/data-plane/v1/reader-bindings", { + method: "POST", + token: provisionerSecret, + body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry }, + }); + assertOneTimeCapabilityResponse(writerIssuance.response); + assertOneTimeCapabilityResponse(readerIssuance.response); + const writer = writerIssuance.value; + const reader = readerIssuance.value; + assert.equal(typeof writer.token, "string"); + assert.equal(typeof reader.token, "string"); + + const catalog = await jsonRequest("/internal/data-plane/v1/writer/data-products", { token: writer.token }); + assert.deepEqual(catalog.dataProducts.map((value) => value.id), [productId]); + + const first = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, { + method: "POST", + token: writer.token, + body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61), + }); + assert.equal(first.currentUpdatedCount, 1); + const duplicate = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, { + method: "POST", + token: writer.token, + body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61), + }); + assert.equal(duplicate.idempotent, true); + const reusedKey = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/publish`, { + method: "POST", + headers: { Authorization: `Bearer ${writer.token}`, "Content-Type": "application/json" }, + body: JSON.stringify(publish("api-run-01", "2026-07-15T12:00:00.000Z", 99.9)), + }); + assert.equal(reusedKey.status, 409); + assert.equal((await reusedKey.json()).error, "idempotency_key_reused"); + + const snapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token }); + assert.equal(validateDataProductSnapshot(snapshot).ok, true); + assert.equal(snapshot.facts.length, 1); + + const controller = new AbortController(); + const response = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${snapshot.cursor}`, { + headers: { Authorization: `Bearer ${reader.token}` }, + signal: controller.signal, + }); + assert.equal(response.status, 200); + const patchPromise = readFirstPatch(response, controller); + await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, { + method: "POST", + token: writer.token, + body: publish("api-run-02", "2026-07-15T12:00:10.000Z", 37.62), + }); + const patch = await patchPromise; + assert.equal(validateDataProductPatch(patch).ok, true); + assert.equal(patch.previousCursor, snapshot.cursor); + + const forbidden = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, { + headers: { + Authorization: `Bearer ${reader.token}`, + "X-NODEDC-Tenant-Id": scope.tenantId, + "X-NODEDC-Connection-Id": scope.connectionId, + }, + }); + assert.equal(forbidden.status, 400); + + const revocationSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token }); + const revocationStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${revocationSnapshot.cursor}`, { + headers: { Authorization: `Bearer ${reader.token}` }, + }); + assert.equal(revocationStream.status, 200); + const revokedEventPromise = readNamedEvent(revocationStream, "error"); + const rotatedReaderIssuance = await rawJsonRequest(`/internal/data-plane/v1/reader-bindings/${reader.readerBinding.id}/rotate`, { + method: "POST", + token: provisionerSecret, + }); + assertOneTimeCapabilityResponse(rotatedReaderIssuance.response); + const rotatedReader = rotatedReaderIssuance.value; + const revokedEvent = await revokedEventPromise; + assert.equal(revokedEvent.error, "reader_stream_access_revoked"); + const rejectedOldReader = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, { + headers: { Authorization: `Bearer ${reader.token}` }, + }); + assert.equal(rejectedOldReader.status, 401); + const rotatedSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: rotatedReader.token }); + assert.equal(rotatedSnapshot.dataProduct.id, productId); + + const shutdownStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${rotatedSnapshot.cursor}`, { + headers: { Authorization: `Bearer ${rotatedReader.token}` }, + }); + assert.equal(shutdownStream.status, 200); + child.kill("SIGTERM"); + assert.equal(await waitForChildExit(child, 3_000), true); + + console.log("external-data-plane API integration: ok"); +} catch (error) { + error.message = `${error.message}\nserver_output=${childOutput.replace(/[A-Za-z0-9_-]{48,}/g, "[redacted]").slice(-4000)}`; + throw error; +} finally { + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolve) => child.once("exit", resolve)), + new Promise((resolve) => setTimeout(resolve, 3000)), + ]); + await rm(directory, { recursive: true, force: true }); +} + +function publish(runId, observedAt, longitude) { + return { + schemaVersion: "nodedc.data-product.publish/v1", + batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0` }, + facts: [{ + sourceId: "unit-01", + semanticType: "map.moving_object", + observedAt, + attributes: { status: "online" }, + geometry: { type: "Point", coordinates: [longitude, 55.75] }, + }], + }; +} + +async function jsonRequest(path, { method = "GET", token, body } = {}) { + const { response, value } = await rawJsonRequest(path, { method, token, body }); + if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`); + return value; +} + +async function rawJsonRequest(path, { method = "GET", token, body } = {}) { + const response = await fetch(`${baseUrl}${path}`, { + method, + headers: { + ...(token ? { Authorization: `Bearer ${token}` } : {}), + ...(body ? { "Content-Type": "application/json" } : {}), + }, + body: body ? JSON.stringify(body) : undefined, + }); + const value = await response.json(); + if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`); + return { response, value }; +} + +function assertOneTimeCapabilityResponse(response) { + assert.equal(response.headers.get("cache-control"), "no-store, max-age=0"); + assert.equal(response.headers.get("pragma"), "no-cache"); + assert.equal(response.headers.get("expires"), "0"); +} + +async function waitForHealth() { + const deadline = Date.now() + 15_000; + while (Date.now() < deadline) { + try { + const response = await fetch(`${baseUrl}/healthz`); + if (response.ok) return; + } catch {} + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error("server_health_timeout"); +} + +async function readFirstPatch(response, controller) { + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + const timeout = setTimeout(() => controller.abort(), 10_000); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) throw new Error("stream_closed_before_patch"); + buffer += decoder.decode(value, { stream: true }); + for (const frame of buffer.split("\n\n")) { + const data = frame.split("\n").find((line) => line.startsWith("data: ")); + if (frame.includes("event: nodedc.data-product.patch.v1") && data) { + controller.abort(); + return JSON.parse(data.slice(6)); + } + } + const boundary = buffer.lastIndexOf("\n\n"); + if (boundary >= 0) buffer = buffer.slice(boundary + 2); + } + } finally { + clearTimeout(timeout); + } +} + +async function readNamedEvent(response, eventName) { + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const next = await Promise.race([ + reader.read(), + new Promise((_, reject) => setTimeout(() => reject(new Error("stream_event_timeout")), 10_000)), + ]); + if (next.done) throw new Error(`stream_closed_before_${eventName}`); + buffer += decoder.decode(next.value, { stream: true }).replace(/\r\n/g, "\n"); + let boundary; + while ((boundary = buffer.indexOf("\n\n")) !== -1) { + const frame = buffer.slice(0, boundary); + buffer = buffer.slice(boundary + 2); + if (!frame.includes(`event: ${eventName}`)) continue; + const data = frame.split("\n").find((line) => line.startsWith("data: ")); + if (data) return JSON.parse(data.slice(6)); + } + } + throw new Error(`stream_event_timeout:${eventName}`); +} + +async function waitForChildExit(process, timeoutMs) { + if (process.exitCode !== null) return true; + return Promise.race([ + new Promise((resolve) => process.once("exit", () => resolve(true))), + new Promise((resolve) => setTimeout(() => resolve(false), timeoutMs)), + ]); +} + +async function freePort() { + const server = net.createServer(); + await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject)); + const { port } = server.address(); + await new Promise((resolve) => server.close(resolve)); + return port; +} diff --git a/services/external-data-plane/test/config.test.mjs b/services/external-data-plane/test/config.test.mjs new file mode 100644 index 0000000..5a5f4ac --- /dev/null +++ b/services/external-data-plane/test/config.test.mjs @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readConfig } from "../src/config.mjs"; + +const base = { + EXTERNAL_DATA_PLANE_DATABASE_URL: "postgresql://user:pass@example.invalid/data_plane", + NODEDC_INTERNAL_ACCESS_TOKEN: "legacy-internal-token", +}; + +const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-config-")); +const secretPath = join(directory, "provisioner-token"); +const secret = "provisioner_secret_is_separate_from_legacy_internal_token_123456"; +try { + await writeFile(secretPath, `${secret}\n`, { encoding: "utf8", mode: 0o600 }); + const config = readConfig({ + ...base, + EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath, + EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS: "120", + EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000", + }); + assert.equal(config.provisionerAccessToken, secret); + assert.equal(config.maxFutureSkewSeconds, 120); + assert.equal(config.retentionSweepMs, 60000); + assert.equal(config.maxPatchBytes, 262144); + assert.equal(config.receiptRetentionMs, 604800000); + assert.equal(config.legacyIntakeEnabled, false); + assert.equal(readConfig({ ...base, EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED: "true" }).legacyIntakeEnabled, true); + assert.throws(() => readConfig({ + ...base, + EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath, + NODEDC_INTERNAL_ACCESS_TOKEN: secret, + }), /provisioner_token_must_differ_from_internal_token/); + assert.throws(() => readConfig({ + ...base, + EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"), + }), /external_data_plane_provisioner_token_file_unreadable/); + assert.equal(readConfig({ + ...base, + EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "false", + EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"), + }).provisionerAccessToken, ""); +} finally { + await rm(directory, { recursive: true, force: true }); +} + +console.log("external-data-plane config: ok"); diff --git a/services/external-data-plane/test/data-product-delivery.integration.test.mjs b/services/external-data-plane/test/data-product-delivery.integration.test.mjs new file mode 100644 index 0000000..2e99f16 --- /dev/null +++ b/services/external-data-plane/test/data-product-delivery.integration.test.mjs @@ -0,0 +1,184 @@ +import assert from "node:assert/strict"; +import { Pool } from "pg"; +import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract"; +import { + loadDataProductDefinition, + persistDataProductDefinition, + persistDataProductPublish, + readDataProductSnapshot, + readPatchEvents, +} from "../src/data-product-delivery.mjs"; +import { normalizeDataProductDefinition } from "../src/data-product-policy.mjs"; +import { migrate } from "../src/schema.mjs"; + +const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL; +if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required"); +const pool = new Pool({ connectionString: databaseUrl, max: 4 }); + +try { + await migrate(pool); + await pool.query(`truncate table + external_data_plane_patch_outbox, + external_data_plane_history, + external_data_plane_current, + external_data_plane_facts, + external_data_plane_raw_envelopes, + external_data_plane_batches, + external_data_plane_delivery_state, + external_data_plane_reader_bindings, + external_data_plane_writer_bindings, + external_data_plane_products + restart identity cascade`); + + const definition = normalizeDataProductDefinition({ + id: "test.positions.current.v1", + version: "1.0.0", + ontologyRevision: "ontology.test.positions.v1", + deliveryMode: "snapshot+patch", + semanticTypes: ["map.moving_object"], + fields: ["source_id", "observed_at", "geometry", "status"], + history: { + mode: "sampled", + intervalMs: 60_000, + strategy: "latest-per-entity-per-bucket", + retentionDays: 30, + }, + }); + await persistDataProductDefinition(pool, definition); + const storedDefinition = await loadDataProductDefinition(pool, definition.id); + const binding = { + id: "reader-binding-test", + tenantId: "tenant-test", + connectionId: "connection-test", + providerId: "provider-test", + allowedDataProductIds: [definition.id], + active: true, + expiresAt: "2027-01-01T00:00:00.000Z", + }; + + const firstBatch = batch("run-01", "2026-07-15T10:00:05.000Z", [ + fact("unit-01", 37.61, 55.75, "online"), + fact("unit-02", 37.62, 55.76, "online"), + ]); + const first = await persistDataProductPublish(pool, firstBatch, storedDefinition); + assert.equal(first.idempotent, false); + assert.equal(first.currentUpdatedCount, 2); + assert.equal(first.historyInsertedCount, 2); + assert.equal(first.cursor, "1"); + + const duplicate = await persistDataProductPublish(pool, firstBatch, storedDefinition); + assert.equal(duplicate.idempotent, true); + assert.equal(duplicate.cursor, "1"); + + await assert.rejects( + persistDataProductPublish(pool, { + ...firstBatch, + facts: [fact("unit-01", 99.9, 55.75, "changed")], + }, storedDefinition), + (error) => error?.status === 409 && error?.code === "idempotency_key_reused", + ); + + const concurrentBatch = batch("run-concurrent", "2026-07-15T10:00:08.000Z", [ + fact("unit-03", 37.63, 55.77, "online", "2026-07-15T10:00:08.000Z"), + ]); + const concurrent = await Promise.all([ + persistDataProductPublish(pool, concurrentBatch, storedDefinition), + persistDataProductPublish(pool, concurrentBatch, storedDefinition), + ]); + assert.deepEqual(concurrent.map((value) => value.idempotent).sort(), [false, true]); + + const second = await persistDataProductPublish(pool, batch( + "run-02", + "2026-07-15T10:00:15.000Z", + [fact("unit-01", 37.611, 55.751, "moving", "2026-07-15T10:00:15.000Z")], + ), storedDefinition); + assert.equal(second.currentUpdatedCount, 1); + assert.equal(second.cursor, "3"); + + const latest = await persistDataProductPublish(pool, batch( + "run-latest", + "2026-07-15T10:02:05.000Z", + [fact("unit-01", 37.7, 55.8, "latest", "2026-07-15T10:02:00.000Z")], + ), storedDefinition); + assert.equal(latest.currentUpdatedCount, 1); + const late = await persistDataProductPublish(pool, batch( + "run-late", + "2026-07-15T10:03:00.000Z", + [fact("unit-01", 37.65, 55.78, "late", "2026-07-15T10:01:30.000Z")], + ), storedDefinition); + assert.equal(late.currentUpdatedCount, 0); + assert.equal(late.historyInsertedCount, 1); + assert.equal(late.patchOperationCount, 0); + + await assert.rejects( + persistDataProductPublish(pool, batch("run-bad-type", "2026-07-15T10:03:10.000Z", [{ + ...fact("unit-04", 37.6, 55.7, "online"), + semanticType: "map.forbidden", + }]), storedDefinition), + (error) => error?.status === 422 && error?.code === "data_product_semantic_type_forbidden", + ); + await assert.rejects( + persistDataProductPublish(pool, batch("run-bad-field", "2026-07-15T10:03:11.000Z", [{ + ...fact("unit-04", 37.6, 55.7, "online"), + attributes: { undeclared: true }, + }]), storedDefinition), + (error) => error?.status === 422 && error?.code === "data_product_field_forbidden", + ); + await assert.rejects( + persistDataProductPublish(pool, batch("run-duplicate-fact", "2026-07-15T10:03:12.000Z", [ + fact("unit-04", 37.6, 55.7, "online"), + fact("unit-04", 37.61, 55.71, "moving"), + ]), storedDefinition), + (error) => error?.status === 422 && error?.code === "data_product_duplicate_entity_key", + ); + + const snapshot = await readDataProductSnapshot(pool, binding, storedDefinition); + assert.equal(validateDataProductSnapshot(snapshot).ok, true); + assert.equal(snapshot.cursor, "4"); + assert.equal(snapshot.facts.length, 3); + assert.equal(snapshot.facts.find((value) => value.sourceId === "unit-01").attributes.status, "latest"); + + const patches = await readPatchEvents(pool, binding, storedDefinition, 0n); + assert.equal(patches.length, 4); + assert.equal(patches.every((patch) => validateDataProductPatch(patch).ok), true); + assert.deepEqual(patches.map((patch) => patch.cursor), ["1", "2", "3", "4"]); + + await assert.rejects( + readPatchEvents(pool, binding, storedDefinition, 999n), + (error) => error?.status === 409 && error?.code === "resync_required", + ); + + const history = await pool.query("select source_id, observed_at from external_data_plane_history order by source_id"); + assert.equal(history.rowCount, 5); + const unitOneHistory = history.rows.filter((row) => row.source_id === "unit-01"); + assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:01:30.000Z"), true); + assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:02:00.000Z"), true); + + console.log("external-data-plane delivery integration: ok"); +} finally { + await pool.end(); +} + +function batch(runId, receivedAt, facts) { + return { + schemaVersion: "nodedc.external-provider-contract/v1", + source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" }, + contract: { + dataProductId: "test.positions.current.v1", + ontologyRevision: "ontology.test.positions.v1", + version: "1.0.0", + }, + batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0`, receivedAt }, + facts, + }; +} + +function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10:00:00.000Z") { + return { + sourceId, + semanticType: "map.moving_object", + observedAt, + attributes: { status }, + geometry: { type: "Point", coordinates: [longitude, latitude] }, + }; +} diff --git a/services/external-data-plane/test/data-product-policy.test.mjs b/services/external-data-plane/test/data-product-policy.test.mjs new file mode 100644 index 0000000..5eb8a4f --- /dev/null +++ b/services/external-data-plane/test/data-product-policy.test.mjs @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import { normalizeDataProductDefinition } from "../src/data-product-policy.mjs"; + +const input = { + id: "fleet.positions.current.v1", + version: "1.0.0", + ontologyRevision: "ontology.map.v1", + deliveryMode: "snapshot+patch", + semanticTypes: ["vehicle.trike", "map.moving_object"], + fields: ["status", "source_id", "observed_at", "geometry"], + history: { + mode: "sampled", + intervalMs: 60_000, + strategy: "latest-per-entity-per-bucket", + retentionDays: 365, + }, +}; +const definition = normalizeDataProductDefinition(input); +assert.equal(definition.history.mode, "sampled"); +assert.equal(definition.history.intervalMs, 60_000); +assert.deepEqual(definition.semanticTypes, ["map.moving_object", "vehicle.trike"]); +assert.deepEqual(definition.fields, ["geometry", "observed_at", "source_id", "status"]); +assert.deepEqual(input.semanticTypes, ["vehicle.trike", "map.moving_object"]); +assert.deepEqual(input.fields, ["status", "source_id", "observed_at", "geometry"]); +assert.equal(Object.isFrozen(definition.semanticTypes), true); +assert.equal(Object.isFrozen(definition.fields), true); + +const reordered = normalizeDataProductDefinition({ + ...input, + semanticTypes: [...input.semanticTypes].reverse(), + fields: [...input.fields].reverse(), +}); +assert.deepEqual(reordered.semanticTypes, definition.semanticTypes); +assert.deepEqual(reordered.fields, definition.fields); + +assert.throws(() => normalizeDataProductDefinition({ ...definition, providerId: "gelios" }), /data_product_definition_invalid/); +assert.throws(() => normalizeDataProductDefinition({ ...definition, history: { mode: "none", intervalMs: 1000 } }), /history_policy_none_has_sampling_fields/); + +for (const semanticTypes of [ + undefined, + "map.moving_object", + [], + ["map.moving_object", null], + ["map.moving_object", "Map.invalid"], +]) { + assert.throws( + () => normalizeDataProductDefinition({ ...input, semanticTypes }), + /data_product_definition_(semantic_types_invalid|shape_invalid)/, + ); +} +assert.throws( + () => normalizeDataProductDefinition({ ...input, semanticTypes: ["map.moving_object", "map.moving_object"] }), + /data_product_definition_semantic_types_duplicate/, +); +assert.throws( + () => normalizeDataProductDefinition({ ...input, semanticTypes: ["map.moving_object", " map.moving_object "] }), + /data_product_definition_semantic_types_duplicate/, +); + +for (const fields of [ + undefined, + "source_id", + [], + ["source_id", null], + ["source_id", "Invalid"], +]) { + assert.throws( + () => normalizeDataProductDefinition({ ...input, fields }), + /data_product_definition_(fields_invalid|shape_invalid)/, + ); +} +assert.throws( + () => normalizeDataProductDefinition({ ...input, fields: ["source_id", "source_id"] }), + /data_product_definition_fields_duplicate/, +); +assert.throws( + () => normalizeDataProductDefinition({ ...input, fields: ["source_id", " source_id "] }), + /data_product_definition_fields_duplicate/, +); + +console.log("external-data-plane data product policy: ok"); diff --git a/services/external-data-plane/test/definitions.test.mjs b/services/external-data-plane/test/definitions.test.mjs new file mode 100644 index 0000000..33e8a4f --- /dev/null +++ b/services/external-data-plane/test/definitions.test.mjs @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { loadDataProductDefinitions } from "../src/definitions.mjs"; + +const bundled = await loadDataProductDefinitions(); +assert.deepEqual(bundled.map((definition) => definition.id), ["fleet.positions.current.v1"]); +assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]); +assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1"); +assert.equal(bundled[0].history.mode, "sampled"); +assert.equal(bundled[0].history.intervalMs, 60_000); +assert.equal(bundled[0].history.retentionDays, 90); +assert.equal(bundled[0].fields.includes("geometry"), true); +assert.equal(bundled[0].fields.includes("providerUnitId"), false); +assert.equal(bundled[0].fields.includes("provider_unit_id"), false); + +const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-")); +try { + await writeFile(join(directory, "wrong-name.json"), JSON.stringify({ + ...bundled[0], + id: "another.product.v1", + })); + await assert.rejects(loadDataProductDefinitions(directory), /data_product_definition_filename_mismatch/); + await mkdir(join(directory, "ignored.json")); + await assert.rejects(loadDataProductDefinitions(directory), /data_product_definition_file_invalid/); +} finally { + await rm(directory, { recursive: true, force: true }); +} + +console.log("external-data-plane definitions: ok"); diff --git a/services/external-data-plane/test/intake-policy.test.mjs b/services/external-data-plane/test/intake-policy.test.mjs new file mode 100644 index 0000000..56a3ffe --- /dev/null +++ b/services/external-data-plane/test/intake-policy.test.mjs @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { assertBatchTimeBounds, rawRetentionExpiry } from "../src/intake-policy.mjs"; + +const now = new Date("2026-07-15T12:00:00.000Z"); +const batch = { + batch: { receivedAt: "2026-07-15T12:04:59.000Z" }, + facts: [{ observedAt: "2026-07-15T12:05:00.000Z" }], +}; + +assert.doesNotThrow(() => assertBatchTimeBounds(batch, { now, maxFutureSkewSeconds: 300 })); +assert.throws(() => assertBatchTimeBounds({ + ...batch, + batch: { receivedAt: "2026-07-15T12:05:01.000Z" }, +}, { now, maxFutureSkewSeconds: 300 }), /batch_received_at_too_far_in_future/); +assert.throws(() => assertBatchTimeBounds({ + ...batch, + facts: [{ observedAt: "2026-07-15T12:05:01.000Z" }], +}, { now, maxFutureSkewSeconds: 300 }), /fact_observed_at_too_far_in_future/); + +assert.equal( + rawRetentionExpiry({ now, rawRetentionDays: 14 }).toISOString(), + "2026-07-29T12:00:00.000Z", +); +assert.throws(() => rawRetentionExpiry({ now, rawRetentionDays: 0 }), /raw_retention_days_invalid/); + +console.log("external-data-plane intake policy: ok"); diff --git a/services/external-data-plane/test/reader-binding.test.mjs b/services/external-data-plane/test/reader-binding.test.mjs new file mode 100644 index 0000000..973d4b0 --- /dev/null +++ b/services/external-data-plane/test/reader-binding.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import { + assertReaderProduct, + createReaderToken, + hashReaderToken, + normalizeReaderBindingRequest, + safeReaderBinding, +} from "../src/reader-binding.mjs"; + +const now = new Date("2026-07-15T12:00:00.000Z"); +const policy = normalizeReaderBindingRequest({ + source: { tenantId: "tenant-01", connectionId: "connection-01", providerId: "example-provider" }, + allowedDataProductIds: ["fleet.positions.current.v1"], + expiresAt: "2026-08-01T12:00:00.000Z", +}, { now }); +const token = createReaderToken(); +assert.match(token, /^ndc_edprb_[A-Za-z0-9_-]{40,}$/); +assert.equal(hashReaderToken(token), hashReaderToken(token)); +assert.equal(assertReaderProduct({ ...policy, active: true }, "fleet.positions.current.v1", now), "fleet.positions.current.v1"); +assert.throws(() => assertReaderProduct({ ...policy, active: true }, "other.product.v1", now), /reader_binding_data_product_forbidden/); +const safe = safeReaderBinding({ id: "reader-01", ...policy, active: true, token, tokenHash: hashReaderToken(token) }); +assert.equal("token" in safe, false); +assert.equal("tokenHash" in safe, false); + +console.log("external-data-plane reader bindings: ok"); diff --git a/services/external-data-plane/test/writer-binding.test.mjs b/services/external-data-plane/test/writer-binding.test.mjs new file mode 100644 index 0000000..726515f --- /dev/null +++ b/services/external-data-plane/test/writer-binding.test.mjs @@ -0,0 +1,117 @@ +import assert from "node:assert/strict"; +import { validateIntakeBatch } from "../../../packages/external-provider-contract/src/index.mjs"; +import { + createWriterToken, + hashWriterToken, + materializeDataProductPublish, + materializeWriterBoundBatch, + normalizeWriterBindingRequest, + safeWriterBinding, +} from "../src/writer-binding.mjs"; + +const now = new Date("2026-07-15T12:00:00.000Z"); +const request = { + source: { + tenantId: "tenant-01", + connectionId: "connection-01", + providerId: "example-provider", + }, + allowedDataProductIds: ["fleet.positions.current.v1"], + expiresAt: "2026-08-01T12:00:00.000Z", +}; + +const bindingPolicy = normalizeWriterBindingRequest(request, { now, maxTtlDays: 90 }); +assert.deepEqual(bindingPolicy, { + tenantId: "tenant-01", + connectionId: "connection-01", + providerId: "example-provider", + allowedDataProductIds: ["fleet.positions.current.v1"], + expiresAt: "2026-08-01T12:00:00.000Z", +}); + +const token = createWriterToken(); +assert.match(token, /^ndc_edpwb_[A-Za-z0-9_-]{40,}$/); +assert.equal(hashWriterToken(token), hashWriterToken(token)); +assert.notEqual(hashWriterToken(token), hashWriterToken(`${token}x`)); + +const unscopedBatch = { + schemaVersion: "nodedc.external-provider-contract/v1", + source: { providerId: "example-provider" }, + contract: { dataProductId: "fleet.positions.current.v1", ontologyRevision: "example.v1", version: "1.0.0" }, + batch: { runId: "run-01", sequence: 0, idempotencyKey: "run-01.batch-0", receivedAt: "2026-07-15T12:00:00.000Z" }, + facts: [{ sourceId: "unit-01", semanticType: "map.moving_object", observedAt: "2026-07-15T12:00:00.000Z" }], +}; +const bound = materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: true }, { now }); +assert.deepEqual(bound.source, { providerId: "example-provider", tenantId: "tenant-01", connectionId: "connection-01" }); +assert.equal(validateIntakeBatch(bound).ok, true); +assert.throws(() => materializeWriterBoundBatch({ + ...unscopedBatch, + source: { ...unscopedBatch.source, tenantId: "forged-tenant" }, +}, { ...bindingPolicy, active: true }, { now }), /writer_bound_scope_forbidden/); +assert.throws(() => materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: true }, { + now, + hasScopeHeaders: true, +}), /writer_bound_scope_forbidden/); +assert.throws(() => materializeWriterBoundBatch(unscopedBatch, { + ...bindingPolicy, + providerId: "other-provider", + active: true, +}, { now }), /writer_binding_provider_forbidden/); +assert.throws(() => materializeWriterBoundBatch({ + ...unscopedBatch, + contract: { ...unscopedBatch.contract, dataProductId: "other.product.v1" }, +}, { ...bindingPolicy, active: true }, { now }), /writer_binding_data_product_forbidden/); +assert.throws(() => materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: false }, { now }), /writer_binding_inactive/); + +const materializedPublish = materializeDataProductPublish({ + schemaVersion: "nodedc.data-product.publish/v1", + batch: { runId: "run-02", sequence: 0, idempotencyKey: "run-02.batch-0" }, + facts: unscopedBatch.facts, +}, { ...bindingPolicy, active: true }, { + id: "fleet.positions.current.v1", + version: "1.0.0", + ontologyRevision: "ontology.example-fleet.v1", +}, "fleet.positions.current.v1", { now }); +assert.deepEqual(materializedPublish.source, { + tenantId: "tenant-01", + connectionId: "connection-01", + providerId: "example-provider", +}); +assert.equal(materializedPublish.batch.receivedAt, now.toISOString()); +assert.equal(materializedPublish.contract.version, "1.0.0"); +assert.equal(validateIntakeBatch(materializedPublish).ok, true); +assert.throws(() => materializeDataProductPublish({ + schemaVersion: "nodedc.data-product.publish/v1", + batch: { runId: "run-02", sequence: 0, idempotencyKey: "run-02.batch-0" }, + facts: unscopedBatch.facts, +}, { ...bindingPolicy, active: true }, { + id: "other.product.v1", + version: "1.0.0", + ontologyRevision: "ontology.example-fleet.v1", +}, "other.product.v1", { now }), /writer_binding_data_product_forbidden/); + +assert.throws(() => normalizeWriterBindingRequest({ + ...request, + expiresAt: "2027-01-01T00:00:00.000Z", +}, { now, maxTtlDays: 90 }), /writer_binding_expiry_invalid/); +assert.throws(() => normalizeWriterBindingRequest({ + ...request, + apiToken: "must-never-be-here", +}, { now, maxTtlDays: 90 }), /writer_binding_request_secret_material_forbidden/); +assert.throws(() => normalizeWriterBindingRequest({ + ...request, + arbitraryRuntimeSetting: "must-not-be-accepted", +}, { now, maxTtlDays: 90 }), /writer_binding_request_fields_invalid/); + +const safeBinding = safeWriterBinding({ + id: "binding-01", + ...bindingPolicy, + active: true, + createdAt: now, + token: "must-not-leak", + tokenHash: "must-not-leak", +}); +assert.equal("token" in safeBinding, false); +assert.equal("tokenHash" in safeBinding, false); + +console.log("external-data-plane writer bindings: ok"); diff --git a/services/gelios-gateway/Dockerfile b/services/gelios-gateway/Dockerfile new file mode 100644 index 0000000..08dcdd5 --- /dev/null +++ b/services/gelios-gateway/Dockerfile @@ -0,0 +1,23 @@ +FROM node:20-alpine AS deps + +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev + +FROM node:20-alpine AS runner + +ENV NODE_ENV=production +ENV PORT=18105 + +WORKDIR /app + +COPY --from=deps /app/package.json /app/package-lock.json ./ +COPY --from=deps /app/node_modules ./node_modules +COPY src ./src + +USER node + +EXPOSE 18105 + +CMD ["node", "src/server.mjs"] diff --git a/services/gelios-gateway/README.md b/services/gelios-gateway/README.md new file mode 100644 index 0000000..d279b64 --- /dev/null +++ b/services/gelios-gateway/README.md @@ -0,0 +1,70 @@ +# NODE.DC Gelios Gateway + +Storage and read-model service for the Gelios provider domain. It has no +provider credential, no provider HTTP client, no scheduler and no command +transport. + +## Boundary + +The protected Engine L2 Gelios Collector is the only component that reaches +the provider and uses the Engine Credential. It submits authenticated unit +intake to this Gateway. The Gateway enforces the connection scope and approved +unit allowlist, persists current state/history, and exposes scoped read/SSE +contracts to other Engine workflows and map consumers. + +Neither a provider token nor a credential reference is accepted, stored or +returned by this service. + +## Runtime contracts + +`GET /healthz` is unprotected and contains no secret material. + +All `/internal/gelios/v1/*` routes require the internal service credential: + +```text +Authorization: Bearer +X-NODEDC-Tenant-Id: +X-NODEDC-Connection-Id: +``` + +```text +POST /internal/gelios/v1/intake/units +GET /internal/gelios/v1/status +GET /internal/gelios/v1/capabilities +GET /internal/gelios/v1/units/current +GET /internal/gelios/v1/stream +GET /internal/gelios/v1/data-products/fleet.positions.current.v1/snapshot +GET /internal/gelios/v1/data-products/fleet.positions.current.v1/stream +``` + +`intake/units` is disabled by default. It accepts a bounded payload with a +`units` array or an equivalent safe unit-list envelope; it does not accept a +URL, method, token or command. The command catalog remains visible only as +disabled ontology metadata. + +The connection policy is explicit: `GELIOS_UNIT_SCOPE=allowlist` uses fixed +`GELIOS_ALLOWED_UNIT_IDS`; `GELIOS_UNIT_SCOPE=all` accepts every unit returned +by that already-approved tenant + connection, including units added later by +the provider. Missing units are never deleted from storage: their stable +provider IDs and last-seen history remain available to the presentation layer. + +`fleet.positions.current.v1` is the first provider-neutral product. Its +snapshot and SSE patch stream contain semantic moving objects, position, +motion, quality and `active` / `stale` / `no-position` state. They deliberately +exclude raw telemetry and provider transport payloads. A consumer first fetches +the snapshot and then attaches the stream; the stream `ready` event never +pretends to be a complete current-state snapshot. + +## Storage + +The database requires TimescaleDB and PostGIS. It keeps a current projection, +append-only time-series snapshots, bounded restricted intake envelopes and +collection audit. Raw envelopes have no browser/API route and expire according +to `GELIOS_RAW_RETENTION_DAYS`. + +## Local checks + +```bash +npm ci +npm run smoke +``` diff --git a/services/gelios-gateway/package-lock.json b/services/gelios-gateway/package-lock.json new file mode 100644 index 0000000..1092436 --- /dev/null +++ b/services/gelios-gateway/package-lock.json @@ -0,0 +1,1008 @@ +{ + "name": "@nodedc/gelios-gateway", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@nodedc/gelios-gateway", + "version": "0.1.0", + "dependencies": { + "express": "^5.2.1", + "pg": "^8.18.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/services/gelios-gateway/package.json b/services/gelios-gateway/package.json new file mode 100644 index 0000000..5a9d05e --- /dev/null +++ b/services/gelios-gateway/package.json @@ -0,0 +1,15 @@ +{ + "name": "@nodedc/gelios-gateway", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/server.mjs", + "dev": "node --watch src/server.mjs", + "smoke": "node src/scripts/smoke.mjs" + }, + "dependencies": { + "express": "^5.2.1", + "pg": "^8.18.0" + } +} diff --git a/services/gelios-gateway/src/capabilities.mjs b/services/gelios-gateway/src/capabilities.mjs new file mode 100644 index 0000000..5d08490 --- /dev/null +++ b/services/gelios-gateway/src/capabilities.mjs @@ -0,0 +1,26 @@ +export const GELIOS_CAPABILITIES = Object.freeze([ + { + id: "gelios.units.current_intake", + class: "internal-safe-intake", + enabled: true, + summary: "Authenticated normalized current-unit intake from the protected Engine Collector.", + }, + { + id: "fleet.positions.current.v1", + class: "internal-data-product", + enabled: true, + summary: "Provider-neutral current fleet positions snapshot and patch stream; no raw telemetry fields.", + }, + { + id: "gelios.commands.catalog", + class: "red-read-catalogue", + enabled: false, + summary: "Catalogued for ontology only; not connected to the collector or Gateway.", + }, + { + id: "gelios.commands.send", + class: "red-write", + enabled: false, + summary: "No transport is implemented.", + }, +]); diff --git a/services/gelios-gateway/src/config.mjs b/services/gelios-gateway/src/config.mjs new file mode 100644 index 0000000..9d0440c --- /dev/null +++ b/services/gelios-gateway/src/config.mjs @@ -0,0 +1,80 @@ +export function readConfig(env = process.env) { + const rawRetentionDays = integer(env.GELIOS_RAW_RETENTION_DAYS, 14, 1, 3650); + + return { + nodeEnv: string(env.NODE_ENV, "development"), + port: integer(env.PORT, 18105, 1, 65535), + databaseUrl: required(env.DATABASE_URL, "DATABASE_URL"), + databasePoolSize: integer(env.GELIOS_DATABASE_POOL_SIZE, 10, 1, 50), + internalAccessToken: optional(env.NODEDC_INTERNAL_ACCESS_TOKEN), + tenantId: slug(required(env.GELIOS_TENANT_ID, "GELIOS_TENANT_ID"), "GELIOS_TENANT_ID"), + connectionId: slug(required(env.GELIOS_CONNECTION_ID, "GELIOS_CONNECTION_ID"), "GELIOS_CONNECTION_ID"), + unitScope: collectionScope(env.GELIOS_UNIT_SCOPE), + allowedUnitIds: intList(env.GELIOS_ALLOWED_UNIT_IDS), + intakeEnabled: boolean(env.GELIOS_INTAKE_ENABLED, false), + rawRetentionDays, + positionStaleAfterMs: integer(env.GELIOS_POSITION_STALE_AFTER_MS, 300_000, 1_000, 86_400_000), + }; +} + +function collectionScope(value) { + const candidate = optional(value).toLowerCase() || "allowlist"; + if (["allowlist", "all"].includes(candidate)) return candidate; + throw new Error("GELIOS_UNIT_SCOPE_must_be_allowlist_or_all"); +} + +export function unitIsInScope(config, sourceUnitId) { + return config?.unitScope === "all" || config?.allowedUnitIds?.includes(sourceUnitId) === true; +} + +function required(value, name) { + const normalized = optional(value); + if (!normalized) throw new Error(`${name}_required`); + return normalized; +} + +function optional(value) { + const normalized = String(value ?? "").trim(); + return normalized || ""; +} + +function string(value, fallback) { + return optional(value) || fallback; +} + +function integer(value, fallback, min, max) { + const candidate = optional(value); + if (!candidate) return fallback; + const number = Number.parseInt(candidate, 10); + if (!Number.isInteger(number) || number < min || number > max) { + throw new Error(`invalid_integer:${candidate}`); + } + return number; +} + +function boolean(value, fallback) { + const candidate = optional(value).toLowerCase(); + if (!candidate) return fallback; + if (["1", "true", "yes", "on"].includes(candidate)) return true; + if (["0", "false", "no", "off"].includes(candidate)) return false; + throw new Error(`invalid_boolean:${candidate}`); +} + +function intList(value) { + const values = optional(value) + .split(",") + .map((item) => item.trim()) + .filter(Boolean) + .map((item) => Number.parseInt(item, 10)); + if (values.some((item) => !Number.isInteger(item) || item <= 0)) { + throw new Error("GELIOS_ALLOWED_UNIT_IDS_must_contain_positive_integers"); + } + return [...new Set(values)].sort((left, right) => left - right); +} + +function slug(value, name) { + if (!/^[a-z0-9][a-z0-9_-]{0,95}$/i.test(value)) { + throw new Error(`${name}_invalid`); + } + return value; +} diff --git a/services/gelios-gateway/src/data-products.mjs b/services/gelios-gateway/src/data-products.mjs new file mode 100644 index 0000000..a004fb3 --- /dev/null +++ b/services/gelios-gateway/src/data-products.mjs @@ -0,0 +1,80 @@ +export const FLEET_POSITIONS_CURRENT = Object.freeze({ + id: "fleet.positions.current.v1", + version: "1.0.0", + delivery: "snapshot+patch", + semanticTypes: Object.freeze(["map.moving_object", "geo.position"]), +}); + +export function toFleetPositionsSnapshot(rows, { tenantId, connectionId, generatedAt = new Date(), staleAfterMs }) { + return { + schemaVersion: "nodedc.data-product.snapshot.v1", + dataProduct: FLEET_POSITIONS_CURRENT, + scope: { tenantId, connectionId }, + generatedAt: iso(generatedAt), + entities: rows.map((row) => toFleetPositionEntity(row, { generatedAt, staleAfterMs })), + }; +} + +export function toFleetPositionPatch(row, { generatedAt = new Date(), staleAfterMs }) { + return { + schemaVersion: "nodedc.data-product.patch.v1", + dataProductId: FLEET_POSITIONS_CURRENT.id, + operation: "upsert", + emittedAt: iso(generatedAt), + entity: toFleetPositionEntity(row, { generatedAt, staleAfterMs }), + }; +} + +export function toFleetPositionEntity(row, { generatedAt, staleAfterMs }) { + const observedAt = date(row.observedAt); + const receivedAt = date(row.receivedAt); + const latitude = finiteNumber(row.latitude); + const longitude = finiteNumber(row.longitude); + const hasPosition = latitude !== null && longitude !== null; + const ageMs = observedAt ? Math.max(0, generatedAt.getTime() - observedAt.getTime()) : null; + const status = !hasPosition ? "no-position" : ageMs !== null && ageMs > staleAfterMs ? "stale" : "active"; + + return { + subjectId: String(row.subjectId), + semanticType: "map.moving_object", + label: String(row.name || row.subjectId), + status, + observedAt: observedAt ? observedAt.toISOString() : null, + receivedAt: receivedAt ? receivedAt.toISOString() : null, + position: hasPosition ? { latitude, longitude } : null, + motion: { + speed: finiteNumber(row.speed), + course: finiteNumber(row.course), + }, + quality: { + gpsValid: booleanOrNull(row.gpsValid), + satellites: finiteNumber(row.satellites), + hdop: finiteNumber(row.hdop), + accuracyM: finiteNumber(row.accuracyM), + }, + }; +} + +function iso(value) { + const parsed = date(value); + if (!parsed) throw new Error("data_product_timestamp_invalid"); + return parsed.toISOString(); +} + +function date(value) { + if (value instanceof Date && !Number.isNaN(value.getTime())) return value; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function finiteNumber(value) { + if (value === null || value === undefined || value === "") return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function booleanOrNull(value) { + if (value === true || value === 1 || value === "1" || value === "true") return true; + if (value === false || value === 0 || value === "0" || value === "false") return false; + return null; +} diff --git a/services/gelios-gateway/src/normalize.mjs b/services/gelios-gateway/src/normalize.mjs new file mode 100644 index 0000000..ab9c7f0 --- /dev/null +++ b/services/gelios-gateway/src/normalize.mjs @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto"; + +export function normalizeUnit(unit, { tenantId, connectionId, receivedAt = new Date() }) { + const sourceUnitId = positiveInteger(first(unit?.id, unit?.unit_id, unit?.unitId)); + if (!sourceUnitId) return null; + + const lastMessage = firstObject(unit?.lmsg, unit?.last_msg, unit?.lastMsg, unit?.lastMessage) || {}; + const latitude = finiteNumber(first(lastMessage.lat, lastMessage.latitude, unit?.lat, unit?.latitude)); + const longitude = finiteNumber(first(lastMessage.lon, lastMessage.lng, lastMessage.longitude, unit?.lon, unit?.lng, unit?.longitude)); + const observedAt = timestamp(first(lastMessage.time, lastMessage.ts, lastMessage.timestamp, unit?.last_msg_time, unit?.lmsg_time), receivedAt); + const telemetry = { + sourceUnitId, + name: optionalString(first(unit?.name, unit?.unit_name)), + observedAt: observedAt.toISOString(), + latitude, + longitude, + speed: finiteNumber(first(lastMessage.speed, unit?.speed)), + course: finiteNumber(first(lastMessage.course, lastMessage.angle, unit?.course)), + gpsValid: booleanOrNull(first(lastMessage.gps_valid, lastMessage.gpsValid, unit?.gps_valid)), + satellites: finiteNumber(first(lastMessage.sats, lastMessage.satellites, unit?.sats)), + hdop: finiteNumber(first(lastMessage.hdop, unit?.hdop)), + accuracyM: finiteNumber(first(lastMessage.accuracy, lastMessage.accuracy_m, unit?.accuracy_m)), + }; + + return { + tenantId, + connectionId, + sourceUnitId, + stableSubjectId: `gelios.unit:${connectionId}:${sourceUnitId}`, + name: telemetry.name || `Gelios unit ${sourceUnitId}`, + observedAt, + receivedAt, + latitude, + longitude, + speed: telemetry.speed, + course: telemetry.course, + gpsValid: telemetry.gpsValid, + satellites: telemetry.satellites, + hdop: telemetry.hdop, + accuracyM: telemetry.accuracyM, + telemetry, + fingerprint: hash(telemetry), + }; +} + +export function extractUnitItems(payload) { + if (Array.isArray(payload?.items)) return payload.items; + if (Array.isArray(payload?.units)) return payload.units; + if (Array.isArray(payload?.data)) return payload.data; + return []; +} + +export function hash(value) { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function first(...values) { + return values.find((value) => value !== undefined && value !== null && value !== ""); +} + +function firstObject(...values) { + return values.find((value) => value && typeof value === "object" && !Array.isArray(value)); +} + +function positiveInteger(value) { + const parsed = Number.parseInt(value, 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +} + +function finiteNumber(value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function optionalString(value) { + const normalized = String(value ?? "").trim(); + return normalized || null; +} + +function booleanOrNull(value) { + if (value === true || value === 1 || value === "1" || value === "true") return true; + if (value === false || value === 0 || value === "0" || value === "false") return false; + return null; +} + +function timestamp(value, fallback) { + const numeric = Number(value); + if (!Number.isFinite(numeric) || numeric <= 0) return fallback; + const milliseconds = numeric < 1_000_000_000_000 ? numeric * 1000 : numeric; + const parsed = new Date(milliseconds); + return Number.isNaN(parsed.getTime()) ? fallback : parsed; +} diff --git a/services/gelios-gateway/src/schema.mjs b/services/gelios-gateway/src/schema.mjs new file mode 100644 index 0000000..cbe6b3b --- /dev/null +++ b/services/gelios-gateway/src/schema.mjs @@ -0,0 +1,116 @@ +export async function migrate(pool) { + await pool.query("create extension if not exists timescaledb"); + await pool.query("create extension if not exists postgis"); + + await pool.query(` + create table if not exists gelios_connections ( + tenant_id text not null, + connection_id text not null, + provider_id text not null default 'gelios', + unit_scope text not null default 'allowlist', + allowed_unit_ids jsonb not null default '[]'::jsonb, + collection_enabled boolean not null default false, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + primary key (tenant_id, connection_id) + ) + `); + await pool.query("alter table gelios_connections add column if not exists unit_scope text not null default 'allowlist'"); + + await pool.query(` + create table if not exists gelios_collection_runs ( + id uuid primary key, + tenant_id text not null, + connection_id text not null, + capability_id text not null, + status text not null, + unit_count integer not null default 0, + inserted_snapshots integer not null default 0, + error_code text, + started_at timestamptz not null default now(), + completed_at timestamptz, + foreign key (tenant_id, connection_id) + references gelios_connections (tenant_id, connection_id) + on delete cascade + ) + `); + + await pool.query(` + create table if not exists gelios_raw_envelopes ( + id uuid primary key, + tenant_id text not null, + connection_id text not null, + collection_run_id uuid references gelios_collection_runs(id) on delete set null, + capability_id text not null, + payload_hash text not null, + payload jsonb not null, + payload_bytes integer not null, + received_at timestamptz not null, + expires_at timestamptz not null, + unique (tenant_id, connection_id, capability_id, payload_hash) + ) + `); + await pool.query("create index if not exists gelios_raw_envelopes_expiry_idx on gelios_raw_envelopes (expires_at)"); + + await pool.query(` + create table if not exists gelios_units ( + tenant_id text not null, + connection_id text not null, + source_unit_id bigint not null, + stable_subject_id text not null, + name text not null, + first_seen_at timestamptz not null default now(), + last_seen_at timestamptz not null default now(), + primary key (tenant_id, connection_id, source_unit_id), + unique (tenant_id, connection_id, stable_subject_id), + foreign key (tenant_id, connection_id) + references gelios_connections (tenant_id, connection_id) + on delete cascade + ) + `); + + await pool.query(` + create table if not exists gelios_unit_current ( + tenant_id text not null, + connection_id text not null, + source_unit_id bigint not null, + stable_subject_id text not null, + name text not null, + observed_at timestamptz not null, + received_at timestamptz not null, + latitude double precision, + longitude double precision, + position geography(Point, 4326), + speed double precision, + course double precision, + gps_valid boolean, + satellites double precision, + hdop double precision, + accuracy_m double precision, + telemetry jsonb not null, + fingerprint text not null, + updated_at timestamptz not null default now(), + primary key (tenant_id, connection_id, source_unit_id), + foreign key (tenant_id, connection_id, source_unit_id) + references gelios_units (tenant_id, connection_id, source_unit_id) + on delete cascade + ) + `); + await pool.query("create index if not exists gelios_unit_current_position_idx on gelios_unit_current using gist (position)"); + await pool.query("create index if not exists gelios_unit_current_observed_idx on gelios_unit_current (tenant_id, connection_id, observed_at desc)"); + + await pool.query(` + create table if not exists gelios_telemetry_snapshots ( + tenant_id text not null, + connection_id text not null, + source_unit_id bigint not null, + observed_at timestamptz not null, + received_at timestamptz not null, + fingerprint text not null, + telemetry jsonb not null, + primary key (tenant_id, connection_id, source_unit_id, observed_at, fingerprint) + ) + `); + await pool.query("select create_hypertable('gelios_telemetry_snapshots', 'observed_at', if_not_exists => true, migrate_data => true)"); + await pool.query("create index if not exists gelios_snapshots_unit_time_idx on gelios_telemetry_snapshots (tenant_id, connection_id, source_unit_id, observed_at desc)"); +} diff --git a/services/gelios-gateway/src/scripts/smoke.mjs b/services/gelios-gateway/src/scripts/smoke.mjs new file mode 100644 index 0000000..7d0c57d --- /dev/null +++ b/services/gelios-gateway/src/scripts/smoke.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { GELIOS_CAPABILITIES } from "../capabilities.mjs"; +import { readConfig, unitIsInScope } from "../config.mjs"; +import { toFleetPositionPatch, toFleetPositionsSnapshot } from "../data-products.mjs"; +import { extractUnitItems, normalizeUnit } from "../normalize.mjs"; + +const unit = normalizeUnit({ + id: 42, + name: "Sample fleet fixture", + lmsg: { time: 1_789_123_456, lat: 55.75, lon: 37.61, speed: 12.5, gps_valid: true, sats: 11 }, +}, { tenantId: "sample-tenant", connectionId: "gelios-sample", receivedAt: new Date("2026-07-13T00:00:00.000Z") }); +assert.equal(unit.stableSubjectId, "gelios.unit:gelios-sample:42"); +assert.equal(unit.latitude, 55.75); +assert.equal(unit.longitude, 37.61); +assert.equal(unit.gpsValid, true); +assert.equal(extractUnitItems({ items: [{ id: 1 }] }).length, 1); +assert.equal(extractUnitItems({ units: [{ id: 1 }] }).length, 1); +assert.equal(GELIOS_CAPABILITIES.some((item) => item.enabled === true && item.class.includes("write")), false); +assert.equal(GELIOS_CAPABILITIES.some((item) => item.id === "gelios.commands.send" && item.enabled === false), true); +const snapshot = toFleetPositionsSnapshot([unit], { + tenantId: "sample-tenant", + connectionId: "gelios-sample", + generatedAt: new Date("2027-09-01T00:00:00.000Z"), + staleAfterMs: 300_000, +}); +assert.equal(snapshot.dataProduct.id, "fleet.positions.current.v1"); +assert.equal(snapshot.entities[0].semanticType, "map.moving_object"); +assert.equal(snapshot.entities[0].status, "stale"); +assert.equal(Object.hasOwn(snapshot.entities[0], "telemetry"), false); +assert.equal(toFleetPositionPatch({ ...unit, latitude: null, longitude: null }, { + generatedAt: new Date("2026-07-13T00:00:00.000Z"), + staleAfterMs: 300_000, +}).entity.status, "no-position"); + +const configBase = { + DATABASE_URL: "postgresql://fixture", + GELIOS_TENANT_ID: "fixture-tenant", + GELIOS_CONNECTION_ID: "fixture-connection", + GELIOS_INTAKE_ENABLED: "true", +}; +const allUnitsConfig = readConfig({ ...configBase, GELIOS_UNIT_SCOPE: "all" }); +assert.equal(allUnitsConfig.unitScope, "all"); +assert.equal(unitIsInScope(allUnitsConfig, 1), true); +assert.equal(unitIsInScope(allUnitsConfig, 99_999), true); +const allowlistConfig = readConfig({ ...configBase, GELIOS_UNIT_SCOPE: "allowlist", GELIOS_ALLOWED_UNIT_IDS: "7,11" }); +assert.equal(unitIsInScope(allowlistConfig, 7), true); +assert.equal(unitIsInScope(allowlistConfig, 8), false); +assert.throws(() => readConfig({ ...configBase, GELIOS_UNIT_SCOPE: "everything" }), /GELIOS_UNIT_SCOPE/); + +console.log(JSON.stringify({ + ok: true, + checks: [ + "unit_normalization", + "safe_intake_registry", + "provider_neutral_data_product", + "all_units_connection_scope", + "command_transport_absent", + ], +})); diff --git a/services/gelios-gateway/src/server.mjs b/services/gelios-gateway/src/server.mjs new file mode 100644 index 0000000..e75e2c9 --- /dev/null +++ b/services/gelios-gateway/src/server.mjs @@ -0,0 +1,406 @@ +import express from "express"; +import { randomUUID, timingSafeEqual } from "node:crypto"; +import { createServer } from "node:http"; +import { Pool } from "pg"; +import { GELIOS_CAPABILITIES } from "./capabilities.mjs"; +import { readConfig, unitIsInScope } from "./config.mjs"; +import { FLEET_POSITIONS_CURRENT, toFleetPositionPatch, toFleetPositionsSnapshot } from "./data-products.mjs"; +import { extractUnitItems, hash, normalizeUnit } from "./normalize.mjs"; +import { migrate } from "./schema.mjs"; + +const config = readConfig(); +const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize }); +const app = express(); +const httpServer = createServer(app); +const streamClients = new Set(); +let intakeInFlight = null; + +app.disable("x-powered-by"); +app.use(express.json({ limit: "1mb" })); + +app.get("/healthz", asyncRoute(async (_req, res) => { + await pool.query("select 1"); + res.json({ + ok: true, + service: "nodedc-gelios-gateway", + database: "ready", + internalApiConfigured: Boolean(config.internalAccessToken), + intakeEnabled: config.intakeEnabled, + unitScope: config.unitScope, + credentialOwner: "engine", + providerTransport: "absent", + commandTransport: "absent", + }); +})); + +app.get("/internal/gelios/v1/status", requireInternalApi, asyncRoute(async (req, res) => { + const scope = requireConnectionScope(req); + const latestRun = await pool.query( + `select id, capability_id, status, unit_count, inserted_snapshots, error_code, started_at, completed_at + from gelios_collection_runs + where tenant_id = $1 and connection_id = $2 + order by started_at desc + limit 1`, + [scope.tenantId, scope.connectionId], + ); + res.json({ + ok: true, + provider: "gelios", + tenantId: scope.tenantId, + connectionId: scope.connectionId, + unitScope: config.unitScope, + approvedUnitCount: config.unitScope === "all" ? null : config.allowedUnitIds.length, + intakeEnabled: config.intakeEnabled, + credentialOwner: "engine", + providerTransport: "absent", + commandTransport: "absent", + lastRun: latestRun.rows[0] || null, + }); +})); + +app.get("/internal/gelios/v1/capabilities", requireInternalApi, (_req, res) => { + res.json({ ok: true, provider: "gelios", capabilities: GELIOS_CAPABILITIES, commandTransport: "absent" }); +}); + +app.post("/internal/gelios/v1/intake/units", requireInternalApi, asyncRoute(async (req, res) => { + const scope = requireConnectionScope(req); + const result = await runIntake({ + scope, + payload: req.body, + receivedAt: parseReceivedAt(req.body?.receivedAt), + }); + res.status(result.status === "completed" ? 200 : 409).json(result); +})); + +app.get("/internal/gelios/v1/units/current", requireInternalApi, asyncRoute(async (req, res) => { + const scope = requireConnectionScope(req); + const limit = boundedLimit(req.query.limit, 200, 1, 1000); + const rows = await listCurrentRows(scope, limit); + res.json({ + ok: true, + contract: "gelios.current-position.v1", + tenantId: scope.tenantId, + connectionId: scope.connectionId, + units: rows, + }); +})); + +app.get("/internal/gelios/v1/data-products/fleet.positions.current.v1/snapshot", requireInternalApi, asyncRoute(async (req, res) => { + const scope = requireConnectionScope(req); + const limit = boundedLimit(req.query.limit, 200, 1, 1000); + const rows = await listCurrentRows(scope, limit); + res.json({ + ok: true, + ...toFleetPositionsSnapshot(rows, { ...scope, staleAfterMs: config.positionStaleAfterMs }), + }); +})); + +app.get("/internal/gelios/v1/data-products/fleet.positions.current.v1/stream", requireInternalApi, (req, res) => { + let scope; + try { + scope = requireConnectionScope(req); + } catch (error) { + res.status(Number(error.status || 400)).json({ ok: false, error: error.message }); + return; + } + const client = openSseClient(req, res, scope, "data-product"); + writeSse(res, "nodedc.data-product.ready.v1", { + dataProduct: FLEET_POSITIONS_CURRENT, + scope, + bootstrap: "fetch-snapshot", + }); + return client; +}); + +app.get("/internal/gelios/v1/stream", requireInternalApi, (req, res) => { + let scope; + try { + scope = requireConnectionScope(req); + } catch (error) { + res.status(Number(error.status || 400)).json({ ok: false, error: error.message }); + return; + } + const client = openSseClient(req, res, scope, "legacy"); + writeSse(res, "gelios.ready", { + ok: true, + contract: "gelios.current-position.v1", + tenantId: scope.tenantId, + connectionId: scope.connectionId, + commandTransport: "absent", + }); + return client; +}); + +async function listCurrentRows(scope, limit) { + const rows = await pool.query( + `select + stable_subject_id as "subjectId", source_unit_id as "sourceUnitId", name, + observed_at as "observedAt", received_at as "receivedAt", latitude, longitude, + speed, course, gps_valid as "gpsValid", satellites, hdop, + accuracy_m as "accuracyM", telemetry + from gelios_unit_current + where tenant_id = $1 and connection_id = $2 + order by observed_at desc, source_unit_id asc + limit $3`, + [scope.tenantId, scope.connectionId, limit], + ); + return rows.rows; +} + +app.use((error, _req, res, _next) => { + const status = Number(error?.status || 500); + const publicStatus = status >= 400 && status < 600 ? status : 500; + console.error(JSON.stringify({ event: "gelios_gateway_error", error: safeErrorCode(error), status: publicStatus })); + res.status(publicStatus).json({ ok: false, error: publicStatus >= 500 ? "internal_error" : safeErrorCode(error) }); +}); + +await migrate(pool); +await upsertConnection(); + +httpServer.listen(config.port, "0.0.0.0", () => { + console.log(`NODE.DC Gelios Gateway listening on http://0.0.0.0:${config.port}`); +}); + +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); + +async function upsertConnection() { + await pool.query( + `insert into gelios_connections ( + tenant_id, connection_id, unit_scope, allowed_unit_ids, collection_enabled, updated_at + ) values ($1, $2, $3, $4::jsonb, $5, now()) + on conflict (tenant_id, connection_id) do update + set unit_scope = excluded.unit_scope, + allowed_unit_ids = excluded.allowed_unit_ids, + collection_enabled = excluded.collection_enabled, + updated_at = now()`, + [config.tenantId, config.connectionId, config.unitScope, JSON.stringify(config.allowedUnitIds), config.intakeEnabled], + ); +} + +async function runIntake({ scope, payload, receivedAt }) { + if (!config.intakeEnabled) return { ok: false, status: "disabled" }; + if (config.unitScope === "allowlist" && !config.allowedUnitIds.length) { + return { ok: false, status: "scope_missing" }; + } + if (intakeInFlight) return { ok: false, status: "busy" }; + intakeInFlight = ingestCurrentUnits({ scope, payload, receivedAt }).finally(() => { + intakeInFlight = null; + }); + return intakeInFlight; +} + +async function ingestCurrentUnits({ scope, payload, receivedAt }) { + const runId = randomUUID(); + await pool.query( + `insert into gelios_collection_runs (id, tenant_id, connection_id, capability_id, status) + values ($1, $2, $3, 'gelios.units.current_intake', 'running')`, + [runId, scope.tenantId, scope.connectionId], + ); + + try { + const sourcePayload = payload?.units ? { units: payload.units } : payload; + const items = extractUnitItems(sourcePayload); + if (!items.length) throw Object.assign(new Error("intake_units_required"), { status: 422 }); + await storeRawEnvelope({ runId, scope, payload: sourcePayload, receivedAt }); + const normalized = items + .map((item) => normalizeUnit(item, { ...scope, receivedAt })) + .filter(Boolean) + .filter((item) => unitIsInScope(config, item.sourceUnitId)); + + let insertedSnapshots = 0; + for (const unit of normalized) { + const result = await persistCurrentUnit(unit); + insertedSnapshots += result.insertedSnapshot ? 1 : 0; + if (result.current) broadcastCurrentUnit(result.current, scope.tenantId, scope.connectionId); + } + await pruneExpiredRawEnvelopes(); + await pool.query( + `update gelios_collection_runs + set status = 'completed', unit_count = $2, inserted_snapshots = $3, completed_at = now() + where id = $1`, + [runId, normalized.length, insertedSnapshots], + ); + return { ok: true, status: "completed", runId, unitCount: normalized.length, insertedSnapshots }; + } catch (error) { + await pool.query( + `update gelios_collection_runs + set status = 'failed', error_code = $2, completed_at = now() + where id = $1`, + [runId, safeErrorCode(error)], + ); + throw error; + } +} + +async function storeRawEnvelope({ runId, scope, payload, receivedAt }) { + const serialized = JSON.stringify(payload); + await pool.query( + `insert into gelios_raw_envelopes ( + id, tenant_id, connection_id, collection_run_id, capability_id, + payload_hash, payload, payload_bytes, received_at, expires_at + ) values ($1, $2, $3, $4, 'gelios.units.current_intake', $5, $6::jsonb, $7, $8, $9) + on conflict (tenant_id, connection_id, capability_id, payload_hash) do nothing`, + [ + randomUUID(), scope.tenantId, scope.connectionId, runId, hash(payload), serialized, + Buffer.byteLength(serialized), receivedAt, + new Date(receivedAt.getTime() + config.rawRetentionDays * 24 * 60 * 60 * 1000), + ], + ); +} + +async function persistCurrentUnit(unit) { + const client = await pool.connect(); + try { + await client.query("begin"); + await client.query( + `insert into gelios_units ( + tenant_id, connection_id, source_unit_id, stable_subject_id, name, last_seen_at + ) values ($1, $2, $3, $4, $5, $6) + on conflict (tenant_id, connection_id, source_unit_id) do update + set stable_subject_id = excluded.stable_subject_id, + name = excluded.name, + last_seen_at = excluded.last_seen_at`, + [unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.stableSubjectId, unit.name, unit.receivedAt], + ); + const current = await client.query( + `insert into gelios_unit_current ( + tenant_id, connection_id, source_unit_id, stable_subject_id, name, + observed_at, received_at, latitude, longitude, position, speed, course, + gps_valid, satellites, hdop, accuracy_m, telemetry, fingerprint + ) values ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, + case when $8::double precision is not null and $9::double precision is not null + then ST_SetSRID(ST_MakePoint($9, $8), 4326)::geography else null end, + $10, $11, $12, $13, $14, $15, $16::jsonb, $17 + ) + on conflict (tenant_id, connection_id, source_unit_id) do update + set stable_subject_id = excluded.stable_subject_id, + name = excluded.name, + observed_at = excluded.observed_at, + received_at = excluded.received_at, + latitude = excluded.latitude, + longitude = excluded.longitude, + position = excluded.position, + speed = excluded.speed, + course = excluded.course, + gps_valid = excluded.gps_valid, + satellites = excluded.satellites, + hdop = excluded.hdop, + accuracy_m = excluded.accuracy_m, + telemetry = excluded.telemetry, + fingerprint = excluded.fingerprint, + updated_at = now() + where excluded.observed_at >= gelios_unit_current.observed_at + returning stable_subject_id as "subjectId", source_unit_id as "sourceUnitId", name, + observed_at as "observedAt", received_at as "receivedAt", latitude, longitude, + speed, course, gps_valid as "gpsValid", satellites, hdop, accuracy_m as "accuracyM", telemetry`, + [ + unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.stableSubjectId, unit.name, + unit.observedAt, unit.receivedAt, unit.latitude, unit.longitude, unit.speed, unit.course, + unit.gpsValid, unit.satellites, unit.hdop, unit.accuracyM, JSON.stringify(unit.telemetry), unit.fingerprint, + ], + ); + const snapshot = await client.query( + `insert into gelios_telemetry_snapshots ( + tenant_id, connection_id, source_unit_id, observed_at, received_at, fingerprint, telemetry + ) values ($1, $2, $3, $4, $5, $6, $7::jsonb) + on conflict do nothing`, + [unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.observedAt, unit.receivedAt, unit.fingerprint, JSON.stringify(unit.telemetry)], + ); + await client.query("commit"); + return { current: current.rows[0] || null, insertedSnapshot: snapshot.rowCount > 0 }; + } catch (error) { + await client.query("rollback"); + throw error; + } finally { + client.release(); + } +} + +async function pruneExpiredRawEnvelopes() { + await pool.query("delete from gelios_raw_envelopes where expires_at < now()"); +} + +function requireInternalApi(req, res, next) { + const expected = Buffer.from(config.internalAccessToken || ""); + const received = Buffer.from(String(req.get("authorization") || "").replace(/^Bearer\s+/i, "").trim()); + if (!expected.length) { + res.status(503).json({ ok: false, error: "internal_api_not_configured" }); + return; + } + if (expected.length !== received.length || !timingSafeEqual(expected, received)) { + res.status(401).json({ ok: false, error: "internal_authorization_required" }); + return; + } + next(); +} + +function requireConnectionScope(req) { + const tenantId = String(req.get("x-nodedc-tenant-id") || "").trim(); + const connectionId = String(req.query.connectionId || req.get("x-nodedc-connection-id") || config.connectionId).trim(); + if (tenantId !== config.tenantId) throw Object.assign(new Error("tenant_scope_required"), { status: 403 }); + if (connectionId !== config.connectionId) throw Object.assign(new Error("connection_scope_denied"), { status: 403 }); + return { tenantId, connectionId }; +} + +function broadcastCurrentUnit(unit, tenantId, connectionId) { + for (const client of streamClients) { + if (client.tenantId === tenantId && client.connectionId === connectionId) { + if (client.kind === "data-product") { + writeSse(client.res, "nodedc.data-product.patch.v1", toFleetPositionPatch(unit, { + staleAfterMs: config.positionStaleAfterMs, + })); + } else { + writeSse(client.res, "gelios.current.updated", unit); + } + } + } +} + +function openSseClient(req, res, scope, kind) { + const client = { id: randomUUID(), res, kind, ...scope }; + res.setHeader("Content-Type", "text/event-stream"); + res.setHeader("Cache-Control", "no-cache, no-transform"); + res.setHeader("Connection", "keep-alive"); + res.setHeader("X-Accel-Buffering", "no"); + res.flushHeaders?.(); + const keepAlive = setInterval(() => res.write(": keep-alive\n\n"), 30000); + streamClients.add(client); + req.on("close", () => { + clearInterval(keepAlive); + streamClients.delete(client); + }); + return client; +} + +function writeSse(res, event, payload) { + res.write(`event: ${event}\ndata: ${JSON.stringify(payload)}\n\n`); +} + +function parseReceivedAt(value) { + if (value === undefined || value === null || value === "") return new Date(); + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) throw Object.assign(new Error("intake_received_at_invalid"), { status: 422 }); + return parsed; +} + +function boundedLimit(value, fallback, min, max) { + const parsed = Number.parseInt(String(value || ""), 10); + if (!Number.isInteger(parsed)) return fallback; + return Math.max(min, Math.min(max, parsed)); +} + +function safeErrorCode(error) { + return String(error?.message || "internal_error").replace(/[^a-zA-Z0-9_:.-]+/g, "_").slice(0, 160); +} + +function asyncRoute(handler) { + return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next); +} + +async function shutdown() { + await new Promise((resolve) => httpServer.close(resolve)); + await pool.end(); + process.exit(0); +} diff --git a/services/ontology-core/Dockerfile b/services/ontology-core/Dockerfile new file mode 100644 index 0000000..8a1edbc --- /dev/null +++ b/services/ontology-core/Dockerfile @@ -0,0 +1,16 @@ +FROM node:22-alpine + +WORKDIR /app + +COPY package.json ./ +COPY catalog ./catalog +COPY docs ./docs +COPY examples ./examples +COPY src ./src + +ENV NODE_ENV=production +ENV PORT=18104 + +EXPOSE 18104 + +CMD ["node", "src/mcp-server.mjs"] diff --git a/services/ontology-core/README.md b/services/ontology-core/README.md index 9ee0ed2..8eee682 100644 --- a/services/ontology-core/README.md +++ b/services/ontology-core/README.md @@ -4,6 +4,9 @@ Docs-first ontology service/module for NODE.DC. Current status: `v0.4-pre` implementation slice. +The read-only dynamic MCP integration for AI Workspace is documented in +[`../../docs/AI_WORKSPACE_ONTOLOGY_MCP.md`](../../docs/AI_WORKSPACE_ONTOLOGY_MCP.md). + This module owns: - canonical entity catalog; @@ -62,6 +65,7 @@ src/adapters/hub-launcher-admin.mjs ```text npm run validate +npm run smoke:mcp ``` ## Resolver Smoke @@ -159,8 +163,10 @@ Each package may contribute entities, relations, aliases, guardrails, evidence, Current package: +- `integration` - provider-neutral external provider connection, capability, collection, read-model and red command-domain ontology used by all adapter services. - `seo` - NDC SEO mod domain ontology for site scans, project ontology instances, scope contracts, semantic analysis, market evidence, optimization planning, validation, changesets, and future app-owned SEO assistant actions. - `map` - provider-neutral NDC Module Studio map ontology for spatial subjects, layers, routes, zones, shared labels, visibility rules, selection, and replaceable renderer adapters. +- `gelios` - provider-neutral Gelios fleet and telemetry ontology, bound to map.moving_object, map.zone and map.place_target without exposing provider credentials or renderer objects. The package loader merges domain packages into the base catalog before validation. Domain packages extend core meanings; they do not own app data or execute mutations. diff --git a/services/ontology-core/catalog/aliases.json b/services/ontology-core/catalog/aliases.json index 336620b..693cbd4 100644 --- a/services/ontology-core/catalog/aliases.json +++ b/services/ontology-core/catalog/aliases.json @@ -16,9 +16,8 @@ { "alias": "app", "canonicalId": "hub.application" }, { "alias": "application tile", "canonicalId": "hub.application_card" }, { "alias": "app card", "canonicalId": "hub.application_card" }, - { "alias": "n8n workflow id", "canonicalId": "engine.workflow_l2_runtime_id" }, { "alias": "runtime workflow id", "canonicalId": "engine.workflow_l2_runtime_id" }, - { "alias": "n8n execution", "canonicalId": "engine.l2_execution" }, + { "alias": "l2 execution", "canonicalId": "engine.l2_execution" }, { "alias": "Engine-side OPS", "canonicalId": "engine.tech_debt.ops_layer" }, { "alias": "Agent Monitor OPS", "canonicalId": "engine.tech_debt.ops_layer" }, { "alias": "Tender Agent Monitor UI", "canonicalId": "engine.tech_debt.tender_domain_ui" }, diff --git a/services/ontology-core/catalog/domain-packages/gelios/aliases.json b/services/ontology-core/catalog/domain-packages/gelios/aliases.json new file mode 100644 index 0000000..38f9529 --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/gelios/aliases.json @@ -0,0 +1,29 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "aliases": [ + { "alias": "gelios", "canonicalId": "gelios.integration" }, + { "alias": "gelius", "canonicalId": "gelios.integration" }, + { "alias": "helius", "canonicalId": "gelios.integration" }, + { "alias": "hel i os", "canonicalId": "gelios.integration" }, + { "alias": "гелиос", "canonicalId": "gelios.integration" }, + { "alias": "трайк robot2b", "canonicalId": "gelios.unit" }, + { "alias": "юнит гелиос", "canonicalId": "gelios.unit" }, + { "alias": "группа юнитов гелиос", "canonicalId": "gelios.unit_group" }, + { "alias": "последняя телеметрия гелиос", "canonicalId": "gelios.telemetry_snapshot" }, + { "alias": "сырая телеметрия гелиос", "canonicalId": "gelios.raw_telemetry_message" }, + { "alias": "позиция трайка", "canonicalId": "gelios.position_fix" }, + { "alias": "статус трайка", "canonicalId": "gelios.operational_status" }, + { "alias": "датчик гелиос", "canonicalId": "gelios.sensor_definition" }, + { "alias": "показание датчика", "canonicalId": "gelios.sensor_reading" }, + { "alias": "калибровка датчика", "canonicalId": "gelios.sensor_conversion" }, + { "alias": "топливный профиль", "canonicalId": "gelios.fuel_profile" }, + { "alias": "геозона гелиос", "canonicalId": "gelios.geozone" }, + { "alias": "геоточка гелиос", "canonicalId": "gelios.geopoint" }, + { "alias": "шаблон команды гелиос", "canonicalId": "gelios.command_template" }, + { "alias": "диспетчеризация команды гелиос", "canonicalId": "gelios.command_dispatch" }, + { "alias": "аудит команды гелиос", "canonicalId": "gelios.command_audit" }, + { "alias": "контур robot2b", "canonicalId": "gelios.access_scope" }, + { "alias": "курсор загрузки гелиос", "canonicalId": "gelios.ingestion_cursor" } + ] +} diff --git a/services/ontology-core/catalog/domain-packages/gelios/entities.json b/services/ontology-core/catalog/domain-packages/gelios/entities.json new file mode 100644 index 0000000..edefa07 --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/gelios/entities.json @@ -0,0 +1,34 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "entities": [ + { "id": "gelios.integration", "name": "Gelios Integration", "surface": "platform", "status": ["product-required", "source-evidenced"], "authority": "Gelios Gateway", "summary": "Server-side provider integration boundary for Gelios REST OAuth and legacy compatibility. It owns no UI, map renderer or workflow secret." }, + { "id": "gelios.access_scope", "name": "Gelios Access Scope", "surface": "platform", "status": ["product-required", "source-evidenced"], "authority": "Robot2B owner + Gelios Gateway", "summary": "Approved collection boundary expressed as an allowlist or owner rule for units, groups and allowed capabilities; distinct from broad provider-account visibility." }, + { "id": "gelios.unit", "name": "Gelios Unit", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Tracked operational object, including a Robot2B trike when its stable source unit ID is inside the approved scope." }, + { "id": "gelios.unit_group", "name": "Gelios Unit Group", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider-managed grouping of units. It is evidence for navigation and access, not an automatic definition of the Robot2B business scope." }, + { "id": "gelios.tracker_device", "name": "Tracker Device", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Hardware tracker associated with a unit, carrying device type/manufacturer and restricted hardware identity fields." }, + { "id": "gelios.telemetry_snapshot", "name": "Telemetry Snapshot", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Normalized current telemetry state for one unit at observed and received time; derived from the Gelios pure last message contract." }, + { "id": "gelios.raw_telemetry_message", "name": "Raw Telemetry Message", "surface": "telemetry", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider hardware message preserved only behind restricted raw-data policy; it is not the default Studio or analytics contract." }, + { "id": "gelios.position_fix", "name": "Position Fix", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Time-qualified geographic position with latitude, longitude, height, course, speed, satellite and quality attributes for one unit." }, + { "id": "gelios.telemetry_parameter", "name": "Telemetry Parameter", "surface": "telemetry", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Name/value parameter received with a message. Parameters are dynamic and require a namespaced registry and whitelist before product exposure." }, + { "id": "gelios.operational_status", "name": "Operational Status", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Derived unit state such as online, offline, moving, parked, no-position or low-GPS; distinct from raw provider message fields." }, + { "id": "gelios.sensor_definition", "name": "Sensor Definition", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Declared sensor on a unit with message parameter, type, unit of measure, visibility and optional fuel semantics." }, + { "id": "gelios.sensor_reading", "name": "Sensor Reading", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Time-qualified normalized reading of a defined sensor, retaining both numeric value and human-readable representation when permitted." }, + { "id": "gelios.sensor_conversion", "name": "Sensor Conversion", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Calibration/conversion rule or row for translating raw sensor values into operational measures." }, + { "id": "gelios.fuel_profile", "name": "Fuel Profile", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Fuel and consumption configuration associated with a unit or fuel sensor; use only after sensor semantics are approved." }, + { "id": "gelios.maintenance_plan", "name": "Maintenance Plan", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Planned maintenance configuration for a unit, separate from historical maintenance events." }, + { "id": "gelios.custom_field", "name": "Custom Field", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider-configurable unit field. Its value is restricted until explicitly classified because semantics and PII risk are dynamic." }, + { "id": "gelios.geozone", "name": "Gelios Geozone", "surface": "spatial", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider geofence or zone geometry. Large geometry collections require a separate loading and LOD strategy." }, + { "id": "gelios.geozone_group", "name": "Gelios Geozone Group", "surface": "spatial", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider grouping of geozones." }, + { "id": "gelios.geopoint", "name": "Gelios Geopoint", "surface": "spatial", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Named provider spatial point; it must remain distinct from a live unit position." }, + { "id": "gelios.report_template", "name": "Gelios Report Template", "surface": "analytics", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Reusable provider report definition available to the account or scope." }, + { "id": "gelios.report_result", "name": "Gelios Report Result", "surface": "analytics", "status": ["source-evidenced"], "authority": "Gelios Pro + Gelios Gateway", "summary": "Bounded result, table, graphic or map output produced from an approved report request; not a default realtime stream." }, + { "id": "gelios.collection_run", "name": "Gelios Collection Run", "surface": "platform", "status": ["product-required"], "authority": "Gelios Gateway", "summary": "Auditable read-only collection attempt with scope, endpoint family, volume controls and outcome; it contains no token or raw payload." }, + { "id": "gelios.ingestion_cursor", "name": "Gelios Ingestion Cursor", "surface": "platform", "status": ["product-required"], "authority": "Gelios Gateway + telemetry storage", "summary": "Checkpoint and watermark controlling resumable historical or incremental ingestion for an approved unit and data family." }, + { "id": "gelios.command_template", "name": "Gelios Command Template", "surface": "command", "status": ["source-evidenced", "product-required"], "authority": "Gelios Pro", "summary": "Read-only catalogued command template assigned to a unit. Template content is red-domain configuration, not a telemetry field." }, + { "id": "gelios.command_group", "name": "Gelios Command Group", "surface": "command", "status": ["source-evidenced", "product-required"], "authority": "Gelios Pro", "summary": "Provider grouping of command templates, managed separately from operational unit groups." }, + { "id": "gelios.command_dispatch", "name": "Gelios Command Dispatch", "surface": "command", "status": ["product-required"], "authority": "Future Command Gateway", "summary": "Explicit, user-confirmed request to send a command to one approved unit or approved group. It must never be created by background ingestion or map rendering." }, + { "id": "gelios.command_delivery", "name": "Gelios Command Delivery", "surface": "command", "status": ["product-required", "source-evidenced"], "authority": "Gelios Pro + Future Command Gateway", "summary": "Provider delivery/task status associated with a command dispatch." }, + { "id": "gelios.command_audit", "name": "Gelios Command Audit", "surface": "command", "status": ["product-required"], "authority": "Future Command Gateway", "summary": "Immutable governance record of an explicit command decision, confirmation, target, outcome and actor; separate from provider command history." } + ] +} diff --git a/services/ontology-core/catalog/domain-packages/gelios/evidence.json b/services/ontology-core/catalog/domain-packages/gelios/evidence.json new file mode 100644 index 0000000..6e87f0e --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/gelios/evidence.json @@ -0,0 +1,49 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "sourceRoots": [ + { + "surface": "gelios-rest", + "path": "https://api.geliospro.com/docs", + "mode": "OpenAPI inspection and safe read-only runtime audit; no write routes executed" + }, + { + "surface": "engine", + "path": "/Users/dcconstructions/Downloads/mnt/NODEDC/NODEDC_ENGINE_INFRA/nodedc-source", + "mode": "Read-only MMAP workflow, Gelios node and historical snapshot donor inspection" + }, + { + "surface": "module-studio", + "path": "/Users/dcconstructions/Downloads/mnt/NODEDC/NODEDC_DESIGN_GUIDELINE", + "mode": "Future provider-neutral map target surface" + } + ], + "ledgers": [ + { + "id": "ledger.gelios_domain_v0", + "path": "docs/GELIOS_DOMAIN_ONTOLOGY.md", + "entityIds": [ + "gelios.integration", + "gelios.access_scope", + "gelios.unit", + "gelios.telemetry_snapshot", + "gelios.position_fix", + "gelios.sensor_definition", + "gelios.sensor_reading", + "gelios.geozone", + "gelios.report_template", + "gelios.command_template", + "gelios.command_dispatch", + "gelios.ingestion_cursor" + ] + } + ], + "baselineDocs": ["docs/GELIOS_DOMAIN_ONTOLOGY.md"], + "restrictions": [ + "Do not copy access/refresh tokens, passwords, hardware decrypt keys or runtime snapshots into Ontology Core.", + "Do not make the current Engine Gelios node, Cesium entities or provider group names canonical domain identities.", + "Do not invoke command send, create, update, delete or purge routes as ontology evidence.", + "Do not treat the 107 currently visible provider-account units or the 95 legacy snapshot units as the final Robot2B scope without owner approval.", + "Do not bulk-download history, media or full geozone geometry before collection policy and storage architecture are approved." + ] +} diff --git a/services/ontology-core/catalog/domain-packages/gelios/guardrails.json b/services/ontology-core/catalog/domain-packages/gelios/guardrails.json new file mode 100644 index 0000000..5c9d161 --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/gelios/guardrails.json @@ -0,0 +1,57 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "rules": [ + { + "id": "guardrail.gelios.ontology_not_runtime_store", + "severity": "error", + "summary": "Ontology Core describes Gelios meanings and contracts only. It must not contain access/refresh tokens, raw messages, live positions, provider command text, hardware decrypt keys or bulk geometry.", + "entityIds": ["gelios.integration", "gelios.telemetry_snapshot", "gelios.raw_telemetry_message", "gelios.position_fix", "gelios.command_template", "gelios.geozone"] + }, + { + "id": "guardrail.gelios.scope_precedes_collection", + "severity": "error", + "summary": "A Gelios collection run must resolve an approved Robot2B access scope before addressing units. Provider-account visibility and provider group membership are not sufficient product scope on their own.", + "entityIds": ["gelios.integration", "gelios.access_scope", "gelios.collection_run", "gelios.unit", "gelios.unit_group"] + }, + { + "id": "guardrail.gelios.commands_are_red_domain", + "severity": "error", + "summary": "Command template catalogue is read-only data. Command dispatch, command group mutation, history purge and every send route are separate red-domain actions requiring explicit human confirmation and audit; ingestion and map rendering may never create them.", + "entityIds": ["gelios.command_template", "gelios.command_group", "gelios.command_dispatch", "gelios.command_delivery", "gelios.command_audit"] + }, + { + "id": "guardrail.gelios.map_uses_stable_subjects", + "severity": "error", + "summary": "Cesium pins and labels must consume stable gelios.unit and normalized gelios.position_fix through map.moving_object. They must not use raw message IDs, transient Cesium entity IDs or provider credentials as map identity.", + "entityIds": ["gelios.unit", "gelios.position_fix", "gelios.telemetry_snapshot", "map.moving_object", "map.pin", "map.label"] + }, + { + "id": "guardrail.gelios.raw_and_pii_are_restricted", + "severity": "warning", + "summary": "IMEI/hardware identifiers, phone fields, precise addresses, raw params, custom fields and unclassified sensor values require explicit field policy before analytics, Studio or external exposure.", + "entityIds": ["gelios.tracker_device", "gelios.position_fix", "gelios.telemetry_parameter", "gelios.custom_field", "gelios.sensor_reading"] + }, + { + "id": "guardrail.gelios.geometry_requires_bounded_loading", + "severity": "warning", + "summary": "Geozone and report-map loading must use explicit paging, filtering, volume limits, LOD and retention policy. The unpaged provider geozone response is not an acceptable realtime renderer contract.", + "entityIds": ["gelios.geozone", "gelios.geozone_group", "gelios.collection_run", "map.zone", "map.visibility_rule"] + }, + { + "id": "guardrail.gelios.history_requires_cursor", + "severity": "warning", + "summary": "Any future history ingestion must be bounded by unit scope, time window, field whitelist and durable cursor. It must not turn a historical API family into an uncontrolled account dump.", + "entityIds": ["gelios.collection_run", "gelios.ingestion_cursor", "gelios.raw_telemetry_message", "gelios.report_result"] + } + ], + "blockedConflations": [ + ["gelios.integration", "gelios.access_scope"], + ["gelios.unit", "gelios.tracker_device"], + ["gelios.telemetry_snapshot", "gelios.raw_telemetry_message"], + ["gelios.position_fix", "map.moving_object"], + ["gelios.command_template", "gelios.command_dispatch"], + ["gelios.command_dispatch", "gelios.command_delivery"], + ["gelios.geozone", "map.zone"] + ] +} diff --git a/services/ontology-core/catalog/domain-packages/gelios/package.json b/services/ontology-core/catalog/domain-packages/gelios/package.json new file mode 100644 index 0000000..8eacdc6 --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/gelios/package.json @@ -0,0 +1,7 @@ +{ + "id": "gelios", + "version": "0.1.0", + "updatedAt": "2026-07-13", + "status": "product-required/source-evidenced", + "summary": "Gelios Pro telemetry, fleet, spatial, reporting and command-domain ontology for the Robot2B client context. The package describes canonical meanings and contracts; it does not store credentials or runtime telemetry." +} diff --git a/services/ontology-core/catalog/domain-packages/gelios/relations.json b/services/ontology-core/catalog/domain-packages/gelios/relations.json new file mode 100644 index 0000000..333590a --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/gelios/relations.json @@ -0,0 +1,44 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "relations": [ + { "id": "gelios.integration.is_provider_connection", "from": ["gelios.integration"], "to": ["integration.connection"], "status": "product-required", "summary": "A Gelios integration is a tenant-scoped instance of the common external provider connection contract." }, + { "id": "gelios.access_scope.specializes_integration_scope", "from": ["gelios.access_scope"], "to": ["integration.access_scope"], "status": "product-required", "summary": "Robot2B/Gelios object approval specializes the common provider access-scope contract." }, + { "id": "gelios.collection_run.is_integration_collection_run", "from": ["gelios.collection_run"], "to": ["integration.collection_run"], "status": "product-required", "summary": "Gelios collection runs inherit the bounded collection and audit lifecycle of external integrations." }, + { "id": "gelios.raw_message.is_integration_raw_envelope", "from": ["gelios.raw_telemetry_message"], "to": ["integration.raw_envelope"], "status": "product-required", "summary": "A Gelios raw message is a restricted provider envelope, not a UI read model." }, + { "id": "gelios.unit.is_integration_canonical_subject", "from": ["gelios.unit"], "to": ["integration.canonical_subject"], "status": "product-required", "summary": "A Gelios unit is a stable canonical subject produced by the provider adapter." }, + { "id": "gelios.command_template.is_integration_command_capability", "from": ["gelios.command_template"], "to": ["integration.command_capability"], "status": "product-required", "summary": "Gelios command templates are catalogued provider command capabilities only." }, + { "id": "gelios.command_dispatch.is_integration_command_intent", "from": ["gelios.command_dispatch"], "to": ["integration.command_intent"], "status": "future-concept", "summary": "Any future Gelios command dispatch must become an explicitly governed generic command intent." }, + { "id": "gelios.command_audit.is_integration_command_audit", "from": ["gelios.command_audit"], "to": ["integration.command_audit"], "status": "future-concept", "summary": "Gelios command audit specializes the generic red-domain audit contract." }, + { "id": "gelios.integration.enforces_scope", "from": ["gelios.integration"], "to": ["gelios.access_scope"], "status": "product-required", "summary": "The Gateway applies an approved Robot2B scope before reading or publishing data." }, + { "id": "gelios.access_scope.includes_unit", "from": ["gelios.access_scope"], "to": ["gelios.unit", "gelios.unit_group"], "status": "product-required", "summary": "An approved scope can include explicit units and approved group criteria, but group membership alone is not authoritative." }, + { "id": "gelios.unit.belongs_to_group", "from": ["gelios.unit"], "to": ["gelios.unit_group"], "status": "source-evidenced", "summary": "A provider unit can belong to one or more provider unit groups." }, + { "id": "gelios.unit.uses_tracker_device", "from": ["gelios.unit"], "to": ["gelios.tracker_device"], "status": "source-evidenced", "summary": "A tracked unit is associated with a hardware tracker device." }, + { "id": "gelios.unit.has_telemetry_snapshot", "from": ["gelios.unit"], "to": ["gelios.telemetry_snapshot"], "status": "product-required", "summary": "The Gateway maintains normalized current telemetry for an approved unit." }, + { "id": "gelios.raw_message.normalizes_to_snapshot", "from": ["gelios.raw_telemetry_message"], "to": ["gelios.telemetry_snapshot"], "status": "product-required", "summary": "Restricted raw provider telemetry can be normalized into the stable snapshot contract." }, + { "id": "gelios.telemetry_snapshot.has_position_fix", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.position_fix"], "status": "product-required", "summary": "A current telemetry snapshot can carry one time-qualified position fix." }, + { "id": "gelios.telemetry_snapshot.has_operational_status", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.operational_status"], "status": "product-required", "summary": "Gateway derives normalized operational state from telemetry and data-quality policy." }, + { "id": "gelios.telemetry_snapshot.has_parameter", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.telemetry_parameter"], "status": "source-evidenced", "summary": "A provider snapshot can contain dynamic telemetry parameters." }, + { "id": "gelios.unit.has_sensor", "from": ["gelios.unit"], "to": ["gelios.sensor_definition"], "status": "source-evidenced", "summary": "A unit exposes zero or more sensor definitions." }, + { "id": "gelios.sensor_definition.has_conversion", "from": ["gelios.sensor_definition"], "to": ["gelios.sensor_conversion"], "status": "source-evidenced", "summary": "A sensor can define conversion/calibration rows." }, + { "id": "gelios.sensor_reading.observes_sensor", "from": ["gelios.sensor_reading"], "to": ["gelios.sensor_definition"], "status": "product-required", "summary": "Each normalized reading identifies the sensor definition it observes." }, + { "id": "gelios.telemetry_snapshot.has_sensor_reading", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.sensor_reading"], "status": "product-required", "summary": "Current telemetry can expose normalized readings for available sensors." }, + { "id": "gelios.unit.has_fuel_profile", "from": ["gelios.unit"], "to": ["gelios.fuel_profile"], "status": "source-evidenced", "summary": "Fuel configuration belongs to a unit and is interpreted through approved sensor semantics." }, + { "id": "gelios.unit.has_maintenance_plan", "from": ["gelios.unit"], "to": ["gelios.maintenance_plan"], "status": "source-evidenced", "summary": "A unit can have zero or more planned maintenance configurations." }, + { "id": "gelios.unit.has_custom_field", "from": ["gelios.unit"], "to": ["gelios.custom_field"], "status": "source-evidenced", "summary": "A unit can have provider-defined custom fields." }, + { "id": "gelios.geozone.belongs_to_group", "from": ["gelios.geozone"], "to": ["gelios.geozone_group"], "status": "source-evidenced", "summary": "A provider geozone can belong to a provider geozone group." }, + { "id": "gelios.report_result.uses_template", "from": ["gelios.report_result"], "to": ["gelios.report_template"], "status": "source-evidenced", "summary": "A report result is produced from an approved report template and bounded request." }, + { "id": "gelios.collection_run.checkpoints_with_cursor", "from": ["gelios.collection_run"], "to": ["gelios.ingestion_cursor"], "status": "product-required", "summary": "Collection runs advance resumable ingestion only through a durable cursor." }, + { "id": "gelios.collection_run.collects_unit", "from": ["gelios.collection_run"], "to": ["gelios.unit"], "status": "product-required", "summary": "A read-only collection run records the approved unit scope it addressed." }, + { "id": "gelios.unit.has_command_template", "from": ["gelios.unit"], "to": ["gelios.command_template"], "status": "source-evidenced", "summary": "Command templates are catalogued per unit through read-only routes." }, + { "id": "gelios.command_template.belongs_to_group", "from": ["gelios.command_template"], "to": ["gelios.command_group"], "status": "source-evidenced", "summary": "A command template can belong to a provider command group." }, + { "id": "gelios.command_dispatch.uses_template", "from": ["gelios.command_dispatch"], "to": ["gelios.command_template"], "status": "product-required", "summary": "A dispatch may reference a reviewed command template." }, + { "id": "gelios.command_dispatch.targets_scope", "from": ["gelios.command_dispatch"], "to": ["gelios.unit", "gelios.unit_group", "gelios.access_scope"], "status": "product-required", "summary": "A dispatch target must resolve to an approved unit or explicitly approved group in scope." }, + { "id": "gelios.command_dispatch.has_delivery", "from": ["gelios.command_dispatch"], "to": ["gelios.command_delivery"], "status": "product-required", "summary": "A dispatch is associated with delivery/task state rather than assumed successful on request creation." }, + { "id": "gelios.command_dispatch.has_audit", "from": ["gelios.command_dispatch"], "to": ["gelios.command_audit"], "status": "product-required", "summary": "Every dispatch requires immutable user-confirmation and outcome audit." }, + { "id": "gelios.unit.is_map_moving_object", "from": ["gelios.unit"], "to": ["map.moving_object"], "status": "product-required", "summary": "An approved Gelios unit is rendered through the provider-neutral moving-object contract, never as a Cesium entity identity." }, + { "id": "gelios.position_fix.positions_map_subject", "from": ["gelios.position_fix"], "to": ["map.moving_object"], "status": "product-required", "summary": "A normalized position fix updates the stable map subject bound to the Gelios unit." }, + { "id": "gelios.geozone.is_map_zone", "from": ["gelios.geozone"], "to": ["map.zone"], "status": "product-required", "summary": "An approved Gelios geozone is exposed to the map through the generic zone contract." }, + { "id": "gelios.geopoint.is_map_place_target", "from": ["gelios.geopoint"], "to": ["map.place_target"], "status": "product-required", "summary": "An approved Gelios geopoint is exposed through a provider-neutral place target." } + ] +} diff --git a/services/ontology-core/catalog/domain-packages/integration/aliases.json b/services/ontology-core/catalog/domain-packages/integration/aliases.json new file mode 100644 index 0000000..93944ff --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/integration/aliases.json @@ -0,0 +1,11 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "aliases": [ + {"alias":"коннектор","canonicalId":"integration.connection","language":"ru","status":"product-required"}, + {"alias":"connector","canonicalId":"integration.connection","language":"en","status":"product-required"}, + {"alias":"внешний провайдер","canonicalId":"integration.provider","language":"ru","status":"product-required"}, + {"alias":"provider","canonicalId":"integration.provider","language":"en","status":"product-required"}, + {"alias":"источник данных","canonicalId":"integration.provider","language":"ru","status":"product-required"} + ] +} diff --git a/services/ontology-core/catalog/domain-packages/integration/entities.json b/services/ontology-core/catalog/domain-packages/integration/entities.json new file mode 100644 index 0000000..0fcc764 --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/integration/entities.json @@ -0,0 +1,22 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "entities": [ + {"id":"integration.provider","name":"External Provider","surface":"integration","status":["product-required"],"authority":"Platform External Provider Data Plane","summary":"A third-party product or API family integrated through an app-owned adapter, not a tenant or a renderer."}, + {"id":"integration.connection","name":"Provider Connection","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"A tenant-scoped configured connection to one provider account or endpoint; it contains policy and secret references, never secret values."}, + {"id":"integration.credential_reference","name":"Provider Credential Reference","surface":"integration","status":["product-required"],"authority":"Provider Gateway / secret store","summary":"Opaque reference to a server-side secret or token lifecycle, not a token, login or password."}, + {"id":"integration.access_scope","name":"Integration Access Scope","surface":"integration","status":["product-required"],"authority":"Client owner + Provider Gateway","summary":"Approved product boundary for source accounts, objects, fields and permitted operations; provider visibility alone is insufficient."}, + {"id":"integration.capability","name":"Provider Capability","surface":"integration","status":["product-required"],"authority":"Provider Adapter","summary":"Versioned provider operation or feature classified as read, metadata, write, destructive or unknown."}, + {"id":"integration.field_definition","name":"Provider Field Definition","surface":"integration","status":["product-required"],"authority":"Provider Adapter + Ontology Core","summary":"A documented source field, its semantics, sensitivity, units and normalisation mapping; it is not a live value."}, + {"id":"integration.collection_profile","name":"Collection Profile","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Approved rate, paging, cursor, capability and field selection for a connection; prevents uncontrolled account-wide polling."}, + {"id":"integration.retention_policy","name":"Integration Retention Policy","surface":"integration","status":["product-required"],"authority":"Client owner + Provider Gateway","summary":"Lifecycle rule for current state, raw envelope, normalised history, aggregates and backups."}, + {"id":"integration.collection_run","name":"Integration Collection Run","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Auditable bounded execution of a safe-read collection profile with cursor, response metrics and outcome."}, + {"id":"integration.raw_envelope","name":"Raw Provider Envelope","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Restricted provenance-preserving representation of a safe-read provider response, stored in a bounded cold layer rather than exposed to UI."}, + {"id":"integration.canonical_subject","name":"Canonical Integrated Subject","surface":"integration","status":["product-required"],"authority":"Provider Domain Adapter","summary":"Stable domain subject normalised from a provider object and suitable for relations, analytics and interfaces."}, + {"id":"integration.read_model","name":"Integration Read Model","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Scoped, versioned current, history or aggregate projection delivered to approved consumers without raw provider bypass."}, + {"id":"integration.realtime_channel","name":"Integration Realtime Channel","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Internal bounded event or subscription contract for projection changes; its emission rate is independent from provider collection rate."}, + {"id":"integration.command_capability","name":"Provider Command Capability","surface":"integration","status":["product-required"],"authority":"Provider Adapter","summary":"Catalogued command metadata, parameters and risk classification. It is not permission or a send route."}, + {"id":"integration.command_intent","name":"Provider Command Intent","surface":"integration","status":["future-concept"],"authority":"Future Provider Command Gateway","summary":"A future explicitly confirmed, scoped and idempotent request to execute a catalogued command; no read collector may create it."}, + {"id":"integration.command_audit","name":"Provider Command Audit","surface":"integration","status":["future-concept"],"authority":"Future Provider Command Gateway","summary":"Immutable approval, execution and delivery audit for a red-domain command lifecycle."} + ] +} diff --git a/services/ontology-core/catalog/domain-packages/integration/evidence.json b/services/ontology-core/catalog/domain-packages/integration/evidence.json new file mode 100644 index 0000000..4f5daf1 --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/integration/evidence.json @@ -0,0 +1,12 @@ +{ + "sourceRoots": ["platform/docs", "platform/packages", "platform/services"], + "ledgers": [ + { + "id": "external_provider_data_plane_adr", + "path": "../../docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md", + "entityIds": ["integration.provider", "integration.connection", "integration.access_scope", "integration.capability", "integration.collection_profile", "integration.read_model", "integration.command_capability"] + } + ], + "baselineDocs": ["../../docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md", "../../packages/external-provider-contract/README.md"], + "restrictions": ["The package is a semantic contract. Provider secrets, runtime data and command transport remain outside Ontology Core."] +} diff --git a/services/ontology-core/catalog/domain-packages/integration/guardrails.json b/services/ontology-core/catalog/domain-packages/integration/guardrails.json new file mode 100644 index 0000000..07b194b --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/integration/guardrails.json @@ -0,0 +1,18 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "rules": [ + {"id":"guardrail.integration.ontology_is_not_data_plane","severity":"error","summary":"Ontology Core may describe providers and contracts but must not store secret values, live payloads, tenant telemetry, raw envelopes or provider execution state.","entityIds":["integration.provider","integration.connection","integration.credential_reference","integration.raw_envelope","integration.read_model"]}, + {"id":"guardrail.integration.scope_precedes_collection","severity":"error","summary":"A collection run must resolve tenant scope, field policy and bounded collection profile before it addresses a provider. Account visibility does not define product scope.","entityIds":["integration.connection","integration.access_scope","integration.collection_profile","integration.collection_run"]}, + {"id":"guardrail.integration.read_model_not_raw_bypass","severity":"error","summary":"L2 workflows, interfaces and renderer adapters consume scoped read models or realtime channels, never provider credentials, raw envelopes or direct adapter database access.","entityIds":["integration.credential_reference","integration.raw_envelope","integration.read_model","integration.realtime_channel"]}, + {"id":"guardrail.integration.commands_are_separate_red_domain","severity":"error","summary":"Cataloguing a command does not activate it. Read collectors, L2 workflows, maps and AI Workspace may not create command intent or delivery. A later command gateway requires explicit human confirmation, scope, idempotency and audit.","entityIds":["integration.command_capability","integration.command_intent","integration.command_audit","integration.collection_run"]}, + {"id":"guardrail.integration.full_catalog_not_full_rate","severity":"warning","summary":"Every provider field may be catalogued, but enabling continuous collection requires an explicit collection, retention and rate policy. Do not turn a capability catalog into unbounded account polling.","entityIds":["integration.capability","integration.field_definition","integration.collection_profile","integration.retention_policy"]} + ], + "blockedConflations": [ + ["integration.provider","integration.connection"], + ["integration.connection","integration.credential_reference"], + ["integration.raw_envelope","integration.read_model"], + ["integration.command_capability","integration.command_intent"], + ["integration.collection_run","integration.command_intent"] + ] +} diff --git a/services/ontology-core/catalog/domain-packages/integration/package.json b/services/ontology-core/catalog/domain-packages/integration/package.json new file mode 100644 index 0000000..e504b7a --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/integration/package.json @@ -0,0 +1,7 @@ +{ + "id": "integration", + "version": "0.1.0", + "updatedAt": "2026-07-13", + "status": "product-required/source-evidenced", + "summary": "Provider-neutral external integration ontology: tenant-scoped connections, capabilities, collection, controlled storage projections and a separated red command domain." +} diff --git a/services/ontology-core/catalog/domain-packages/integration/relations.json b/services/ontology-core/catalog/domain-packages/integration/relations.json new file mode 100644 index 0000000..3c0fd3b --- /dev/null +++ b/services/ontology-core/catalog/domain-packages/integration/relations.json @@ -0,0 +1,20 @@ +{ + "version": "0.1.0", + "updatedAt": "2026-07-13", + "relations": [ + {"id":"integration.provider.offers_connection","from":["integration.provider"],"to":["integration.connection"],"status":"product-required","summary":"A provider is connected through one or more tenant-scoped connection instances."}, + {"id":"integration.connection.references_credential","from":["integration.connection"],"to":["integration.credential_reference"],"status":"product-required","summary":"A connection uses an opaque server-side credential reference rather than exposing a secret."}, + {"id":"integration.connection.enforces_scope","from":["integration.connection"],"to":["integration.access_scope"],"status":"product-required","summary":"Every source request is bounded by an approved product scope."}, + {"id":"integration.provider.declares_capability","from":["integration.provider"],"to":["integration.capability","integration.command_capability"],"status":"product-required","summary":"Adapter manifests catalogue source read and command capabilities independently from their activation."}, + {"id":"integration.capability.exposes_field","from":["integration.capability"],"to":["integration.field_definition"],"status":"product-required","summary":"A safe-read capability documents the fields it may return before a collection profile enables them."}, + {"id":"integration.collection_profile.selects_capability","from":["integration.collection_profile"],"to":["integration.capability","integration.field_definition"],"status":"product-required","summary":"A profile selects bounded capabilities and fields rather than polling every API feature continuously."}, + {"id":"integration.collection_profile.applies_retention","from":["integration.collection_profile"],"to":["integration.retention_policy"],"status":"product-required","summary":"Collection and storage lifecycle are explicit per connection profile."}, + {"id":"integration.collection_run.uses_connection","from":["integration.collection_run"],"to":["integration.connection","integration.collection_profile"],"status":"product-required","summary":"A collection run records which connection and bounded policy it used."}, + {"id":"integration.collection_run.records_raw_envelope","from":["integration.collection_run"],"to":["integration.raw_envelope"],"status":"product-required","summary":"Safe-read response provenance is retained only according to the active policy."}, + {"id":"integration.raw_envelope.normalizes_subject","from":["integration.raw_envelope"],"to":["integration.canonical_subject"],"status":"product-required","summary":"Provider payloads are normalised into stable domain subjects before consumption."}, + {"id":"integration.read_model.projects_subject","from":["integration.read_model"],"to":["integration.canonical_subject"],"status":"product-required","summary":"Consumers receive a scoped projection rather than a provider payload or direct database access."}, + {"id":"integration.realtime_channel.emits_read_model","from":["integration.realtime_channel"],"to":["integration.read_model"],"status":"product-required","summary":"Realtime delivery emits approved projection changes with independently controlled sampling."}, + {"id":"integration.command_intent.uses_capability","from":["integration.command_intent"],"to":["integration.command_capability"],"status":"future-concept","summary":"A future command intent may select only a catalogued command capability."}, + {"id":"integration.command_intent.has_audit","from":["integration.command_intent"],"to":["integration.command_audit"],"status":"future-concept","summary":"Every future command lifecycle requires immutable approval and delivery audit."} + ] +} diff --git a/services/ontology-core/catalog/domain-packages/map/entities.json b/services/ontology-core/catalog/domain-packages/map/entities.json index 314be22..ddfc8b2 100644 --- a/services/ontology-core/catalog/domain-packages/map/entities.json +++ b/services/ontology-core/catalog/domain-packages/map/entities.json @@ -9,7 +9,7 @@ { "id": "map.grid_layer", "name": "Map Grid Layer", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Planetary or situational grid with configurable modes, levels of detail, step, radius and visual primitives." }, { "id": "map.place_target", "name": "Map Place Target", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Geographic place target such as a city or territory with location, pulse, radius and distance-dependent presentation." }, { "id": "map.moving_object", "name": "Map Moving Object", "surface": "map", "status": ["source-evidenced"], "authority": "NDC domain source", "summary": "Dynamic spatial object such as transport, robot or vehicle with identity, position, movement and status." }, - { "id": "map.pin", "name": "Map Pin", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Reusable spatial pin presentation with stem, point, height, color, status and visibility rules." }, + { "id": "map.pin", "name": "Map Pin", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Reusable provider-neutral elevated-spike presentation with ground anchor, stem, head, label anchor, semantic colour/status and camera-height visibility rules." }, { "id": "map.label", "name": "Map Label", "surface": "map", "status": ["source-evidenced", "product-required"], "authority": "NDC Module Studio", "summary": "Reusable information plate attached to a spatial subject with style, size variant, anchor, offset and visibility rules." }, { "id": "map.zone", "name": "Map Zone", "surface": "map", "status": ["source-evidenced"], "authority": "NDC domain source", "summary": "Polygon or multipolygon zone, sector or geofence with optional height, extrusion and level-dependent style." }, { "id": "map.route", "name": "Map Route", "surface": "map", "status": ["source-evidenced"], "authority": "NDC domain source", "summary": "Logical ordered path or service route independent of renderer geometry." }, diff --git a/services/ontology-core/catalog/entities.json b/services/ontology-core/catalog/entities.json index 2394053..8f4a2e8 100644 --- a/services/ontology-core/catalog/entities.json +++ b/services/ontology-core/catalog/entities.json @@ -81,14 +81,14 @@ { "id": "engine.workflow_owner", "name": "ENGINE Workflow Owner", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE", "summary": "Owner field in ACL." }, { "id": "engine.workflow_share", "name": "ENGINE Workflow Share", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE", "summary": "Share/invite/user access routes." }, { "id": "engine.workflow_access_request", "name": "ENGINE Workflow Access Request", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE", "summary": "Access request route/model for workflow access." }, - { "id": "engine.workflow_l2", "name": "ENGINE L2 Workflow", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE/n8n bridge", "summary": "n8n subworkflow attached to L1 node." }, - { "id": "engine.node_l2", "name": "ENGINE L2 Node", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE/n8n bridge", "summary": "Node inside compiled/deployed n8n workflow." }, - { "id": "engine.l2_runtime_core", "name": "ENGINE L2 Runtime Core", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE/n8n bridge", "summary": "Target n8n runtime/core instance." }, - { "id": "engine.workflow_l2_runtime_id", "name": "ENGINE L2 Runtime Workflow ID", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE/n8n bridge", "summary": "n8n runtime workflow id, separate from NodeDC workflow id." }, - { "id": "engine.l2_execution", "name": "ENGINE L2 Execution", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE/n8n bridge", "summary": "n8n execution id mapped to run/execution records." }, + { "id": "engine.workflow_l2", "name": "ENGINE L2 Workflow", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE execution bridge", "summary": "L2 execution workflow attached to an L1 node." }, + { "id": "engine.node_l2", "name": "ENGINE L2 Node", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE execution bridge", "summary": "Node inside a compiled/deployed L2 workflow." }, + { "id": "engine.l2_runtime_core", "name": "ENGINE L2 Runtime Core", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE execution bridge", "summary": "Target Engine execution core." }, + { "id": "engine.workflow_l2_runtime_id", "name": "ENGINE L2 Runtime Workflow ID", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE execution bridge", "summary": "L2 runtime workflow id, separate from the L1 workflow id." }, + { "id": "engine.l2_execution", "name": "ENGINE L2 Execution", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE execution bridge", "summary": "L2 execution id mapped to run/execution records." }, { "id": "engine.l2_run", "name": "ENGINE L2 Run", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE-side OpsLayer", "summary": "Runtime run id; tech-debt-adjacent." }, { "id": "engine.l2_session", "name": "ENGINE L2 Session", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE-side OpsLayer", "summary": "Runtime session anchor." }, - { "id": "engine.l2_runtime_event", "name": "ENGINE L2 Runtime Event", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE/n8n bridge", "summary": "Runtime event: workflow/node start/finish/success/error." }, + { "id": "engine.l2_runtime_event", "name": "ENGINE L2 Runtime Event", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE execution bridge", "summary": "Runtime event: workflow/node start/finish/success/error." }, { "id": "engine.source_stamp", "name": "ENGINE Source Stamp", "surface": "engine", "status": ["pending"], "authority": "ENGINE", "summary": "Future source/version traceability candidate." }, { "id": "assistant.assistant", "name": "Assistant", "surface": "assistant", "status": ["product-required", "pending"], "authority": "AI Workspace", "summary": "User-facing assistant capability/persona." }, diff --git a/services/ontology-core/catalog/relations.json b/services/ontology-core/catalog/relations.json index a0f0309..db7d7bd 100644 --- a/services/ontology-core/catalog/relations.json +++ b/services/ontology-core/catalog/relations.json @@ -49,10 +49,10 @@ { "id": "engine.workflow_acl.has_owner", "from": ["engine.workflow_acl"], "to": ["engine.workflow_owner"], "status": "source-confirmed", "summary": "ACL owns owner field." }, { "id": "engine.workflow_l1.has_share", "from": ["engine.workflow_l1"], "to": ["engine.workflow_share"], "status": "source-confirmed", "summary": "Workflow can be shared/invited." }, { "id": "engine.workflow_l1.has_access_request", "from": ["engine.workflow_l1"], "to": ["engine.workflow_access_request"], "status": "source-confirmed", "summary": "Access requests target workflow." }, - { "id": "engine.node_l1.embeds_l2_workflow", "from": ["engine.node_l1"], "to": ["engine.workflow_l2"], "status": "source-evidenced", "summary": "n8n subworkflow is attached to L1 node." }, - { "id": "engine.workflow_l2.deployed_as_runtime_workflow", "from": ["engine.workflow_l2"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "Compiled n8n workflow deploy returns runtime workflow id." }, - { "id": "engine.workflow_l2.runs_on_runtime_core", "from": ["engine.workflow_l2"], "to": ["engine.l2_runtime_core"], "status": "source-evidenced", "summary": "L2 workflow targets n8n instance/core." }, - { "id": "engine.l2_execution.belongs_to_runtime_workflow", "from": ["engine.l2_execution"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "Execution is emitted for n8n workflow id." }, + { "id": "engine.node_l1.embeds_l2_workflow", "from": ["engine.node_l1"], "to": ["engine.workflow_l2"], "status": "source-evidenced", "summary": "L2 execution workflow is attached to an L1 node." }, + { "id": "engine.workflow_l2.deployed_as_runtime_workflow", "from": ["engine.workflow_l2"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "L2 deploy returns a runtime workflow id." }, + { "id": "engine.workflow_l2.runs_on_runtime_core", "from": ["engine.workflow_l2"], "to": ["engine.l2_runtime_core"], "status": "source-evidenced", "summary": "L2 workflow targets the Engine execution core." }, + { "id": "engine.l2_execution.belongs_to_runtime_workflow", "from": ["engine.l2_execution"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "Execution is emitted for the L2 runtime workflow id." }, { "id": "engine.l2_runtime_event.describes_execution", "from": ["engine.l2_runtime_event"], "to": ["engine.l2_execution"], "status": "source-confirmed", "summary": "Runtime events carry execution id." }, { "id": "engine.l2_runtime_event.binds_run", "from": ["engine.l2_runtime_event"], "to": ["engine.l2_run"], "status": "source-confirmed", "summary": "Event binds to run id. Current storage is ENGINE-side OpsLayer." }, { "id": "engine.l2_run.has_session", "from": ["engine.l2_run"], "to": ["engine.l2_session"], "status": "source-confirmed", "summary": "Session anchors related runtime events/runs." }, diff --git a/services/ontology-core/catalog/resolver-rules.json b/services/ontology-core/catalog/resolver-rules.json index 1d3edde..3ddd42f 100644 --- a/services/ontology-core/catalog/resolver-rules.json +++ b/services/ontology-core/catalog/resolver-rules.json @@ -7,7 +7,7 @@ "relationId": "ontology.resolves_ops_card_to_engine_context", "fromSurface": "ops", "inputEntityIds": ["ops.card", "ops.project"], - "intentHints": ["engine", "workflow", "l1", "l2", "n8n", "inspect", "switch", "runtime", "докодим", "воркфлоу"], + "intentHints": ["engine", "workflow", "l1", "l2", "inspect", "switch", "runtime", "докодим", "воркфлоу"], "outputSurface": "engine", "outputEntityIds": ["engine.workflow_l1", "engine.node_l1", "engine.workflow_l2", "engine.workflow_l2_runtime_id"], "requiredBindings": [ diff --git a/services/ontology-core/docs/CONTEXT_BINDINGS.md b/services/ontology-core/docs/CONTEXT_BINDINGS.md index 901a582..64c705b 100644 --- a/services/ontology-core/docs/CONTEXT_BINDINGS.md +++ b/services/ontology-core/docs/CONTEXT_BINDINGS.md @@ -79,7 +79,7 @@ If L2 runtime is known, add: ```json { "runtime_workflow_id": "...", - "n8n_instance_id": "..." + "runtime_instance_id": "..." } ``` diff --git a/services/ontology-core/docs/GELIOS_DOMAIN_ONTOLOGY.md b/services/ontology-core/docs/GELIOS_DOMAIN_ONTOLOGY.md new file mode 100644 index 0000000..a36c5b9 --- /dev/null +++ b/services/ontology-core/docs/GELIOS_DOMAIN_ONTOLOGY.md @@ -0,0 +1,109 @@ +# Gelios Domain Ontology + +Package: `catalog/domain-packages/gelios` + +Status: `v0.1.0` source-evidenced and product-required slice for the Robot2B client context and the Gelios provider. + +## Purpose and boundary + +The package describes the meaning of Gelios fleet, telemetry, sensor, spatial, report and command concepts. It is not a Gelios client, token store, telemetry database, Cesium implementation or Engine workflow. + +The practical audit established a wide REST read surface: 107 currently visible units, 105 with `lastMsg`, a visible administrative account, unit groups, telemetry variants, sensor catalogues, command-template catalogues, geozone groups and report templates. The REST OpenAPI documents 295 operations. This ontology therefore separates three things that must not be conflated: + +1. documented provider capability; +2. actual runtime read evidence for the current account; +3. approved Robot2B product scope. + +The 95-unit legacy Engine snapshot is a temporary compatibility reference only. It is not the final business scope. The provider's current groups also do not resolve that scope automatically. A Robot2B owner must approve an allowlist or equivalent rule represented by `gelios.access_scope`. + +## Canonical value contracts + +### Unit and device + +`gelios.unit` is the stable business subject. The durable key is a namespaced provider identity such as `gelios.unit:`, never a renderer entity ID or a display name. A unit can use a `gelios.tracker_device`, belong to multiple `gelios.unit_group` records and expose sensor, fuel, maintenance and custom-field configurations. + +Hardware IDs, IMEI, phones and decrypt-related fields are restricted or secret-class data. They are not part of the default Studio contract. + +### Current telemetry + +`gelios.telemetry_snapshot` is the normalized current state produced by the Gateway, with at least: + +```text +unitSubjectId +observedAt +receivedAt +position: { latitude, longitude, height?, course?, speed?, satellites? } +operationalStatus +sensorReadings[] +counterValues[] +qualityFlags[] +source: gelios-rest +``` + +The source supports a raw and a pure last-message variant. Pure data may include sensors, counters and accumulated fields. Raw messages and dynamic `params` remain behind restricted policy and may be retained separately for diagnostics only. `gelios.position_fix` is the time-qualified position portion used by spatial consumers; it is not itself a pin. + +### Sensors and operational semantics + +`gelios.sensor_definition`, `gelios.sensor_conversion` and `gelios.sensor_reading` separate stable device configuration, calibration and time-series readings. Fuel profile, maintenance plan and custom field are independent configuration concepts. No assumed sensor meaning should be added to analytics until its message parameter, measure and conversion semantics are approved. + +### Spatial binding and Cesium pins + +Gelios is a source domain; Cesium is a renderer adapter. The binding is deliberately provider-neutral: + +```text +gelios.unit + -> map.moving_object (stable spatial subject) +gelios.position_fix + -> map.moving_object (current position update) +map.moving_object + -> map.pin + map.label + map.visibility_rule + -> map.renderer_adapter (Cesium today, replaceable later) +``` + +The future map payload selects a stable `subjectId`, position, normalized operational status, observed time and approved label fields. It must not contain Gelios credentials, raw payloads or Cesium transient entity references. This is the replacement boundary for the current M-map node-to-Cesium coupling. + +The Map Page resolves the selected Studio profile to the shared map.pin contract. The initial accepted presentation is an elevated spike — ground anchor, stem, outlined head and label — but that visual configuration remains on the Studio side. Gelios publishes data for a stable subject; it never publishes Cesium primitive settings, pin pixels or a renderer entity ID. + +The proposed renderer-neutral payload is in [`examples/gelios-map-moving-object.fixture.json`](../examples/gelios-map-moving-object.fixture.json). It uses synthetic coordinates only: it specifies the interface between Gateway and Map View, not a live trike position and not a Cesium entity. + +### Commands + +`gelios.command_template` and `gelios.command_group` are read-only catalogue concepts. `gelios.command_dispatch`, `gelios.command_delivery` and `gelios.command_audit` are a separate red domain. + +No collection run, workflow, map click or autonomous agent may create a dispatch. A future dispatch requires an explicit human action, approved scope, selected unit/group and template/parameters, confirmation, an audit record and delivery-state reconciliation. This ontology package does not grant or test a write permission. + +## Future data plane + +The intended flow is: + +```text +Gelios REST OAuth + -> server-side Gelios Gateway + -> scope filter + field policy + normalizer + -> durable telemetry/configuration storage + -> NDC level-2 workflow / internal data contract + -> Map View binding / Cesium renderer adapter +``` + +`gelios.collection_run` and `gelios.ingestion_cursor` define collection governance independently of the transport implementation. They support bounded incremental history ingestion later: one approved unit scope, limited time window, field whitelist, volume limit and durable checkpoint. They do not imply that all history should be collected now. + +Database technology is intentionally not chosen by Ontology Core. The proposed platform data-plane boundary and its validation gates are documented in [`platform/docs/ADR_GELIOS_DATA_PLANE.md`](../../../docs/ADR_GELIOS_DATA_PLANE.md). The ontology supplies the record boundaries needed by that ADR. + +## Guardrails + +- Ontology Core never stores secrets, runtime snapshots, full raw telemetry or large geometry datasets. +- Current provider-account visibility is not the approved Robot2B scope; collection must enforce an owner-approved scope before every run. +- GET alone does not make a route safe: login-as, temporary tokens, configuration and WLN download are excluded from telemetry collection. +- Geozones require paging, filtering, volume controls and map LOD. +- Command send and mutation routes remain red even if the account has documented access. +- Map selection and pins target stable domain subjects, not provider payload IDs or renderer objects. + +## Evidence and next implementation decisions + +Evidence is limited to official REST OpenAPI, safe GET results, historical Engine donor inspection and the MAP package. The next decisions outside this ontology package are: + +1. owner-approved Robot2B allowlist/rule; +2. field policy for Studio, analytics and raw retention; +3. data-plane ADR for realtime telemetry and history; +4. Gelios Gateway implementation boundary and deployment location; +5. fixture and renderer-adapter contract for Map View pins and labels. diff --git a/services/ontology-core/docs/MAP_DOMAIN_ONTOLOGY.md b/services/ontology-core/docs/MAP_DOMAIN_ONTOLOGY.md index 7224c91..2589994 100644 --- a/services/ontology-core/docs/MAP_DOMAIN_ONTOLOGY.md +++ b/services/ontology-core/docs/MAP_DOMAIN_ONTOLOGY.md @@ -30,6 +30,33 @@ NDC domain source Cesium is one possible renderer adapter. A future Cesium version or another provider can replace it without changing Map View, application manifests or domain subject identifiers. +## Interface semantic ownership + +Interface entities belong in this shared Ontology Core, not in a parallel Studio-specific ontology: + +| Layer | Owns | +| --- | --- | +| Ontology Core | Canonical meanings and relations: future.interface_view, future.interface_widget, future.interface_binding, map.view, map.moving_object, map.pin, map.label, map.visibility_rule and map.renderer_adapter. | +| NDC Module Studio | React implementation, saved visual profiles, page templates, application manifests and the concrete layout chosen by an application owner. | +| Engine / NDC workflow | Source integration, normalization and emission of stable subject IDs, current positions, statuses and approved display fields. | +| Platform Map Gateway | Provider tokens, cache, proxying and the renderer runtime boundary. | + +Ontology Core never becomes a UI database. It does not store a user's live page layout, camera snapshot, provider token, renderer object ID, telemetry feed or tile cache. + +## Pin presentation contract v0.1 + +map.pin is a reusable interface semantic, not a Cesium entity. The initial canonical variant is elevated-spike, based on the accepted Gelios donor: + +~~~text +ground anchor + -> coloured stem at a semantic height + -> outlined point/head + -> shared map.label with explicit anchor and offset + -> separate pin and label visibility thresholds +~~~ + +Studio resolves a selected map.pin style profile into this contract. The renderer adapter decides whether its concrete implementation is a polyline and point, a mesh, a sprite or another primitive. A source payload only identifies the stable moving subject, its position/status and approved label fields; it never specifies renderer primitives or provider-specific IDs. + ## First acceptance slice The first fixture-backed Map Page must cover: diff --git a/services/ontology-core/docs/baseline/CANONICAL_ENTITY_CATALOG.md b/services/ontology-core/docs/baseline/CANONICAL_ENTITY_CATALOG.md index 87a1520..ac6ad05 100644 --- a/services/ontology-core/docs/baseline/CANONICAL_ENTITY_CATALOG.md +++ b/services/ontology-core/docs/baseline/CANONICAL_ENTITY_CATALOG.md @@ -112,14 +112,14 @@ ENGINE remains workflow/dev environment. It may reference OPS and assistants, bu | `engine.workflow_owner` | ENGINE Workflow Owner | source-confirmed | ENGINE | Owner field in ACL. | | `engine.workflow_share` | ENGINE Workflow Share | source-confirmed | ENGINE | Share/invite/user access routes. | | `engine.workflow_access_request` | ENGINE Workflow Access Request | source-confirmed | ENGINE | Access request route/model for workflow access. | -| `engine.workflow_l2` | ENGINE L2 Workflow | source-evidenced | ENGINE / n8n bridge | n8n subworkflow attached to L1 node by `workflowId` + `nodeId`. | -| `engine.node_l2` | ENGINE L2 Node | source-evidenced | ENGINE / n8n bridge | Node inside compiled/deployed n8n workflow. | -| `engine.l2_runtime_core` | ENGINE L2 Runtime Core | source-evidenced | ENGINE / n8n bridge | Target n8n runtime/core instance. | -| `engine.workflow_l2_runtime_id` | ENGINE L2 Runtime Workflow ID | source-confirmed | ENGINE / n8n bridge | n8n runtime workflow id, separate from NodeDC workflow id. | -| `engine.l2_execution` | ENGINE L2 Execution | source-confirmed | ENGINE / n8n bridge | n8n execution id mapped to run/execution records. | +| `engine.workflow_l2` | ENGINE L2 Workflow | source-evidenced | ENGINE execution bridge | L2 workflow attached to L1 node by `workflowId` + `nodeId`. | +| `engine.node_l2` | ENGINE L2 Node | source-evidenced | ENGINE execution bridge | Node inside a compiled/deployed L2 workflow. | +| `engine.l2_runtime_core` | ENGINE L2 Runtime Core | source-evidenced | ENGINE execution bridge | Target Engine execution core. | +| `engine.workflow_l2_runtime_id` | ENGINE L2 Runtime Workflow ID | source-confirmed | ENGINE execution bridge | L2 runtime workflow id, separate from NodeDC L1 workflow id. | +| `engine.l2_execution` | ENGINE L2 Execution | source-confirmed | ENGINE execution bridge | L2 execution id mapped to run/execution records. | | `engine.l2_run` | ENGINE L2 Run | source-confirmed | ENGINE-side OpsLayer | Runtime run id used to bind execution/session. Current implementation is tech-debt-adjacent. | | `engine.l2_session` | ENGINE L2 Session | source-confirmed | ENGINE-side OpsLayer | Runtime session anchor, often derived from run id if upstream omits session id. | -| `engine.l2_runtime_event` | ENGINE L2 Runtime Event | source-confirmed | ENGINE / n8n bridge | `workflow:start`, `workflow:finish`, `node:start`, `node:success`, `node:error`. | +| `engine.l2_runtime_event` | ENGINE L2 Runtime Event | source-confirmed | ENGINE execution bridge | `workflow:start`, `workflow:finish`, `node:start`, `node:success`, `node:error`. | | `engine.source_stamp` | ENGINE Source Stamp | pending | ENGINE | Candidate for future source/version traceability. | ## Assistant / AI Workspace @@ -161,4 +161,3 @@ These names identify current working areas that must not leak into canonical cor | `engine.tech_debt.tender_storage` | Tender Storage | tech-debt-noncanonical | ENGINE | Current domain storage in ENGINE-side implementation. | | `engine.tech_debt.tender_ai_review` | Tender AI Review | tech-debt-noncanonical | ENGINE | Current AI review implementation for tender domain. | | `engine.tech_debt.ops_tabs_control` | ENGINE OPS Tabs Control | tech-debt-noncanonical | ENGINE | UI/control surface tied to current OpsLayer experiment. | - diff --git a/services/ontology-core/docs/baseline/EVIDENCE_HARDENING_PLAN_V0_3_1.md b/services/ontology-core/docs/baseline/EVIDENCE_HARDENING_PLAN_V0_3_1.md index 8cc4c0a..fc3419a 100644 --- a/services/ontology-core/docs/baseline/EVIDENCE_HARDENING_PLAN_V0_3_1.md +++ b/services/ontology-core/docs/baseline/EVIDENCE_HARDENING_PLAN_V0_3_1.md @@ -73,7 +73,7 @@ runtime/data/storage/env/secrets/logs/dumps/node_modules/build outputs .env / .env.* credentials / keys / pem / ssh DB dumps / archives / backup contents -postgres/n8n runtime data +postgres/Engine L2 runtime data ``` Не менять: @@ -600,7 +600,7 @@ search: /api/workflows/:id/share search: workflow-access-requests search: engine-role-access-requests -NODEDC/NODEDC_ENGINE_INFRA/nodedc-source/server/routes/n8n.js +Engine execution deployment route search: runtimeWorkflowId search: resolveNodeDcByRuntimeWorkflowId search: resolveWorkflowIdByNodeDcBindingRest @@ -608,12 +608,12 @@ search: executionId NODEDC/NODEDC_ENGINE_INFRA/nodedc-source/server/ops/runtimeTap.js search: runtimeWorkflowId -search: n8nCoreId +search: runtime core id search: executionId search: runId search: sessionId -NODEDC/NODEDC_ENGINE_INFRA/nodedc-source/services/n8n/runtime-plugin/hooks.js +Engine execution runtime hooks search: workflow:start search: workflow:finish search: node:start @@ -641,7 +641,7 @@ source-evidenced by previous ENGINE pass; direct links pending. Риск смешения терминов: ```text -n8n names are implementation aliases for Engine L2 runtime, not canonical product names. +The Engine L2 runtime is the only canonical product term in ontology and documentation. ``` ### 3.9. ENGINE / Assistant P1 — Assistant, Provider, Executor, Bridge diff --git a/services/ontology-core/docs/baseline/EVIDENCE_LEDGER.md b/services/ontology-core/docs/baseline/EVIDENCE_LEDGER.md index bba324c..b816b51 100644 --- a/services/ontology-core/docs/baseline/EVIDENCE_LEDGER.md +++ b/services/ontology-core/docs/baseline/EVIDENCE_LEDGER.md @@ -13,7 +13,7 @@ This file indexes the source-backed evidence ledgers. It is not a replacement fo | `EVIDENCE_LEDGER_OPS_P0.md` | OPS Product | source-backed P0 | Confirms workspace/project/card model and `ops.card` as canonical work object. | | `EVIDENCE_LEDGER_OPS_GATEWAY_P1.md` | OPS Gateway / MCP access | source-backed P1 | Confirms scoped agent identity/token/grant/scope/tool/idempotency/audit boundary. | | `EVIDENCE_LEDGER_ENGINE_DIRTY_BOUNDARY_P1.md` | ENGINE-side OpsLayer / Agent Monitor / tender tech debt | source-backed P1 | Confirms this is non-canonical tech debt and must not define OPS Product. | -| `EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md` | ENGINE L1/L2 workflow and runtime evidence | source-backed P1 | Confirms L1 graph, workflow ACL/share, L2 n8n runtime ids/events/run/session bridge. | +| `EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md` | ENGINE L1/L2 workflow and runtime evidence | source-backed P1 | Confirms L1 graph, workflow ACL/share, L2 runtime ids/events/run/session bridge. | ## Source Roots Used @@ -36,7 +36,7 @@ This file indexes the source-backed evidence ledgers. It is not a replacement fo - ENGINE-side Agent Monitor/tender UI is working tech debt, not target architecture. - ENGINE L1/L2 split is real enough for first ontology implementation: - L1 = NodeDC canvas/workflow graph; - - L2 = n8n/subworkflow runtime attached to L1 node. + - L2 = Engine execution workflow attached to an L1 node. - Future interface layer is a planned platform service/layer, motivated by current ENGINE-side custom UI debt. ## Evidence Gaps That Remain Non-Blocking @@ -46,4 +46,3 @@ This file indexes the source-backed evidence ledgers. It is not a replacement fo - `ops.card_type` needs taxonomy. - Gateway pairing/entitlement flow needs route-level hardening. - Domain ontology packages need separate passes; core must not absorb domain-specific tender/ecology/transport concepts. - diff --git a/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_DIRTY_BOUNDARY_P1.md b/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_DIRTY_BOUNDARY_P1.md index 0f3892b..b8c26e2 100644 --- a/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_DIRTY_BOUNDARY_P1.md +++ b/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_DIRTY_BOUNDARY_P1.md @@ -28,7 +28,7 @@ OPS product != ENGINE-side OpsLayer / tender-agent / Agent Monitor ``` -The ENGINE repository contains a substantial embedded `OpsLayer` around n8n workflows, tender search/review, monitor profiles, agent runs, trace events, and Agent Monitor UI nodes. +The ENGINE repository contains a substantial embedded `OpsLayer` around L2 execution workflows, tender search/review, monitor profiles, agent runs, trace events, and Agent Monitor UI nodes. This code is working product/legacy infrastructure, but it must be treated as: @@ -43,14 +43,14 @@ It should not be promoted into canonical OPS product truth and should not define | Boundary ID | Canonical treatment | Evidence status | Evidence file:line | Evidence explanation | Notes | | --- | --- | --- | --- | --- | --- | | `engine.tech_debt.ops_layer` | Engine-side legacy OpsLayer | source-confirmed tech debt | `server/index.js:25`, `server/index.js:333`, `server/routes/ops.js:1-34`, `server/routes/ops.js:6412-11855` | ENGINE imports `opsRouter` and mounts it at `/api/ops`; `routes/ops.js` implements a large set of run/result/tender/AI review/monitor endpoints. | This is not the separate OPS product / Tasker model. | -| `engine.tech_debt.ops_agent_instance` | Legacy Engine agent instance | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:89-137`, `src/store.ts:444-480`, `src/utils/opsApi.ts:759-855` | OpsLayer has `agents`, `agent_instances`, `agent_runs`; frontend prevents copied n8n nodes from inheriting `opsAgentInstanceId`; API can start/stop agent instances and record runs. | Do not merge with OPS Gateway `agent.identity`. | +| `engine.tech_debt.ops_agent_instance` | Legacy Engine agent instance | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:89-137`, `src/store.ts:444-480`, `src/utils/opsApi.ts:759-855` | OpsLayer has `agents`, `agent_instances`, `agent_runs`; frontend prevents copied L2 nodes from inheriting `opsAgentInstanceId`; API can start/stop agent instances and record runs. | Do not merge with OPS Gateway `agent.identity`. | | `engine.tech_debt.agent_monitor_node` | Agent Monitor canvas node | source-confirmed tech debt | `src/nodes/AgentMonitorNode.tsx:10-14`, `src/nodes/AgentMonitorNode.tsx:21-38`, `src/driveinspector/nodes/AgentMonitor.definition.ts:1-13` | ENGINE has a visible `agentMonitor` React Flow node and inspector definition that binds monitor profiles/agents through `opsApi`. | This is current UI embedding, not final interface layer. | -| `engine.tech_debt.monitor_profile` | Engine-side monitor profile | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:240-299`, `src/utils/opsApi.ts:260-360`, `src/utils/opsApi.ts:460-520` | OpsLayer stores monitor profiles, profile assignments, bound agent instances, n8n cores, and monitor nodes; frontend can list/create/update/bind profiles and save monitor layouts. | Useful evidence for future work-view requirements. | -| `engine.tech_debt.tender_domain_ui` | Tender-specific embedded UI/domain slice | source-confirmed tech debt | `server/routes/ops.js:45-63`, `server/routes/ops.js:75-145`, `src/n8n/N8nAgentMonitorPanel.tsx:1-44`, `src/n8n/N8nAgentMonitorPanel.tsx:1160-1245`, `src/n8n/tenderElectronicPlaceCountries.ts:1-12` | ENGINE includes Kontur tender URLs, tender categories/electronic places, tender search settings, and a large Agent Monitor panel. | Treat as procurement/tender domain evidence, not core ontology. | +| `engine.tech_debt.monitor_profile` | Engine-side monitor profile | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:240-299`, `src/utils/opsApi.ts:260-360`, `src/utils/opsApi.ts:460-520` | OpsLayer stores monitor profiles, profile assignments, bound agent instances, execution cores, and monitor nodes; frontend can list/create/update/bind profiles and save monitor layouts. | Useful evidence for future work-view requirements. | +| `engine.tech_debt.tender_domain_ui` | Tender-specific embedded UI/domain slice | source-confirmed tech debt | Engine internal monitor implementation | ENGINE includes Kontur tender URLs, tender categories/electronic places, tender search settings, and a large Agent Monitor panel. | Treat as procurement/tender domain evidence, not core ontology. | | `engine.tech_debt.tender_storage` | Tender storage inside Engine OpsLayer | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:210-239`, `server/ops/migrations/0006_tender_dedup_registry.sql:1-17`, `server/ops/migrations/0007_tender_details.sql:1-24`, `server/ops/migrations/0008_tender_documents.sql:1-37` | OpsLayer stores tender results, dedup registry, details, and downloaded documents. | Do not confuse with future domain ontology persistence. | -| `engine.tech_debt.tender_ai_review` | Tender AI review workflow state | source-confirmed tech debt | `server/ops/migrations/0009_tender_ai_review.sql:1-90`, `src/n8n/N8nAgentMonitorPanel.tsx:1214-1238`, `src/utils/opsApi.ts:577-688` | OpsLayer stores AI review tasks/latest state; UI has default manual AI criteria/response format; API enqueues/retries review tasks. | Useful sample for domain-specific assistant review flows. | +| `engine.tech_debt.tender_ai_review` | Tender AI review workflow state | source-confirmed tech debt | Engine internal monitor implementation | OpsLayer stores AI review tasks/latest state; UI has default manual AI criteria/response format; API enqueues/retries review tasks. | Useful sample for domain-specific assistant review flows. | | `engine.tech_debt.ops_tabs_control` | Engine inspector ops control | source-confirmed tech debt | `src/driveinspector/ControlRegistry.tsx:27-50` | Drive inspector registers `opsTabs` control. | Another marker that Ops UI is embedded in ENGINE editor controls. | -| `future.interface_layer` | Future platform UI/work-view layer | product-required / source-informed | `src/n8n/N8nAgentMonitorPanel.tsx:46-120`, `src/utils/opsApi.ts:460-520`, `server/ops/migrations/0001_init_opslayer.sql:240-299` | Current monitor panel/layout/profile code shows the kind of configurable work views the future interface layer must support. | Future home should be separate platform service/app, not Engine core. | +| `future.interface_layer` | Future platform UI/work-view layer | product-required / source-informed | Engine internal monitor implementation; `src/utils/opsApi.ts:460-520`, `server/ops/migrations/0001_init_opslayer.sql:240-299` | Current monitor panel/layout/profile code shows the kind of configurable work views the future interface layer must support. | Future home should be separate platform service/app, not Engine core. | ## Route / Surface Evidence @@ -58,9 +58,9 @@ It should not be promoted into canonical OPS product truth and should not define | --- | --- | --- | --- | | `/api/ops` is mounted inside ENGINE server | source-confirmed | `server/index.js:25`, `server/index.js:333` | ENGINE server imports and mounts `opsRouter`. | | Ops router owns runs/results/tender/AI review/monitor APIs | source-confirmed | `server/routes/ops.js:6412-11855` | Router exposes health, runs, tender results/details/docs, AI review, trace, monitor profiles/layout/admin, agent instance start/stop, actions, contour limits/search. | -| Agent Monitor is an ENGINE canvas node | source-confirmed | `src/nodes/AgentMonitorNode.tsx:10-14`, `src/nodes/AgentMonitorNode.tsx:73-170` | Node type is `agentMonitor`; UI opens monitor through n8n subworkflow event. | -| Agent Monitor panel directly consumes Ops APIs | source-confirmed | `src/n8n/N8nAgentMonitorPanel.tsx:5-31` | Panel imports many Ops API functions for tender details, AI review, layout, runs, trace, start/stop, take-in-work. | -| Engine graph copy logic special-cases Ops bindings | source-confirmed | `src/store.ts:444-480` | Copied n8n nodes clear `opsAgentInstanceId`; copied `agentMonitor` nodes clear monitor profile/binding fields. | +| Agent Monitor is an ENGINE canvas node | source-confirmed | `src/nodes/AgentMonitorNode.tsx:10-14`, `src/nodes/AgentMonitorNode.tsx:73-170` | Node type is `agentMonitor`; UI opens monitor through an L2 execution event. | +| Agent Monitor panel directly consumes Ops APIs | source-confirmed | Engine internal monitor implementation | Panel imports many Ops API functions for tender details, AI review, layout, runs, trace, start/stop, take-in-work. | +| Engine graph copy logic special-cases Ops bindings | source-confirmed | `src/store.ts:444-480` | Copied L2 nodes clear `opsAgentInstanceId`; copied `agentMonitor` nodes clear monitor profile/binding fields. | ## Guardrails Confirmed diff --git a/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md b/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md index 35a4456..ad61480 100644 --- a/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md +++ b/services/ontology-core/docs/baseline/EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md @@ -3,7 +3,7 @@ Status: source-backed ledger section Baseline: Canonical Entity Catalog v0.4-pre Date: 2026-06-18 -Scope: ENGINE L1/L2 workflow, ACL/share, n8n runtime bridge, runtime event tracking +Scope: ENGINE L1/L2 workflow, ACL/share, execution bridge, runtime event tracking Source root: @@ -25,7 +25,7 @@ This pass confirms the minimum ENGINE ontology needed for the first implementati - `engine.workflow_l1`, `engine.node_l1`, and `engine.edge_l1` exist as NodeDC/React Flow graph concepts. - `engine.node_type_l1` is sourced from the auto node registry. - Workflow ACL/share/access-request concepts are implemented in source. -- `engine.workflow_l2` is source-evidenced as an n8n subworkflow attached to an L1 workflow/node pair. +- `engine.workflow_l2` is source-evidenced as a protected execution workflow attached to an L1 workflow/node pair. - Runtime events carry `workflowId`, `nodeId`, `runtimeWorkflowId`, `executionId`, `runId`, and `sessionId`. - Current runtime run/session tracking is tied to ENGINE-side OpsLayer tables and should remain tech-debt-adjacent. @@ -41,14 +41,14 @@ This pass confirms the minimum ENGINE ontology needed for the first implementati | `engine.workflow_owner` | source-confirmed | `server/workflows/acl.js:109-165` | ACL owner field is normalized and role checked. | | `engine.workflow_share` | source-confirmed | `server/index.js:3185-3245`, `server/index.js:3250-3312`, `server/index.js:3371-3635` | Workflow share routes expose owner/users/groups/invites and email/internal invite operations. | | `engine.workflow_access_request` | source-confirmed | `server/index.js:2792`, `server/index.js:2860`, `server/index.js:2916`, `server/index.js:3314` | Workflow access request, approve, reject, and request routes are present. | -| `engine.workflow_l2` | source-evidenced | `server/routes/n8n.js:11661-11737` | `/api/n8n/subworkflow/deploy` loads subworkflow graph for `workflowId` + `nodeId`, compiles it, injects runtime events, and writes compiled n8n workflow. | -| `engine.node_l2` | source-evidenced | `server/routes/n8n.js:2288-2365`, `services/n8n/runtime-plugin/hooks.js:132-145` | Runtime event node and hooks capture n8n node name/id. | -| `engine.l2_runtime_core` | source-evidenced | `server/routes/n8n.js:11685-11696`, `server/routes/n8n.js:11739-11748` | Deploy resolves n8n target/base URL/instance and docker compose service/container. | -| `engine.workflow_l2_runtime_id` | source-confirmed | `services/n8n/runtime-plugin/hooks.js:52-61`, `server/routes/n8n.js:2323-2324`, `server/routes/n8n.js:12041-12055` | Runtime plugin extracts n8n workflow id; injected event uses `$workflow.id`; deploy sync stores resolved workflow id. | -| `engine.l2_execution` | source-confirmed | `services/n8n/runtime-plugin/hooks.js:77-86`, `server/routes/n8n.js:2323-2325`, `server/ops/runtimeTap.js:773-818`, `server/ops/runtimeTap.js:820-855` | Runtime events carry execution id; runtime tap writes trace event and run execution binding. | -| `engine.l2_run` | source-confirmed / tech-debt-adjacent | `services/n8n/runtime-plugin/hooks.js:88-106`, `server/ops/runtimeTap.js:247-270`, `server/ops/runtimeTap.js:467-542` | Hooks extract run id; runtime tap resolves/canonicalizes run id through ENGINE-side OpsLayer tables. | -| `engine.l2_session` | source-confirmed / tech-debt-adjacent | `services/n8n/runtime-plugin/hooks.js:108-130`, `server/ops/runtimeTap.js:272-289`, `server/ops/runtimeTap.js:354-430` | Hooks extract session id or fall back to run id; runtime tap resolves session anchors and can create session-workflow split runs. | -| `engine.l2_runtime_event` | source-confirmed | `services/n8n/runtime-plugin/hooks.js:220-231`, `services/n8n/runtime-plugin/hooks.js:233-279`, `services/n8n/runtime-plugin/hooks.js:285-387`, `server/routes/n8n.js:2298-2332`, `services/n8n/runtime-plugin/runtime-bridge.js:200-223` | Runtime events are emitted/enqueued for workflow/node start/finish/success/error and sent to NodeDC runtime endpoint with retry/failure handling. | +| `engine.workflow_l2` | source-evidenced | Engine execution deployment route | Deploy loads the L2 graph for `workflowId` + `nodeId`, compiles it, injects runtime events, and writes the execution workflow. | +| `engine.node_l2` | source-evidenced | Engine execution hooks | Runtime event hooks capture L2 node name/id. | +| `engine.l2_runtime_core` | source-evidenced | Engine execution deployment route | Deploy resolves the execution target/base URL/instance and service/container. | +| `engine.workflow_l2_runtime_id` | source-confirmed | Engine execution hooks and deploy sync | Runtime bridge extracts workflow id; injected event uses `$workflow.id`; deploy sync stores the resolved id. | +| `engine.l2_execution` | source-confirmed | Engine execution hooks and runtime tap | Runtime events carry execution id; runtime tap writes trace event and run execution binding. | +| `engine.l2_run` | source-confirmed / tech-debt-adjacent | Engine execution hooks and runtime tap | Hooks extract run id; runtime tap resolves/canonicalizes it through ENGINE-side OpsLayer tables. | +| `engine.l2_session` | source-confirmed / tech-debt-adjacent | Engine execution hooks and runtime tap | Hooks extract session id or fall back to run id; runtime tap resolves session anchors and can create session-workflow split runs. | +| `engine.l2_runtime_event` | source-confirmed | Engine execution hooks and runtime bridge | Runtime events are emitted/enqueued for workflow/node start/finish/success/error and sent to NodeDC runtime endpoint with retry/failure handling. | ## Guardrails Confirmed @@ -62,6 +62,5 @@ Engine-side OpsLayer runtime tables do not define OPS Product ontology. ## Non-Blocking Follow-Up - Harden `engine.source_stamp` after implementation needs are clearer. -- Separate stable L2 domain concepts from current n8n-specific deployment helpers. +- Separate stable L2 domain concepts from current execution deployment helpers. - Decide whether runtime run/session belongs in future Ontology Core, Interface Layer, OPS Gateway, or a dedicated runtime observability package. - diff --git a/services/ontology-core/docs/baseline/GUARDRAILS.md b/services/ontology-core/docs/baseline/GUARDRAILS.md index 87271f8..07313da 100644 --- a/services/ontology-core/docs/baseline/GUARDRAILS.md +++ b/services/ontology-core/docs/baseline/GUARDRAILS.md @@ -44,7 +44,7 @@ These rules exist to stop future agents and developers from merging similar-look - ENGINE is workflow/dev environment, not the final home for heavy product UIs. - `engine.workflow_l1` is the NodeDC canvas/workflow graph. -- `engine.workflow_l2` is a runtime/subworkflow layer, currently n8n-backed. +- `engine.workflow_l2` is the protected Engine execution layer. - `engine.workflow_l2_runtime_id` is not the same as NodeDC `engine.workflow_l1` id. - `engine.l2_run`, `engine.l2_session`, and current trace tables are useful evidence but are tech-debt-adjacent because they live in ENGINE-side OpsLayer. - Do not turn ENGINE-side Agent Monitor/tender UI into core ontology roots. @@ -74,4 +74,3 @@ These rules exist to stop future agents and developers from merging similar-look - Do not delete files unless explicitly asked by the user. - Do not edit application source code from the ontology workspace. - Do not run Docker/build/install/test for docs-only ontology work. - diff --git a/services/ontology-core/docs/baseline/RELATION_CATALOG.md b/services/ontology-core/docs/baseline/RELATION_CATALOG.md index a1cfc09..116b499 100644 --- a/services/ontology-core/docs/baseline/RELATION_CATALOG.md +++ b/services/ontology-core/docs/baseline/RELATION_CATALOG.md @@ -68,10 +68,10 @@ Relations are written as product-level semantics, not database foreign keys. Imp | `engine.workflow_acl.has_owner` | `engine.workflow_acl` | `engine.workflow_owner` | ACL owns owner field. | source-confirmed | | `engine.workflow_l1.has_share` | `engine.workflow_l1` | `engine.workflow_share` | Workflow can be shared/invited. | source-confirmed | | `engine.workflow_l1.has_access_request` | `engine.workflow_l1` | `engine.workflow_access_request` | Access requests target workflow. | source-confirmed | -| `engine.node_l1.embeds_l2_workflow` | `engine.node_l1` | `engine.workflow_l2` | n8n subworkflow is attached to L1 node. | source-evidenced | -| `engine.workflow_l2.deployed_as_runtime_workflow` | `engine.workflow_l2` | `engine.workflow_l2_runtime_id` | Compiled n8n workflow deploy returns runtime workflow id. | source-confirmed | -| `engine.workflow_l2.runs_on_runtime_core` | `engine.workflow_l2` | `engine.l2_runtime_core` | L2 workflow targets n8n instance/core. | source-evidenced | -| `engine.l2_execution.belongs_to_runtime_workflow` | `engine.l2_execution` | `engine.workflow_l2_runtime_id` | Execution is emitted for n8n workflow id. | source-confirmed | +| `engine.node_l1.embeds_l2_workflow` | `engine.node_l1` | `engine.workflow_l2` | L2 execution workflow is attached to an L1 node. | source-evidenced | +| `engine.workflow_l2.deployed_as_runtime_workflow` | `engine.workflow_l2` | `engine.workflow_l2_runtime_id` | L2 deploy returns a runtime workflow id. | source-confirmed | +| `engine.workflow_l2.runs_on_runtime_core` | `engine.workflow_l2` | `engine.l2_runtime_core` | L2 workflow targets the Engine execution core. | source-evidenced | +| `engine.l2_execution.belongs_to_runtime_workflow` | `engine.l2_execution` | `engine.workflow_l2_runtime_id` | Execution is emitted for the L2 runtime workflow id. | source-confirmed | | `engine.l2_runtime_event.describes_execution` | `engine.l2_runtime_event` | `engine.l2_execution` | Runtime events carry execution id. | source-confirmed | | `engine.l2_runtime_event.binds_run` | `engine.l2_runtime_event` | `engine.l2_run` | Event binds to run id. Current storage is ENGINE-side OpsLayer. | source-confirmed / tech-debt-adjacent | | `engine.l2_run.has_session` | `engine.l2_run` | `engine.l2_session` | Session anchors related runtime events/runs. | source-confirmed / tech-debt-adjacent | @@ -87,4 +87,3 @@ These are the first useful implementation targets for Ontology Core. | `ontology.resolves_hub_app_to_project_context` | `hub.application` | `ops.project` / `engine.workflow_l1` | Application selection can suggest project/workflow context. | product-required | | `ontology.selects_gateway_grant_context` | `assistant.bridge` | `agent.grant` | Ontology can help choose proper MCP/Gateway grant before tool calls. | product-required | | `ontology.routes_interface_view_to_domain_package` | `future.interface_view` | `future.domain_package` | Future UI layer renders domain-specific views from ontology-backed data. | future-concept | - diff --git a/services/ontology-core/docs/baseline/TERMS_GLOSSARY.md b/services/ontology-core/docs/baseline/TERMS_GLOSSARY.md index 89ebf62..f439240 100644 --- a/services/ontology-core/docs/baseline/TERMS_GLOSSARY.md +++ b/services/ontology-core/docs/baseline/TERMS_GLOSSARY.md @@ -17,8 +17,8 @@ Date: 2026-06-18 | Agent token | Opaque credential used by Gateway; not user identity. | | ENGINE | Workflow/dev environment. | | L1 workflow | NodeDC/React Flow canvas workflow. | -| L2 workflow | Runtime/subworkflow layer, currently n8n-backed. | -| Runtime workflow id | n8n runtime workflow id, separate from NodeDC L1 workflow id. | +| L2 workflow | Protected Engine execution workflow layer. | +| Runtime workflow id | Engine L2 runtime workflow id, separate from NodeDC L1 workflow id. | | Runtime event | Event emitted by L2 runtime: workflow/node start/finish/success/error. | | Assistant | Product assistant capability; exact source model pending AI Workspace pass. | | Ontology Core | Future platform service/module for canonical entities, relations, aliases, evidence, and resolver rules. | @@ -34,8 +34,8 @@ Date: 2026-06-18 | Authentik user / OIDC subject / JWT subject | `ndcauth.identity` | | Launcher service / app | `hub.application` | | Application tile / app card | `hub.application_card` | -| n8n workflow id | `engine.workflow_l2_runtime_id` | -| n8n execution | `engine.l2_execution` | +| runtime workflow id | `engine.workflow_l2_runtime_id` | +| L2 execution | `engine.l2_execution` | | Engine-side OPS / Agent Monitor OPS | `engine.tech_debt.ops_layer`, not OPS Product | | Tender Agent Monitor UI | `engine.tech_debt.tender_domain_ui`, not future Interface Layer | @@ -46,4 +46,3 @@ Date: 2026-06-18 - Do not say "user" when the context needs HUB user, NDCAuth identity, OPS member, or Gateway agent identity. - Do not say "task" as the root concept when the object is an OPS card. - Do not treat current tender/Agent Monitor implementation as target architecture. - diff --git a/services/ontology-core/examples/engine-workflow-manifest.example.json b/services/ontology-core/examples/engine-workflow-manifest.example.json index 003aa38..6878c4a 100644 --- a/services/ontology-core/examples/engine-workflow-manifest.example.json +++ b/services/ontology-core/examples/engine-workflow-manifest.example.json @@ -3,8 +3,8 @@ "id": "example-workflow-id", "label": "Example Engine Workflow", "nodeId": "example-node-id", - "runtimeWorkflowId": "example-n8n-runtime-workflow-id", - "n8nInstanceId": "example-n8n-instance" + "runtimeWorkflowId": "example-runtime-workflow-id", + "runtimeInstanceId": "example-engine-runtime-instance" }, "context": { "status": "planned", diff --git a/services/ontology-core/examples/gelios-map-moving-object.fixture.json b/services/ontology-core/examples/gelios-map-moving-object.fixture.json new file mode 100644 index 0000000..a732e52 --- /dev/null +++ b/services/ontology-core/examples/gelios-map-moving-object.fixture.json @@ -0,0 +1,25 @@ +{ + "contractVersion": "0.1.0", + "kind": "map.moving_object.current_position", + "source": "gelios", + "subjectId": "gelios.unit:example-unit-id", + "observedAt": "2026-07-13T00:00:00.000Z", + "receivedAt": "2026-07-13T00:00:03.000Z", + "position": { + "latitude": 55.000001, + "longitude": 37.000001, + "heightMeters": 0, + "courseDegrees": 90, + "speedKph": 0, + "satellites": 0 + }, + "operationalStatus": "unknown", + "display": { + "label": "Robot2B unit", + "visibilityProfile": "fleet-default" + }, + "policy": { + "scopeApproved": true, + "fieldSet": "studio-current-position-v1" + } +} diff --git a/services/ontology-core/package.json b/services/ontology-core/package.json index d757a36..2e38188 100644 --- a/services/ontology-core/package.json +++ b/services/ontology-core/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "scripts": { + "start": "node src/mcp-server.mjs", "assistant:action": "node src/assistant-action-resolver.mjs", "assistant:caller": "node src/assistant-action-caller.mjs", "assistant:execute": "node src/assistant-action-executor.mjs", @@ -17,6 +18,7 @@ "smoke:assistant-executor": "node src/assistant-action-executor.mjs --smoke", "smoke:assistant-actions": "node src/assistant-action-resolver.mjs --smoke", "smoke:assistant-policy": "node src/assistant-policy.mjs --smoke", + "smoke:mcp": "node src/scripts/smoke-mcp.mjs", "smoke:resolver": "node src/resolver.mjs --smoke" } } diff --git a/services/ontology-core/src/catalog.mjs b/services/ontology-core/src/catalog.mjs index 37750f0..ec28e4e 100644 --- a/services/ontology-core/src/catalog.mjs +++ b/services/ontology-core/src/catalog.mjs @@ -5,11 +5,14 @@ import { fileURLToPath } from 'node:url' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) export const serviceRoot = path.resolve(__dirname, '..') -export const catalogRoot = path.join(serviceRoot, 'catalog') +export const catalogRoot = path.resolve( + process.env.NODEDC_ONTOLOGY_CATALOG_ROOT || path.join(serviceRoot, 'catalog'), +) export const domainPackagesRoot = path.join(catalogRoot, 'domain-packages') export async function readJson(relativePath) { - const fullPath = path.join(serviceRoot, relativePath) + const relative = String(relativePath || '').replace(/^catalog\//, '') + const fullPath = path.join(catalogRoot, relative) const raw = await fs.readFile(fullPath, 'utf8') return JSON.parse(raw) } @@ -180,7 +183,7 @@ function mergeCatalog(base, extension) { } async function loadDomainPackage(packageDir) { - const relativeDir = path.relative(serviceRoot, packageDir) + const relativeDir = path.relative(catalogRoot, packageDir) const metadata = await readOptionalJsonAt(path.join(packageDir, 'package.json'), {}) return { @@ -225,6 +228,8 @@ async function loadDomainPackage(packageDir) { }), domainPackages: [{ id: metadata.id || path.basename(packageDir), + version: metadata.version || '', + updatedAt: metadata.updatedAt || '', status: metadata.status || 'unknown', relativePath: relativeDir, summary: metadata.summary || '', diff --git a/services/ontology-core/src/mcp-server.mjs b/services/ontology-core/src/mcp-server.mjs new file mode 100644 index 0000000..80b4129 --- /dev/null +++ b/services/ontology-core/src/mcp-server.mjs @@ -0,0 +1,581 @@ +#!/usr/bin/env node + +import { createHash, timingSafeEqual } from 'node:crypto' +import { createServer } from 'node:http' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { canonicalizeEntityId, loadCatalog } from './catalog.mjs' +import { resolveContext } from './resolver.mjs' + +const MCP_PROTOCOL_VERSION = '2025-06-18' +const MAX_BODY_BYTES = 1024 * 1024 +const MAX_SEARCH_RESULTS = 50 + +const TOOLS = [ + { + name: 'ontology_status', + title: 'Ontology catalog status', + description: 'Returns a safe, read-only summary of the live NODE.DC ontology catalog and its loaded domain packages.', + inputSchema: { type: 'object', additionalProperties: false, properties: {} }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: 'ontology_search', + title: 'Search ontology catalog', + description: 'Searches canonical entities, relations and aliases in the live ontology catalog. Use this before introducing names or data contracts into a workflow.', + inputSchema: { + type: 'object', + additionalProperties: false, + required: ['query'], + properties: { + query: { type: 'string', minLength: 1, maxLength: 200 }, + limit: { type: 'integer', minimum: 1, maximum: MAX_SEARCH_RESULTS }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: 'ontology_get_entity', + title: 'Get ontology entity', + description: 'Resolves an entity id or alias and returns its safe definition, aliases, relations and applicable guardrails. It never returns runtime data, credentials or source filesystem paths.', + inputSchema: { + type: 'object', + additionalProperties: false, + properties: { + entityId: { type: 'string', minLength: 1, maxLength: 240 }, + term: { type: 'string', minLength: 1, maxLength: 240 }, + }, + anyOf: [{ required: ['entityId'] }, { required: ['term'] }], + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: 'ontology_get_guardrails', + title: 'Get ontology guardrails', + description: 'Returns semantic and safety guardrails for the full catalog or for one entity/alias. This is read-only advice; it does not grant capability access.', + inputSchema: { + type: 'object', + additionalProperties: false, + properties: { + entityId: { type: 'string', minLength: 1, maxLength: 240 }, + term: { type: 'string', minLength: 1, maxLength: 240 }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: 'ontology_resolve_context', + title: 'Resolve ontology context', + description: 'Resolves a semantic request to canonical entities and context routes. It deliberately returns no source-system ids, secrets, runtime data or execution capability.', + inputSchema: { + type: 'object', + additionalProperties: false, + properties: { + surface: { type: 'string', maxLength: 120 }, + entityId: { type: 'string', maxLength: 240 }, + alias: { type: 'string', maxLength: 240 }, + term: { type: 'string', maxLength: 240 }, + intent: { type: 'string', maxLength: 1000 }, + contextId: { type: 'string', maxLength: 240 }, + }, + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, +] + +export function createOntologyMcpServer(overrides = {}) { + const config = { ...readConfig(), ...overrides } + return createServer((req, res) => handleRequest(req, res, config).catch((error) => { + if (!res.headersSent) { + sendJson(res, 500, { ok: false, error: 'ontology_mcp_internal_error' }) + } else { + res.destroy(error) + } + })) +} + +async function handleRequest(req, res, config) { + const url = new URL(req.url || '/', 'http://localhost') + if (url.pathname === '/healthz') { + const catalog = await loadCatalog() + sendJson(res, 200, { + ok: true, + service: 'nodedc-ontology-core-mcp', + mcp: { + protocolVersion: MCP_PROTOCOL_VERSION, + tools: TOOLS.length, + readOnly: true, + }, + catalog: catalogStatus(catalog), + internalApiConfigured: config.internalTokens.length > 0, + }) + return + } + + if (url.pathname !== '/mcp') { + sendJson(res, 404, { ok: false, error: 'not_found' }) + return + } + + if (!isAllowedOrigin(req.headers.origin, config.allowedOrigins)) { + sendJson(res, 403, { ok: false, error: 'mcp_origin_forbidden' }) + return + } + + if (!isAuthorized(req.headers.authorization, config.internalTokens)) { + sendJson(res, config.internalTokens.length ? 401 : 503, { + ok: false, + error: config.internalTokens.length ? 'ontology_mcp_unauthorized' : 'ontology_mcp_token_not_configured', + }) + return + } + + const method = String(req.method || 'GET').toUpperCase() + if (method === 'GET') { + res.setHeader('allow', 'POST') + sendJson(res, 405, { ok: false, error: 'mcp_sse_not_supported' }) + return + } + if (method !== 'POST') { + res.setHeader('allow', 'POST') + sendJson(res, 405, { ok: false, error: 'method_not_allowed' }) + return + } + + let payload + try { + payload = JSON.parse((await readRequestBody(req, config.maxBodyBytes)).toString('utf8')) + } catch (error) { + sendMcpError(res, null, -32700, error?.code === 'body_too_large' ? 'request_too_large' : 'parse_error') + return + } + + if (!isPlainObject(payload) || payload.jsonrpc !== '2.0' || !cleanString(payload.method, 160)) { + sendMcpError(res, isPlainObject(payload) ? payload.id ?? null : null, -32600, 'invalid_request') + return + } + + const isNotification = !Object.hasOwn(payload, 'id') + const response = await dispatchMcpRequest(payload) + if (isNotification) { + res.statusCode = 202 + res.end() + return + } + sendJson(res, 200, response, { 'mcp-protocol-version': MCP_PROTOCOL_VERSION }) +} + +async function dispatchMcpRequest(payload) { + const id = payload.id ?? null + const method = cleanString(payload.method, 160) + const params = isPlainObject(payload.params) ? payload.params : {} + + if (method === 'initialize') { + return mcpResult(id, { + protocolVersion: MCP_PROTOCOL_VERSION, + capabilities: { tools: { listChanged: false } }, + serverInfo: { + name: 'nodedc-ontology-core', + version: '0.1.0', + }, + instructions: 'Read-only semantic catalog. It exposes no runtime telemetry, database access, credentials, source paths, workflow mutations, command dispatch or Studio rendering controls.', + }) + } + + if (method === 'notifications/initialized') return mcpResult(id, {}) + if (method === 'ping') return mcpResult(id, {}) + if (method === 'tools/list') return mcpResult(id, { tools: TOOLS }) + if (method !== 'tools/call') return mcpError(id, -32601, 'method_not_found') + + const toolName = cleanString(params.name, 120) + const argumentsValue = isPlainObject(params.arguments) ? params.arguments : {} + return mcpResult(id, await callTool(toolName, argumentsValue)) +} + +async function callTool(name, input) { + if (name === 'ontology_status') { + const catalog = await loadCatalog() + return toolResult({ + schemaVersion: 'nodedc.ontology-mcp.status.v1', + readOnly: true, + catalog: catalogStatus(catalog), + }) + } + + if (name === 'ontology_search') { + const query = cleanString(input.query, 200) + if (!query) return toolError('query_required') + const catalog = await loadCatalog() + return toolResult(searchCatalog(catalog, query, boundedLimit(input.limit))) + } + + if (name === 'ontology_get_entity') { + const catalog = await loadCatalog() + const requested = cleanString(input.entityId || input.term, 240) + const entityId = canonicalizeEntityId(requested, catalog) + const entity = catalog.entityById.get(entityId) + if (!entity) return toolError('entity_not_found', { requested }) + return toolResult(entityDetails(catalog, entity)) + } + + if (name === 'ontology_get_guardrails') { + const catalog = await loadCatalog() + const requested = cleanString(input.entityId || input.term, 240) + const entityId = requested ? canonicalizeEntityId(requested, catalog) : '' + if (requested && !catalog.entityById.has(entityId)) return toolError('entity_not_found', { requested }) + return toolResult(guardrailDetails(catalog, entityId)) + } + + if (name === 'ontology_resolve_context') { + const resolved = await resolveContext({ + surface: cleanString(input.surface, 120), + entityId: cleanString(input.entityId, 240), + alias: cleanString(input.alias, 240), + term: cleanString(input.term, 240), + intent: cleanString(input.intent, 1000), + contextId: cleanString(input.contextId, 240), + }) + return toolResult(sanitizeResolvedContext(resolved)) + } + + return toolError('tool_not_found', { name }) +} + +function catalogStatus(catalog) { + const packages = (catalog.domainPackages || []) + .map((item) => ({ + id: cleanString(item.id, 120), + version: cleanString(item.version, 80), + updatedAt: cleanString(item.updatedAt, 80), + status: cleanString(item.status, 120), + summary: cleanString(item.summary, 1000), + })) + .sort((left, right) => left.id.localeCompare(right.id)) + const snapshot = { + entityIds: (catalog.entities?.entities || []).map((item) => item.id).sort(), + relationIds: (catalog.relations?.relations || []).map((item) => item.id).sort(), + aliases: (catalog.aliases?.aliases || []).map((item) => [item.alias, item.canonicalId]).sort((left, right) => String(left[0]).localeCompare(String(right[0]))), + packages, + } + return { + schemaVersion: 'nodedc.ontology-catalog.status.v1', + catalogHash: createHash('sha256').update(JSON.stringify(snapshot)).digest('hex').slice(0, 16), + entities: snapshot.entityIds.length, + relations: snapshot.relationIds.length, + aliases: snapshot.aliases.length, + guardrails: (catalog.guardrails?.rules || []).length, + blockedConflations: (catalog.guardrails?.blockedConflations || []).length, + domainPackages: packages, + } +} + +function searchCatalog(catalog, query, limit) { + const needle = normalizeSearch(query) + const items = [] + for (const entity of catalog.entities?.entities || []) { + const aliasScore = (catalog.aliases?.aliases || []) + .filter((alias) => alias.canonicalId === entity.id) + .reduce((best, alias) => Math.max(best, searchScore(needle, alias.alias)), 0) + const score = Math.max( + aliasScore, + searchScore(needle, entity.id, entity.name, entity.summary, entity.surface, entity.authority), + ) + if (score > 0) items.push({ kind: 'entity', score, entity: safeEntity(entity) }) + } + for (const relation of catalog.relations?.relations || []) { + const score = searchScore(needle, relation.id, relation.summary, ...(relation.from || []), ...(relation.to || [])) + if (score > 0) items.push({ kind: 'relation', score, relation: safeRelation(relation) }) + } + for (const alias of catalog.aliases?.aliases || []) { + const score = searchScore(needle, alias.alias, alias.canonicalId) + if (score > 0) items.push({ kind: 'alias', score, alias: safeAlias(alias) }) + } + items.sort((left, right) => right.score - left.score || stableItemId(left).localeCompare(stableItemId(right))) + return { + schemaVersion: 'nodedc.ontology-mcp.search.v1', + query, + totalMatches: items.length, + results: items.slice(0, limit).map(({ score, ...item }) => item), + } +} + +function entityDetails(catalog, entity) { + const entityId = entity.id + return { + schemaVersion: 'nodedc.ontology-mcp.entity.v1', + entity: safeEntity(entity), + aliases: (catalog.aliases?.aliases || []) + .filter((item) => item.canonicalId === entityId) + .map(safeAlias) + .sort((left, right) => left.alias.localeCompare(right.alias)), + relations: (catalog.relations?.relations || []) + .filter((item) => (item.from || []).includes(entityId) || (item.to || []).includes(entityId)) + .map(safeRelation) + .sort((left, right) => left.id.localeCompare(right.id)), + guardrails: (catalog.guardrails?.rules || []) + .filter((item) => (item.entityIds || []).includes(entityId)) + .map(safeGuardrail) + .sort((left, right) => left.id.localeCompare(right.id)), + blockedConflations: (catalog.guardrails?.blockedConflations || []) + .filter((pair) => Array.isArray(pair) && pair.includes(entityId)) + .map((pair) => pair.map((item) => cleanString(item, 240))), + } +} + +function guardrailDetails(catalog, entityId) { + const rules = (catalog.guardrails?.rules || []) + .filter((item) => !entityId || (item.entityIds || []).includes(entityId)) + .map(safeGuardrail) + .sort((left, right) => left.id.localeCompare(right.id)) + const blockedConflations = (catalog.guardrails?.blockedConflations || []) + .filter((pair) => !entityId || (Array.isArray(pair) && pair.includes(entityId))) + .map((pair) => Array.isArray(pair) ? pair.map((item) => cleanString(item, 240)) : []) + return { + schemaVersion: 'nodedc.ontology-mcp.guardrails.v1', + entityId: entityId || null, + rules, + blockedConflations, + } +} + +function sanitizeResolvedContext(resolved) { + return { + schemaVersion: 'nodedc.ontology-mcp.context-resolution.v1', + input: { + surface: cleanString(resolved?.input?.surface, 120), + entityId: cleanString(resolved?.input?.entityId, 240), + intent: cleanString(resolved?.input?.intent, 1000), + }, + canonicalEntity: resolved?.canonicalEntity ? safeEntity(resolved.canonicalEntity) : null, + matchedContexts: (resolved?.matchedContexts || []).map((item) => ({ + surface: cleanString(item.surface, 120), + entityId: cleanString(item.entityId, 240), + label: cleanString(item.label, 1000), + sourceSystem: cleanString(item.sourceSystem, 160), + status: cleanString(item.status, 120), + })), + bindings: (resolved?.bindings || []).map((item) => ({ + typeId: cleanString(item.typeId, 240), + status: cleanString(item.status, 120), + relationId: cleanString(item.relationId, 240), + summary: cleanString(item.summary, 1000), + })), + missingBindingTypes: (resolved?.missingBindingTypes || []).map((item) => ({ + id: cleanString(item.id, 240), + relationId: cleanString(item.relationId, 240), + summary: cleanString(item.summary, 1000), + })), + selectedRule: resolved?.selectedRule ? { + id: cleanString(resolved.selectedRule.id, 240), + fromSurface: cleanString(resolved.selectedRule.fromSurface, 120), + outputSurface: cleanString(resolved.selectedRule.outputSurface, 120), + inputEntityIds: stringList(resolved.selectedRule.inputEntityIds), + outputEntityIds: stringList(resolved.selectedRule.outputEntityIds), + requiredBindings: stringList(resolved.selectedRule.requiredBindings), + summary: cleanString(resolved.selectedRule.summary, 1000), + } : null, + candidates: (resolved?.candidates || []).map((item) => ({ + id: cleanString(item.id, 240), + score: Number.isFinite(Number(item.score)) ? Number(item.score) : 0, + outputSurface: cleanString(item.outputSurface, 120), + outputEntityIds: stringList(item.outputEntityIds), + requiredBindings: stringList(item.requiredBindings), + summary: cleanString(item.summary, 1000), + })), + } +} + +function safeEntity(value) { + return { + id: cleanString(value?.id, 240), + name: cleanString(value?.name, 240), + surface: cleanString(value?.surface, 120), + status: stringList(value?.status), + authority: cleanString(value?.authority, 240), + summary: cleanString(value?.summary, 2000), + } +} + +function safeRelation(value) { + return { + id: cleanString(value?.id, 240), + from: stringList(value?.from), + to: stringList(value?.to), + status: cleanString(value?.status, 120), + summary: cleanString(value?.summary, 2000), + } +} + +function safeAlias(value) { + return { + alias: cleanString(value?.alias, 240), + canonicalId: cleanString(value?.canonicalId, 240), + } +} + +function safeGuardrail(value) { + return { + id: cleanString(value?.id, 240), + severity: cleanString(value?.severity, 40), + summary: cleanString(value?.summary, 2000), + entityIds: stringList(value?.entityIds), + } +} + +function toolResult(value) { + return { + content: [{ type: 'text', text: JSON.stringify(value, null, 2) }], + structuredContent: value, + } +} + +function toolError(error, details = {}) { + const value = { ok: false, error, ...details } + return { + content: [{ type: 'text', text: JSON.stringify(value, null, 2) }], + structuredContent: value, + isError: true, + } +} + +function mcpResult(id, result) { + return { jsonrpc: '2.0', id, result } +} + +function mcpError(id, code, message) { + return { jsonrpc: '2.0', id, error: { code, message } } +} + +function sendMcpError(res, id, code, message) { + sendJson(res, 200, mcpError(id, code, message), { 'mcp-protocol-version': MCP_PROTOCOL_VERSION }) +} + +function sendJson(res, status, body, headers = {}) { + res.statusCode = status + res.setHeader('content-type', 'application/json; charset=utf-8') + res.setHeader('cache-control', 'no-store') + for (const [key, value] of Object.entries(headers)) res.setHeader(key, value) + res.end(JSON.stringify(body)) +} + +async function readRequestBody(req, maxBytes) { + const chunks = [] + let size = 0 + for await (const chunk of req) { + size += chunk.length + if (size > maxBytes) { + const error = new Error('body_too_large') + error.code = 'body_too_large' + throw error + } + chunks.push(chunk) + } + return Buffer.concat(chunks) +} + +function isAuthorized(header, candidates) { + const match = String(header || '').match(/^Bearer\s+(.+)$/i) + const token = match?.[1]?.trim() || '' + if (!token) return false + return (candidates || []).some((candidate) => safeEqual(token, candidate)) +} + +function safeEqual(left, right) { + const leftBuffer = Buffer.from(String(left || '')) + const rightBuffer = Buffer.from(String(right || '')) + return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer) +} + +function isAllowedOrigin(origin, allowedOrigins) { + if (!origin) return true + const normalized = normalizeOrigin(origin) + return Boolean(normalized && allowedOrigins.includes(normalized)) +} + +function normalizeOrigin(value) { + try { + const url = new URL(String(value || '')) + if (url.protocol !== 'http:' && url.protocol !== 'https:') return '' + return url.origin + } catch { + return '' + } +} + +function readConfig() { + const internalTokens = uniqueStrings([ + process.env.ONTOLOGY_CORE_TOKEN, + process.env.NODEDC_INTERNAL_ACCESS_TOKEN, + process.env.NODEDC_PLATFORM_SERVICE_TOKEN, + ]) + return { + port: boundedNumber(process.env.PORT || process.env.ONTOLOGY_CORE_PORT, 18104, 1, 65535), + internalTokens, + allowedOrigins: uniqueStrings(String(process.env.ONTOLOGY_MCP_ALLOWED_ORIGINS || '') + .split(/[\n,;]+/) + .map(normalizeOrigin) + .filter(Boolean)), + maxBodyBytes: boundedNumber(process.env.ONTOLOGY_MCP_MAX_BODY_BYTES, MAX_BODY_BYTES, 1024, 10 * 1024 * 1024), + } +} + +function boundedLimit(value) { + return boundedNumber(value, 20, 1, MAX_SEARCH_RESULTS) +} + +function boundedNumber(value, fallback, min, max) { + const numeric = Number(value) + if (!Number.isFinite(numeric)) return fallback + return Math.min(Math.max(Math.trunc(numeric), min), max) +} + +function normalizeSearch(value) { + return String(value || '').trim().toLocaleLowerCase('ru-RU') +} + +function searchScore(needle, ...values) { + let score = 0 + for (const value of values) { + const haystack = normalizeSearch(value) + if (!haystack) continue + if (haystack === needle) score = Math.max(score, 100) + else if (haystack.startsWith(needle)) score = Math.max(score, 60) + else if (haystack.includes(needle)) score = Math.max(score, 30) + } + return score +} + +function stableItemId(item) { + return item.entity?.id || item.relation?.id || item.alias?.alias || '' +} + +function stringList(value) { + if (!Array.isArray(value)) return [] + return value.map((item) => cleanString(item, 240)).filter(Boolean) +} + +function cleanString(value, maxLength) { + return String(value || '').trim().slice(0, maxLength) +} + +function uniqueStrings(values) { + return [...new Set((values || []).map((item) => cleanString(item, 4000)).filter(Boolean))] +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} + +const currentFile = fileURLToPath(import.meta.url) +const isEntrypoint = process.argv[1] && path.resolve(process.argv[1]) === currentFile + +if (isEntrypoint) { + const config = readConfig() + const server = createOntologyMcpServer(config) + server.listen(config.port, '0.0.0.0', () => { + console.log(`NODE.DC Ontology Core MCP listening on http://0.0.0.0:${config.port}`) + }) + const shutdown = () => server.close(() => process.exit(0)) + process.on('SIGTERM', shutdown) + process.on('SIGINT', shutdown) +} diff --git a/services/ontology-core/src/registry.mjs b/services/ontology-core/src/registry.mjs index d362354..f8aff71 100644 --- a/services/ontology-core/src/registry.mjs +++ b/services/ontology-core/src/registry.mjs @@ -191,7 +191,7 @@ function buildEngineContextFromManifest(manifest) { } if (workflow.nodeId) refs.node_id = String(workflow.nodeId) if (workflow.runtimeWorkflowId) refs.runtime_workflow_id = String(workflow.runtimeWorkflowId) - if (workflow.n8nInstanceId) refs.n8n_instance_id = String(workflow.n8nInstanceId) + if (workflow.runtimeInstanceId) refs.runtime_instance_id = String(workflow.runtimeInstanceId) return { id: `context.engine.${slugPart(workflow.id)}`, diff --git a/services/ontology-core/src/resolver.mjs b/services/ontology-core/src/resolver.mjs index a4983dd..86a00bc 100644 --- a/services/ontology-core/src/resolver.mjs +++ b/services/ontology-core/src/resolver.mjs @@ -185,7 +185,7 @@ async function runSmoke() { }, { surface: 'engine', entityId: 'engine.workflow_l1', intent: 'создай карточку в ops по текущему workflow' }, { surface: 'assistant', entityId: 'assistant.bridge', intent: 'select mcp gateway grant and tool' }, - { term: 'n8n workflow id', intent: 'runtime workflow id lookup' }, + { term: 'runtime workflow id', intent: 'runtime workflow id lookup' }, ] const results = [] diff --git a/services/ontology-core/src/scripts/smoke-mcp.mjs b/services/ontology-core/src/scripts/smoke-mcp.mjs new file mode 100644 index 0000000..34684d8 --- /dev/null +++ b/services/ontology-core/src/scripts/smoke-mcp.mjs @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict' +import { createOntologyMcpServer } from '../mcp-server.mjs' + +const TOKEN = 'ontology-mcp-smoke-token' +const server = createOntologyMcpServer({ + internalTokens: [TOKEN], + allowedOrigins: [], + maxBodyBytes: 1024 * 1024, +}) + +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) +const address = server.address() +const baseUrl = `http://127.0.0.1:${address.port}` + +try { + const health = await fetch(`${baseUrl}/healthz`) + const healthPayload = await health.json() + assert.equal(health.ok, true) + assert.equal(healthPayload.ok, true) + assert.equal(healthPayload.mcp.readOnly, true) + + const initialized = await rpc(baseUrl, TOKEN, 1, 'initialize', { protocolVersion: '2025-06-18' }) + assert.equal(initialized.result.protocolVersion, '2025-06-18') + assert.equal(initialized.result.capabilities.tools.listChanged, false) + + const tools = await rpc(baseUrl, TOKEN, 2, 'tools/list', {}) + const toolNames = tools.result.tools.map((tool) => tool.name).sort() + assert.deepEqual(toolNames, [ + 'ontology_get_entity', + 'ontology_get_guardrails', + 'ontology_resolve_context', + 'ontology_search', + 'ontology_status', + ]) + + const search = await rpc(baseUrl, TOKEN, 3, 'tools/call', { + name: 'ontology_search', + arguments: { query: 'трайк', limit: 10 }, + }) + assert.equal(search.result.isError, undefined) + assert.equal(search.result.structuredContent.query, 'трайк') + assert.equal(search.result.structuredContent.results.some((item) => item.entity?.id === 'gelios.unit'), true) + + const entity = await rpc(baseUrl, TOKEN, 4, 'tools/call', { + name: 'ontology_get_entity', + arguments: { term: 'helius' }, + }) + assert.equal(entity.result.structuredContent.entity.id, 'gelios.integration') + assert.equal(JSON.stringify(entity.result.structuredContent).includes('/Users/'), false) + + const guardrails = await rpc(baseUrl, TOKEN, 5, 'tools/call', { + name: 'ontology_get_guardrails', + arguments: { entityId: 'gelios.command_dispatch' }, + }) + assert.equal(guardrails.result.structuredContent.rules.some((rule) => rule.id === 'guardrail.gelios.commands_are_red_domain'), true) + + const unauthorized = await fetch(`${baseUrl}/mcp`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 6, method: 'tools/list', params: {} }), + }) + assert.equal(unauthorized.status, 401) + + console.log(JSON.stringify({ + ok: true, + checks: [ + 'health', + 'mcp_initialize', + 'read_only_tool_catalog', + 'gelios_alias_resolution', + 'gelios_command_guardrail_visible', + 'evidence_paths_not_exposed', + 'internal_bearer_required', + ], + }, null, 2)) +} finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())) +} + +async function rpc(baseUrl, token, id, method, params) { + const response = await fetch(`${baseUrl}/mcp`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + accept: 'application/json, text/event-stream', + 'mcp-protocol-version': '2025-06-18', + }, + body: JSON.stringify({ jsonrpc: '2.0', id, method, params }), + }) + assert.equal(response.ok, true) + return response.json() +}