feat(data-plane): add provider contracts and ontology delivery
This commit is contained in:
Codex
@@ -0,0 +1,32 @@
|
||||
FROM node:20-alpine AS deps
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
COPY packages/external-provider-contract ./packages/external-provider-contract
|
||||
COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/
|
||||
|
||||
WORKDIR /workspace/services/external-data-plane
|
||||
RUN npm ci --omit=dev
|
||||
|
||||
FROM node:20-alpine AS runner
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=18106
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=deps /workspace/services/external-data-plane/package.json /workspace/services/external-data-plane/package-lock.json ./
|
||||
COPY --from=deps /workspace/services/external-data-plane/node_modules ./node_modules
|
||||
COPY --from=deps /workspace/packages/external-provider-contract /packages/external-provider-contract
|
||||
COPY services/external-data-plane/src ./src
|
||||
COPY services/external-data-plane/definitions ./definitions
|
||||
|
||||
# A dedicated numeric identity can read only the EDP provisioning secret mount;
|
||||
# it is intentionally not the shared Node/Map Gateway uid/gid (1000).
|
||||
RUN addgroup -S -g 11006 nodedc-edp && adduser -S -D -H -u 11006 -G nodedc-edp nodedc-edp
|
||||
|
||||
USER 11006:11006
|
||||
|
||||
EXPOSE 18106
|
||||
|
||||
CMD ["node", "src/server.mjs"]
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "fleet.positions.current.v1",
|
||||
"version": "1.0.0",
|
||||
"ontologyRevision": "ontology.map.moving_object.v1",
|
||||
"deliveryMode": "snapshot+patch",
|
||||
"semanticTypes": [
|
||||
"map.moving_object"
|
||||
],
|
||||
"fields": [
|
||||
"course_degrees",
|
||||
"display_name",
|
||||
"elevation_meters",
|
||||
"geometry",
|
||||
"hdop",
|
||||
"horizontal_accuracy_meters",
|
||||
"object_kind",
|
||||
"operational_status",
|
||||
"position_source",
|
||||
"position_valid",
|
||||
"quality_flags",
|
||||
"satellite_count",
|
||||
"speed_kph"
|
||||
],
|
||||
"history": {
|
||||
"mode": "sampled",
|
||||
"intervalMs": 60000,
|
||||
"strategy": "latest-per-entity-per-bucket",
|
||||
"retentionDays": 90
|
||||
}
|
||||
}
|
||||
+1017
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "@nodedc/external-data-plane",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/server.mjs",
|
||||
"dev": "node --watch src/server.mjs",
|
||||
"check": "node --check src/server.mjs && node --check src/schema.mjs && node --check src/config.mjs && node --check src/intake-policy.mjs && node --check src/writer-binding.mjs && node --check src/reader-binding.mjs && node --check src/data-product-policy.mjs && node --check src/data-product-delivery.mjs && node --check src/definitions.mjs",
|
||||
"test": "node test/config.test.mjs && node test/intake-policy.test.mjs && node test/writer-binding.test.mjs && node test/reader-binding.test.mjs && node test/data-product-policy.test.mjs && node test/definitions.test.mjs",
|
||||
"test:integration": "node test/data-product-delivery.integration.test.mjs && node test/api.integration.test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@nodedc/external-provider-contract": "file:../../packages/external-provider-contract",
|
||||
"express": "^5.2.1",
|
||||
"pg": "^8.18.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
export function readConfig(env = process.env) {
|
||||
const provisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED, false);
|
||||
const config = {
|
||||
port: integer(env.PORT, 18106, 1, 65535),
|
||||
databaseUrl: required(env.EXTERNAL_DATA_PLANE_DATABASE_URL, "EXTERNAL_DATA_PLANE_DATABASE_URL"),
|
||||
databasePoolSize: integer(env.EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE, 10, 1, 50),
|
||||
internalAccessToken: optional(env.NODEDC_INTERNAL_ACCESS_TOKEN),
|
||||
provisionerApiEnabled,
|
||||
provisionerAccessToken: provisionerApiEnabled ? secretFile(env.EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE) : "",
|
||||
rawRetentionDays: integer(env.EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS, 14, 1, 3650),
|
||||
maxBatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES, 5 * 1024 * 1024, 1024, 50 * 1024 * 1024),
|
||||
maxFactsPerPublish: integer(env.EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH, 5000, 1, 100_000),
|
||||
maxAttributesBytesPerFact: integer(env.EXTERNAL_DATA_PLANE_MAX_ATTRIBUTES_BYTES_PER_FACT, 64 * 1024, 256, 1024 * 1024),
|
||||
maxPatchOperations: integer(env.EXTERNAL_DATA_PLANE_MAX_PATCH_OPERATIONS, 500, 1, 5000),
|
||||
maxPatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_PATCH_BYTES, 256 * 1024, 16 * 1024, 5 * 1024 * 1024),
|
||||
patchRetentionMs: integer(env.EXTERNAL_DATA_PLANE_PATCH_RETENTION_MS, 60 * 60 * 1000, 60 * 1000, 7 * 24 * 60 * 60 * 1000),
|
||||
receiptRetentionMs: integer(env.EXTERNAL_DATA_PLANE_RECEIPT_RETENTION_MS, 7 * 24 * 60 * 60 * 1000, 24 * 60 * 60 * 1000, 90 * 24 * 60 * 60 * 1000),
|
||||
retentionDeleteLimit: integer(env.EXTERNAL_DATA_PLANE_RETENTION_DELETE_LIMIT, 10_000, 100, 100_000),
|
||||
streamHeartbeatMs: integer(env.EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS, 20_000, 5_000, 60_000),
|
||||
streamPollMs: integer(env.EXTERNAL_DATA_PLANE_STREAM_POLL_MS, 1_000, 250, 10_000),
|
||||
maxReaderStreams: integer(env.EXTERNAL_DATA_PLANE_MAX_READER_STREAMS, 10, 1, 1000),
|
||||
writerBindingMaxTtlDays: integer(env.EXTERNAL_DATA_PLANE_WRITER_BINDING_MAX_TTL_DAYS, 90, 1, 365),
|
||||
maxFutureSkewSeconds: integer(env.EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS, 300, 0, 86400),
|
||||
retentionSweepMs: integer(env.EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS, 60 * 60 * 1000, 60 * 1000, 24 * 60 * 60 * 1000),
|
||||
legacyIntakeEnabled: boolean(env.EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED, false),
|
||||
};
|
||||
if (config.internalAccessToken && config.provisionerAccessToken && config.internalAccessToken === config.provisionerAccessToken) {
|
||||
throw new Error("provisioner_token_must_differ_from_internal_token");
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
function required(value, name) {
|
||||
const normalized = optional(value);
|
||||
if (!normalized) throw new Error(`${name}_required`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function optional(value) {
|
||||
const normalized = String(value ?? "").trim();
|
||||
return normalized || "";
|
||||
}
|
||||
|
||||
function integer(value, fallback, min, max) {
|
||||
const candidate = optional(value);
|
||||
if (!candidate) return fallback;
|
||||
const number = Number.parseInt(candidate, 10);
|
||||
if (!Number.isInteger(number) || number < min || number > max) throw new Error(`invalid_integer:${candidate}`);
|
||||
return number;
|
||||
}
|
||||
|
||||
function boolean(value, fallback) {
|
||||
const normalized = optional(value).toLowerCase();
|
||||
if (!normalized) return fallback;
|
||||
if (normalized === "true") return true;
|
||||
if (normalized === "false") return false;
|
||||
throw new Error(`invalid_boolean:${normalized}`);
|
||||
}
|
||||
|
||||
function secretFile(value) {
|
||||
const path = optional(value);
|
||||
if (!path) return "";
|
||||
let secret = "";
|
||||
try {
|
||||
secret = readFileSync(path, "utf8").trim();
|
||||
} catch {
|
||||
throw new Error("external_data_plane_provisioner_token_file_unreadable");
|
||||
}
|
||||
if (!/^[A-Za-z0-9_-]{48,256}$/.test(secret)) {
|
||||
throw new Error("external_data_plane_provisioner_token_file_invalid");
|
||||
}
|
||||
return secret;
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import {
|
||||
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||
} from "@nodedc/external-provider-contract";
|
||||
|
||||
export async function loadDataProductDefinition(db, dataProductId, { activeOnly = true } = {}) {
|
||||
const result = await db.query(
|
||||
`select id, version, ontology_revision as "ontologyRevision",
|
||||
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
|
||||
fields, history_policy as "historyPolicy", active,
|
||||
created_at as "createdAt", updated_at as "updatedAt"
|
||||
from external_data_plane_products
|
||||
where id = $1 ${activeOnly ? "and active = true" : ""}`,
|
||||
[dataProductId],
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
}
|
||||
|
||||
export async function persistDataProductDefinition(db, definition) {
|
||||
const result = await db.query(
|
||||
`insert into external_data_plane_products (
|
||||
id, version, ontology_revision, delivery_mode, semantic_types, fields, history_policy
|
||||
) values ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7::jsonb)
|
||||
on conflict (id) do update set
|
||||
active = true,
|
||||
updated_at = now()
|
||||
where external_data_plane_products.version = excluded.version
|
||||
and external_data_plane_products.ontology_revision = excluded.ontology_revision
|
||||
and external_data_plane_products.delivery_mode = excluded.delivery_mode
|
||||
and external_data_plane_products.semantic_types = excluded.semantic_types
|
||||
and external_data_plane_products.fields = excluded.fields
|
||||
and external_data_plane_products.history_policy = excluded.history_policy
|
||||
returning id, version, ontology_revision as "ontologyRevision",
|
||||
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
|
||||
fields, history_policy as "historyPolicy", active,
|
||||
created_at as "createdAt", updated_at as "updatedAt"`,
|
||||
[
|
||||
definition.id,
|
||||
definition.version,
|
||||
definition.ontologyRevision,
|
||||
definition.deliveryMode,
|
||||
JSON.stringify(definition.semanticTypes),
|
||||
JSON.stringify(definition.fields),
|
||||
JSON.stringify(definition.history),
|
||||
],
|
||||
);
|
||||
if (!result.rowCount) throw deliveryError("data_product_version_is_immutable", 409);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
export async function persistDataProductPublish(pool, batch, definition, {
|
||||
maxPatchOperations = 500,
|
||||
maxPatchBytes = 256 * 1024,
|
||||
} = {}) {
|
||||
assertPublishMatchesDefinition(batch, definition);
|
||||
const requestFingerprint = publishFingerprint(batch);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
const batchId = randomUUID();
|
||||
const insertedBatch = await client.query(
|
||||
`insert into external_data_plane_batches (
|
||||
id, tenant_id, connection_id, provider_id, data_product_id, contract_version,
|
||||
ontology_revision, run_id, sequence, idempotency_key, received_at, fact_count,
|
||||
request_fingerprint
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, idempotency_key)
|
||||
do nothing
|
||||
returning id`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.contract.version, batch.contract.ontologyRevision,
|
||||
batch.batch.runId, batch.batch.sequence, batch.batch.idempotencyKey,
|
||||
batch.batch.receivedAt, batch.facts.length, requestFingerprint,
|
||||
],
|
||||
);
|
||||
if (!insertedBatch.rowCount) {
|
||||
const existing = await client.query(
|
||||
`select id as "batchId", fact_count as "publishedFactCount",
|
||||
current_updated_count as "currentUpdatedCount",
|
||||
history_inserted_count as "historyInsertedCount",
|
||||
patch_operation_count as "patchOperationCount",
|
||||
delivery_cursor::text as cursor, received_at as "acceptedAt",
|
||||
request_fingerprint as "requestFingerprint"
|
||||
from external_data_plane_batches
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3
|
||||
and data_product_id = $4 and idempotency_key = $5`,
|
||||
scopeValues(batch, batch.batch.idempotencyKey),
|
||||
);
|
||||
if (!existing.rowCount || existing.rows[0].requestFingerprint !== requestFingerprint) {
|
||||
throw deliveryError("idempotency_key_reused", 409);
|
||||
}
|
||||
await client.query("commit");
|
||||
return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
|
||||
}
|
||||
|
||||
const records = batch.facts.map((fact) => ({
|
||||
source_id: fact.sourceId,
|
||||
semantic_type: fact.semanticType,
|
||||
observed_at: fact.observedAt,
|
||||
received_at: batch.batch.receivedAt,
|
||||
attributes: fact.attributes || {},
|
||||
geometry: fact.geometry || null,
|
||||
fingerprint: fingerprint(fact),
|
||||
}));
|
||||
|
||||
const updated = await client.query(
|
||||
`insert into external_data_plane_current (
|
||||
tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, updated_at
|
||||
)
|
||||
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
||||
item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
||||
case when item.geometry is null then null
|
||||
else ST_SetSRID(ST_MakePoint(
|
||||
(item.geometry->'coordinates'->>0)::double precision,
|
||||
(item.geometry->'coordinates'->>1)::double precision
|
||||
), 4326)::geography end,
|
||||
item.fingerprint, now()
|
||||
from jsonb_to_recordset($5::jsonb) as item(
|
||||
source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
|
||||
attributes jsonb, geometry jsonb, fingerprint text
|
||||
)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||
do update set
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
attributes = excluded.attributes,
|
||||
geometry = excluded.geometry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at > external_data_plane_current.observed_at
|
||||
or (excluded.observed_at = external_data_plane_current.observed_at
|
||||
and excluded.fingerprint <> external_data_plane_current.fingerprint)
|
||||
returning source_id as "sourceId", semantic_type as "semanticType",
|
||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||
case when geometry is null then null
|
||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
||||
)) end as geometry,
|
||||
fingerprint`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, JSON.stringify(records),
|
||||
],
|
||||
);
|
||||
|
||||
const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
|
||||
const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) }));
|
||||
const chunks = definition.deliveryMode === "snapshot+patch"
|
||||
? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
|
||||
: [];
|
||||
const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
|
||||
|
||||
await client.query(
|
||||
`update external_data_plane_batches set
|
||||
current_updated_count = $2,
|
||||
history_inserted_count = $3,
|
||||
patch_operation_count = $4,
|
||||
delivery_cursor = $5
|
||||
where id = $1`,
|
||||
[batchId, updated.rowCount, historyInsertedCount, operations.length, cursor],
|
||||
);
|
||||
await client.query("commit");
|
||||
return normalizeReceipt({
|
||||
batchId,
|
||||
idempotent: false,
|
||||
publishedFactCount: batch.facts.length,
|
||||
currentUpdatedCount: updated.rowCount,
|
||||
historyInsertedCount,
|
||||
patchOperationCount: operations.length,
|
||||
cursor: String(cursor),
|
||||
acceptedAt: batch.batch.receivedAt,
|
||||
});
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function readDataProductSnapshot(pool, binding, definition, { limit = 5000 } = {}) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin isolation level repeatable read read only");
|
||||
const cursor = await currentCursor(client, binding, definition.id);
|
||||
const rows = await client.query(
|
||||
`select source_id as "sourceId", semantic_type as "semanticType",
|
||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||
case when geometry is null then null
|
||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
||||
)) end as geometry
|
||||
from external_data_plane_current
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
|
||||
order by source_id asc, semantic_type asc
|
||||
limit $5`,
|
||||
[binding.tenantId, binding.connectionId, binding.providerId, definition.id, limit + 1],
|
||||
);
|
||||
if (rows.rowCount > limit) throw deliveryError("data_product_snapshot_limit_exceeded", 413);
|
||||
await client.query("commit");
|
||||
return {
|
||||
schemaVersion: DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||
dataProduct: { id: definition.id, version: definition.version },
|
||||
generatedAt: new Date().toISOString(),
|
||||
cursor: String(cursor),
|
||||
facts: rows.rows.map(publicFact),
|
||||
};
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function readPatchEvents(pool, binding, definition, after, { limit = 100 } = {}) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin isolation level repeatable read read only");
|
||||
const current = await currentCursor(client, binding, definition.id);
|
||||
if (after > current) throw deliveryError("resync_required", 409);
|
||||
const floor = await patchFloor(client, binding, definition.id);
|
||||
if (after < floor - 1n) throw deliveryError("resync_required", 409);
|
||||
const rows = await client.query(
|
||||
`select cursor::text, previous_cursor::text as "previousCursor",
|
||||
operations, emitted_at as "emittedAt"
|
||||
from external_data_plane_patch_outbox
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3
|
||||
and data_product_id = $4 and cursor > $5
|
||||
order by cursor asc limit $6`,
|
||||
[binding.tenantId, binding.connectionId, binding.providerId, definition.id, after.toString(), limit],
|
||||
);
|
||||
await client.query("commit");
|
||||
return rows.rows.map((row) => ({
|
||||
schemaVersion: DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||
dataProduct: { id: definition.id, version: definition.version },
|
||||
previousCursor: row.previousCursor,
|
||||
cursor: row.cursor,
|
||||
emittedAt: new Date(row.emittedAt).toISOString(),
|
||||
operations: row.operations,
|
||||
}));
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function prunePatchOutbox(db, { retentionMs, limit = 10_000 }) {
|
||||
const cutoff = new Date(Date.now() - retentionMs);
|
||||
const result = await db.query(
|
||||
`with expired as (
|
||||
select tenant_id, connection_id, provider_id, data_product_id, cursor
|
||||
from external_data_plane_patch_outbox
|
||||
where emitted_at < $1
|
||||
order by emitted_at asc
|
||||
limit $2
|
||||
)
|
||||
delete from external_data_plane_patch_outbox as target
|
||||
using expired
|
||||
where target.tenant_id = expired.tenant_id
|
||||
and target.connection_id = expired.connection_id
|
||||
and target.provider_id = expired.provider_id
|
||||
and target.data_product_id = expired.data_product_id
|
||||
and target.cursor = expired.cursor`,
|
||||
[cutoff, limit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
export async function pruneDataProductHistory(db, { limit = 10_000 } = {}) {
|
||||
const result = await db.query(
|
||||
`with expired as (
|
||||
select history.tenant_id, history.connection_id, history.provider_id,
|
||||
history.data_product_id, history.source_id, history.semantic_type, history.bucket_start
|
||||
from external_data_plane_history as history
|
||||
join external_data_plane_products as product on product.id = history.data_product_id
|
||||
where history.bucket_start < now() - (
|
||||
greatest(1, coalesce((product.history_policy->>'retentionDays')::integer, 1)) * interval '1 day'
|
||||
)
|
||||
order by history.bucket_start asc
|
||||
limit $1
|
||||
)
|
||||
delete from external_data_plane_history as target
|
||||
using expired
|
||||
where target.tenant_id = expired.tenant_id
|
||||
and target.connection_id = expired.connection_id
|
||||
and target.provider_id = expired.provider_id
|
||||
and target.data_product_id = expired.data_product_id
|
||||
and target.source_id = expired.source_id
|
||||
and target.semantic_type = expired.semantic_type
|
||||
and target.bucket_start = expired.bucket_start`,
|
||||
[limit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) {
|
||||
const cutoff = new Date(Date.now() - retentionMs);
|
||||
const result = await db.query(
|
||||
`with expired as (
|
||||
select batch.id
|
||||
from external_data_plane_batches as batch
|
||||
where batch.created_at < $1
|
||||
and not exists (select 1 from external_data_plane_patch_outbox as patch where patch.batch_id = batch.id)
|
||||
and not exists (select 1 from external_data_plane_raw_envelopes as raw where raw.batch_id = batch.id)
|
||||
order by batch.created_at asc
|
||||
limit $2
|
||||
)
|
||||
delete from external_data_plane_batches as target
|
||||
using expired
|
||||
where target.id = expired.id`,
|
||||
[cutoff, limit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
async function persistHistory(client, batch, policy, facts) {
|
||||
if (!facts.length || policy?.mode === "none") return 0;
|
||||
const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
|
||||
const records = facts.map((fact) => ({
|
||||
...fact,
|
||||
bucket_start: new Date(Math.floor(new Date(fact.observed_at).getTime() / intervalMs) * intervalMs).toISOString(),
|
||||
}));
|
||||
const result = await client.query(
|
||||
`insert into external_data_plane_history (
|
||||
tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
bucket_start, observed_at, received_at, attributes, geometry, fingerprint, updated_at
|
||||
)
|
||||
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
||||
item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
||||
case when item.geometry is null then null
|
||||
else ST_SetSRID(ST_MakePoint(
|
||||
(item.geometry->'coordinates'->>0)::double precision,
|
||||
(item.geometry->'coordinates'->>1)::double precision
|
||||
), 4326)::geography end,
|
||||
item.fingerprint, now()
|
||||
from jsonb_to_recordset($5::jsonb) as item(
|
||||
source_id text, semantic_type text, bucket_start timestamptz,
|
||||
observed_at timestamptz, received_at timestamptz,
|
||||
attributes jsonb, geometry jsonb, fingerprint text
|
||||
)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, bucket_start)
|
||||
do update set
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
attributes = excluded.attributes,
|
||||
geometry = excluded.geometry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at >= external_data_plane_history.observed_at`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId,
|
||||
JSON.stringify(records),
|
||||
],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
async function persistPatchChunks(client, batch, definition, batchId, chunks) {
|
||||
const endCursorResult = await client.query(
|
||||
`insert into external_data_plane_delivery_state (
|
||||
tenant_id, connection_id, provider_id, data_product_id, current_cursor
|
||||
) values ($1, $2, $3, $4, $5)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id)
|
||||
do update set
|
||||
current_cursor = external_data_plane_delivery_state.current_cursor + excluded.current_cursor,
|
||||
updated_at = now()
|
||||
returning current_cursor`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
definition.id, chunks.length,
|
||||
],
|
||||
);
|
||||
const end = BigInt(endCursorResult.rows[0].current_cursor);
|
||||
const start = end - BigInt(chunks.length) + 1n;
|
||||
for (let index = 0; index < chunks.length; index += 1) {
|
||||
const cursor = start + BigInt(index);
|
||||
await client.query(
|
||||
`insert into external_data_plane_patch_outbox (
|
||||
tenant_id, connection_id, provider_id, data_product_id,
|
||||
cursor, previous_cursor, batch_id, operations
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
definition.id, cursor.toString(), (cursor - 1n).toString(), batchId,
|
||||
JSON.stringify(chunks[index]),
|
||||
],
|
||||
);
|
||||
}
|
||||
return end;
|
||||
}
|
||||
|
||||
async function currentCursor(db, scope, explicitProductId) {
|
||||
const dataProductId = explicitProductId || scope.contract.dataProductId;
|
||||
const result = await db.query(
|
||||
`select current_cursor from external_data_plane_delivery_state
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
|
||||
[scope.tenantId || scope.source.tenantId, scope.connectionId || scope.source.connectionId,
|
||||
scope.providerId || scope.source.providerId, dataProductId],
|
||||
);
|
||||
return result.rowCount ? BigInt(result.rows[0].current_cursor) : 0n;
|
||||
}
|
||||
|
||||
async function patchFloor(db, binding, dataProductId) {
|
||||
const result = await db.query(
|
||||
`select min(cursor) as floor from external_data_plane_patch_outbox
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
|
||||
[binding.tenantId, binding.connectionId, binding.providerId, dataProductId],
|
||||
);
|
||||
if (result.rows[0]?.floor !== null && result.rows[0]?.floor !== undefined) return BigInt(result.rows[0].floor);
|
||||
const current = await currentCursor(db, binding, dataProductId);
|
||||
return current + 1n;
|
||||
}
|
||||
|
||||
function scopeValues(batch, tail) {
|
||||
return [
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, tail,
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeReceipt(value) {
|
||||
return {
|
||||
batchId: value.batchId,
|
||||
idempotent: value.idempotent === true,
|
||||
publishedFactCount: Number(value.publishedFactCount || 0),
|
||||
currentUpdatedCount: Number(value.currentUpdatedCount || 0),
|
||||
historyInsertedCount: Number(value.historyInsertedCount || 0),
|
||||
patchOperationCount: Number(value.patchOperationCount || 0),
|
||||
cursor: String(value.cursor || "0"),
|
||||
acceptedAt: new Date(value.acceptedAt).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function publicFact(fact) {
|
||||
const value = {
|
||||
sourceId: fact.sourceId,
|
||||
semanticType: fact.semanticType,
|
||||
observedAt: new Date(fact.observedAt).toISOString(),
|
||||
receivedAt: new Date(fact.receivedAt).toISOString(),
|
||||
attributes: fact.attributes || {},
|
||||
};
|
||||
if (fact.geometry) value.geometry = fact.geometry;
|
||||
return value;
|
||||
}
|
||||
|
||||
function chunkOperations(values, maxOperations, maxBytes) {
|
||||
const result = [];
|
||||
let current = [];
|
||||
let currentBytes = 2;
|
||||
for (const value of values) {
|
||||
const serializedBytes = Buffer.byteLength(JSON.stringify(value));
|
||||
if (serializedBytes + 2 > maxBytes) throw deliveryError("data_product_patch_operation_size_exceeded", 413);
|
||||
const separatorBytes = current.length ? 1 : 0;
|
||||
if (current.length && (current.length >= maxOperations || currentBytes + separatorBytes + serializedBytes > maxBytes)) {
|
||||
result.push(current);
|
||||
current = [];
|
||||
currentBytes = 2;
|
||||
}
|
||||
current.push(value);
|
||||
currentBytes += (current.length > 1 ? 1 : 0) + serializedBytes;
|
||||
}
|
||||
if (current.length) result.push(current);
|
||||
return result;
|
||||
}
|
||||
|
||||
function fingerprint(value) {
|
||||
return createHash("sha256").update(stableJson(value)).digest("hex");
|
||||
}
|
||||
|
||||
function publishFingerprint(batch) {
|
||||
return fingerprint({
|
||||
schemaVersion: batch.schemaVersion,
|
||||
source: batch.source,
|
||||
contract: batch.contract,
|
||||
batch: {
|
||||
runId: batch.batch.runId,
|
||||
sequence: batch.batch.sequence,
|
||||
idempotencyKey: batch.batch.idempotencyKey,
|
||||
},
|
||||
facts: batch.facts,
|
||||
});
|
||||
}
|
||||
|
||||
export function assertPublishMatchesDefinition(batch, definition) {
|
||||
const semanticTypes = new Set(Array.isArray(definition?.semanticTypes) ? definition.semanticTypes : []);
|
||||
const fields = new Set(Array.isArray(definition?.fields) ? definition.fields : []);
|
||||
const entityKeys = new Set();
|
||||
for (const fact of batch?.facts || []) {
|
||||
if (!semanticTypes.has(fact.semanticType)) throw deliveryError("data_product_semantic_type_forbidden", 422);
|
||||
const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`;
|
||||
if (entityKeys.has(entityKey)) throw deliveryError("data_product_duplicate_entity_key", 422);
|
||||
entityKeys.add(entityKey);
|
||||
for (const attribute of Object.keys(fact.attributes || {})) {
|
||||
if (!fields.has(attribute) && !fields.has(`attributes.${attribute}`)) {
|
||||
throw deliveryError("data_product_field_forbidden", 422);
|
||||
}
|
||||
}
|
||||
if (fact.geometry !== undefined && !geometryDeclared(fields)) {
|
||||
throw deliveryError("data_product_geometry_forbidden", 422);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function geometryDeclared(fields) {
|
||||
return fields.has("geometry")
|
||||
|| fields.has("coordinates")
|
||||
|| (fields.has("longitude") && fields.has("latitude"));
|
||||
}
|
||||
|
||||
function stableJson(value) {
|
||||
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function deliveryError(code, status) {
|
||||
return Object.assign(new Error(code), { status, code });
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
||||
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
|
||||
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
|
||||
const DEFINITION_KEYS = new Set([
|
||||
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "history",
|
||||
]);
|
||||
const HISTORY_KEYS = new Set(["mode", "intervalMs", "retentionDays", "strategy"]);
|
||||
|
||||
export function normalizeDataProductDefinition(value) {
|
||||
if (!isPlainObject(value) || !hasOnlyKeys(value, DEFINITION_KEYS)) {
|
||||
throw policyError("data_product_definition_invalid");
|
||||
}
|
||||
const id = identifier(value.id);
|
||||
const version = string(value.version);
|
||||
const ontologyRevision = identifier(value.ontologyRevision);
|
||||
const deliveryMode = string(value.deliveryMode);
|
||||
const semanticTypes = identifierSet(
|
||||
value.semanticTypes,
|
||||
"data_product_definition_semantic_types_invalid",
|
||||
"data_product_definition_semantic_types_duplicate",
|
||||
);
|
||||
const fields = identifierSet(
|
||||
value.fields,
|
||||
"data_product_definition_fields_invalid",
|
||||
"data_product_definition_fields_duplicate",
|
||||
);
|
||||
if (!id || !SEMVER.test(version) || !ontologyRevision || !DELIVERY_MODES.has(deliveryMode)) {
|
||||
throw policyError("data_product_definition_identity_invalid");
|
||||
}
|
||||
if (!semanticTypes.length || !fields.length) throw policyError("data_product_definition_shape_invalid");
|
||||
|
||||
const history = normalizeHistoryPolicy(value.history);
|
||||
return Object.freeze({ id, version, ontologyRevision, deliveryMode, semanticTypes, fields, history });
|
||||
}
|
||||
|
||||
export function normalizeHistoryPolicy(value = { mode: "none" }) {
|
||||
if (!isPlainObject(value) || !hasOnlyKeys(value, HISTORY_KEYS)) throw policyError("history_policy_invalid");
|
||||
const mode = string(value.mode);
|
||||
if (!HISTORY_MODES.has(mode)) throw policyError("history_policy_mode_invalid");
|
||||
const retentionDays = integer(value.retentionDays, mode === "none" ? 1 : 90, 1, 3650);
|
||||
if (mode === "none") {
|
||||
if (value.intervalMs !== undefined || value.strategy !== undefined) throw policyError("history_policy_none_has_sampling_fields");
|
||||
return Object.freeze({ mode, retentionDays });
|
||||
}
|
||||
if (mode === "all") {
|
||||
if (value.intervalMs !== undefined || value.strategy !== undefined) throw policyError("history_policy_all_has_sampling_fields");
|
||||
return Object.freeze({ mode, retentionDays });
|
||||
}
|
||||
|
||||
const intervalMs = integer(value.intervalMs, 60_000, 1_000, 24 * 60 * 60 * 1000);
|
||||
const strategy = string(value.strategy || "latest-per-entity-per-bucket");
|
||||
if (strategy !== "latest-per-entity-per-bucket") throw policyError("history_policy_strategy_invalid");
|
||||
return Object.freeze({ mode, intervalMs, strategy, retentionDays });
|
||||
}
|
||||
|
||||
export function safeDataProductDefinition(row) {
|
||||
return {
|
||||
id: row.id,
|
||||
version: row.version,
|
||||
ontologyRevision: row.ontologyRevision,
|
||||
deliveryMode: row.deliveryMode,
|
||||
semanticTypes: array(row.semanticTypes),
|
||||
fields: array(row.fields),
|
||||
history: isPlainObject(row.historyPolicy) ? row.historyPolicy : {},
|
||||
active: row.active === true,
|
||||
createdAt: iso(row.createdAt),
|
||||
updatedAt: iso(row.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function policyError(code) {
|
||||
return Object.assign(new Error(code), { status: 400, code });
|
||||
}
|
||||
|
||||
function identifier(value) {
|
||||
const normalized = string(value);
|
||||
return IDENTIFIER.test(normalized) ? normalized : "";
|
||||
}
|
||||
|
||||
function identifierSet(value, invalidCode, duplicateCode) {
|
||||
if (!Array.isArray(value)) throw policyError(invalidCode);
|
||||
|
||||
const normalized = value.map((entry) => {
|
||||
const result = identifier(entry);
|
||||
if (!result) throw policyError(invalidCode);
|
||||
return result;
|
||||
});
|
||||
if (new Set(normalized).size !== normalized.length) throw policyError(duplicateCode);
|
||||
|
||||
return Object.freeze(normalized.sort());
|
||||
}
|
||||
|
||||
function integer(value, fallback, min, max) {
|
||||
const number = value === undefined ? fallback : Number(value);
|
||||
if (!Number.isInteger(number) || number < min || number > max) throw policyError("history_policy_number_invalid");
|
||||
return number;
|
||||
}
|
||||
|
||||
function string(value) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
function array(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function iso(value) {
|
||||
return value ? new Date(value).toISOString() : undefined;
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function hasOnlyKeys(value, allowed) {
|
||||
return Object.keys(value).every((key) => allowed.has(key));
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { lstat, readFile, readdir } from "node:fs/promises";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { normalizeDataProductDefinition } from "./data-product-policy.mjs";
|
||||
import { persistDataProductDefinition } from "./data-product-delivery.mjs";
|
||||
|
||||
const bundledDefinitionsDir = fileURLToPath(new URL("../definitions/", import.meta.url));
|
||||
|
||||
export async function loadDataProductDefinitions(directory = bundledDefinitionsDir) {
|
||||
const root = resolve(directory);
|
||||
const entries = (await readdir(root, { withFileTypes: true }))
|
||||
.filter((entry) => entry.name.endsWith(".json"))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
const definitions = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("data_product_definition_file_invalid");
|
||||
const path = join(root, entry.name);
|
||||
const metadata = await lstat(path);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size < 2 || metadata.size > 256 * 1024) {
|
||||
throw new Error("data_product_definition_file_invalid");
|
||||
}
|
||||
let value;
|
||||
try {
|
||||
value = JSON.parse(await readFile(path, "utf8"));
|
||||
} catch {
|
||||
throw new Error("data_product_definition_json_invalid");
|
||||
}
|
||||
const definition = normalizeDataProductDefinition(value);
|
||||
if (`${definition.id}.json` !== basename(path)) throw new Error("data_product_definition_filename_mismatch");
|
||||
definitions.push(definition);
|
||||
}
|
||||
if (new Set(definitions.map((definition) => definition.id)).size !== definitions.length) {
|
||||
throw new Error("data_product_definition_duplicate");
|
||||
}
|
||||
return Object.freeze(definitions);
|
||||
}
|
||||
|
||||
export async function reconcileDataProductDefinitions(db, directory = bundledDefinitionsDir) {
|
||||
const definitions = await loadDataProductDefinitions(directory);
|
||||
for (const definition of definitions) await persistDataProductDefinition(db, definition);
|
||||
return definitions;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
export function assertBatchTimeBounds(batch, { now = new Date(), maxFutureSkewSeconds = 300 } = {}) {
|
||||
const acceptedAt = validDate(now, "intake_policy_clock_invalid");
|
||||
const maxFutureAt = new Date(acceptedAt.getTime() + Number(maxFutureSkewSeconds) * 1000);
|
||||
if (!Number.isInteger(maxFutureSkewSeconds) || maxFutureSkewSeconds < 0) {
|
||||
throw intakePolicyError("intake_policy_future_skew_invalid");
|
||||
}
|
||||
|
||||
const receivedAt = validDate(batch?.batch?.receivedAt, "batch_received_at_invalid");
|
||||
if (receivedAt > maxFutureAt) throw intakePolicyError("batch_received_at_too_far_in_future");
|
||||
|
||||
for (const fact of batch?.facts || []) {
|
||||
const observedAt = validDate(fact?.observedAt, "fact_observed_at_invalid");
|
||||
if (observedAt > maxFutureAt) throw intakePolicyError("fact_observed_at_too_far_in_future");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retention is based on the server's acceptance time, never on provider or L2
|
||||
* timestamps supplied in a batch.
|
||||
*/
|
||||
export function rawRetentionExpiry({ now = new Date(), rawRetentionDays }) {
|
||||
const acceptedAt = validDate(now, "intake_policy_clock_invalid");
|
||||
if (!Number.isInteger(rawRetentionDays) || rawRetentionDays < 1) {
|
||||
throw intakePolicyError("raw_retention_days_invalid");
|
||||
}
|
||||
return new Date(acceptedAt.getTime() + rawRetentionDays * 24 * 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
export function intakePolicyError(code) {
|
||||
return Object.assign(new Error(code), { status: 422, code });
|
||||
}
|
||||
|
||||
function validDate(value, code) {
|
||||
const parsed = value instanceof Date ? new Date(value.getTime()) : new Date(String(value ?? ""));
|
||||
if (Number.isNaN(parsed.getTime())) throw intakePolicyError(code);
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
|
||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||
const TOKEN_PREFIX = "ndc_edprb_";
|
||||
const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i;
|
||||
const REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]);
|
||||
const SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]);
|
||||
|
||||
export function createReaderToken() {
|
||||
return `${TOKEN_PREFIX}${randomBytes(32).toString("base64url")}`;
|
||||
}
|
||||
|
||||
export function hashReaderToken(token) {
|
||||
return createHash("sha256").update(String(token), "utf8").digest("hex");
|
||||
}
|
||||
|
||||
export function normalizeReaderBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(value.source)) throw readerError("reader_binding_request_invalid");
|
||||
if (containsSecretLikeKey(value)) throw readerError("reader_binding_request_secret_material_forbidden");
|
||||
if (!hasOnlyKeys(value, REQUEST_KEYS) || !hasOnlyKeys(value.source, SOURCE_KEYS)) {
|
||||
throw readerError("reader_binding_request_fields_invalid");
|
||||
}
|
||||
const tenantId = identifier(value.source.tenantId);
|
||||
const connectionId = identifier(value.source.connectionId);
|
||||
const providerId = identifier(value.source.providerId);
|
||||
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
|
||||
if (!tenantId || !connectionId || !providerId || !allowedDataProductIds.length) {
|
||||
throw readerError("reader_binding_scope_invalid");
|
||||
}
|
||||
const expiresAt = new Date(String(value.expiresAt || ""));
|
||||
const maxExpiresAt = new Date(now.getTime() + maxTtlDays * 24 * 60 * 60 * 1000);
|
||||
if (Number.isNaN(expiresAt.getTime()) || expiresAt <= now || expiresAt > maxExpiresAt) {
|
||||
throw readerError("reader_binding_expiry_invalid");
|
||||
}
|
||||
return Object.freeze({ tenantId, connectionId, providerId, allowedDataProductIds, expiresAt: expiresAt.toISOString() });
|
||||
}
|
||||
|
||||
export function assertReaderProduct(binding, dataProductId, now = new Date()) {
|
||||
if (!isPlainObject(binding) || binding.active !== true || new Date(binding.expiresAt) <= now) {
|
||||
throw readerError("reader_binding_inactive", 401);
|
||||
}
|
||||
const normalized = identifier(dataProductId);
|
||||
if (!normalized || !uniqueIdentifiers(binding.allowedDataProductIds).includes(normalized)) {
|
||||
throw readerError("reader_binding_data_product_forbidden", 403);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function safeReaderBinding(binding) {
|
||||
return {
|
||||
id: binding.id,
|
||||
tenantId: binding.tenantId,
|
||||
connectionId: binding.connectionId,
|
||||
providerId: binding.providerId,
|
||||
allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds),
|
||||
active: binding.active === true,
|
||||
expiresAt: new Date(binding.expiresAt).toISOString(),
|
||||
createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined,
|
||||
rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined,
|
||||
revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function readerError(code, status = 400) {
|
||||
return Object.assign(new Error(code), { status, code });
|
||||
}
|
||||
|
||||
function identifier(value) {
|
||||
const normalized = typeof value === "string" ? value.trim() : "";
|
||||
return IDENTIFIER.test(normalized) ? normalized : "";
|
||||
}
|
||||
|
||||
function uniqueIdentifiers(value) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return [...new Set(value.map(identifier).filter(Boolean))];
|
||||
}
|
||||
|
||||
function containsSecretLikeKey(value) {
|
||||
if (Array.isArray(value)) return value.some(containsSecretLikeKey);
|
||||
if (!isPlainObject(value)) return false;
|
||||
return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeKey(child));
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function hasOnlyKeys(value, allowed) {
|
||||
return Object.keys(value).every((key) => allowed.has(key));
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
export async function migrate(pool) {
|
||||
await pool.query("create extension if not exists timescaledb");
|
||||
await pool.query("create extension if not exists postgis");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_products (
|
||||
id text primary key,
|
||||
version text not null,
|
||||
ontology_revision text not null,
|
||||
delivery_mode text not null,
|
||||
semantic_types jsonb not null,
|
||||
fields jsonb not null,
|
||||
history_policy jsonb not null,
|
||||
active boolean not null default true,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now(),
|
||||
check (jsonb_typeof(semantic_types) = 'array' and jsonb_array_length(semantic_types) > 0),
|
||||
check (jsonb_typeof(fields) = 'array' and jsonb_array_length(fields) > 0)
|
||||
)
|
||||
`);
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_batches (
|
||||
id uuid primary key,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
contract_version text not null,
|
||||
ontology_revision text not null,
|
||||
run_id text not null,
|
||||
sequence integer not null,
|
||||
idempotency_key text not null,
|
||||
received_at timestamptz not null,
|
||||
fact_count integer not null default 0,
|
||||
inserted_fact_count integer not null default 0,
|
||||
created_at timestamptz not null default now(),
|
||||
unique (tenant_id, connection_id, provider_id, data_product_id, idempotency_key)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists request_fingerprint text");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_raw_envelopes (
|
||||
id uuid primary key,
|
||||
batch_id uuid not null unique references external_data_plane_batches(id) on delete cascade,
|
||||
payload_hash text not null,
|
||||
content_type text not null,
|
||||
payload jsonb,
|
||||
payload_ref text,
|
||||
payload_bytes integer,
|
||||
received_at timestamptz not null,
|
||||
expires_at timestamptz not null,
|
||||
check (payload is not null or payload_ref is not null)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_raw_expiry_idx on external_data_plane_raw_envelopes (expires_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_writer_bindings (
|
||||
id uuid primary key,
|
||||
token_hash text not null unique,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
allowed_data_product_ids jsonb not null,
|
||||
expires_at timestamptz not null,
|
||||
active boolean not null default true,
|
||||
created_at timestamptz not null default now(),
|
||||
rotated_at timestamptz,
|
||||
revoked_at timestamptz,
|
||||
check (
|
||||
case when jsonb_typeof(allowed_data_product_ids) = 'array'
|
||||
then jsonb_array_length(allowed_data_product_ids) > 0
|
||||
else false
|
||||
end
|
||||
)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_writer_bindings_active_idx on external_data_plane_writer_bindings (active, expires_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_reader_bindings (
|
||||
id uuid primary key,
|
||||
token_hash text not null unique,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
allowed_data_product_ids jsonb not null,
|
||||
expires_at timestamptz not null,
|
||||
active boolean not null default true,
|
||||
created_at timestamptz not null default now(),
|
||||
rotated_at timestamptz,
|
||||
revoked_at timestamptz,
|
||||
check (
|
||||
case when jsonb_typeof(allowed_data_product_ids) = 'array'
|
||||
then jsonb_array_length(allowed_data_product_ids) > 0
|
||||
else false
|
||||
end
|
||||
)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_reader_bindings_active_idx on external_data_plane_reader_bindings (active, expires_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_facts (
|
||||
id uuid not null,
|
||||
batch_id uuid not null references external_data_plane_batches(id) on delete cascade,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
source_id text not null,
|
||||
semantic_type text not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
attributes jsonb not null default '{}'::jsonb,
|
||||
geometry geography(Point, 4326),
|
||||
fingerprint text not null,
|
||||
primary key (id, observed_at),
|
||||
unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint)
|
||||
)
|
||||
`);
|
||||
await pool.query("select create_hypertable('external_data_plane_facts', 'observed_at', if_not_exists => true, migrate_data => true)");
|
||||
await pool.query("create index if not exists external_data_plane_facts_source_time_idx on external_data_plane_facts (tenant_id, connection_id, data_product_id, source_id, observed_at desc)");
|
||||
await pool.query("create index if not exists external_data_plane_facts_geometry_idx on external_data_plane_facts using gist (geometry)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_current (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
source_id text not null,
|
||||
semantic_type text not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
attributes jsonb not null default '{}'::jsonb,
|
||||
geometry geography(Point, 4326),
|
||||
fingerprint text not null,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)");
|
||||
await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_delivery_state (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
current_cursor bigint not null default 0,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, provider_id, data_product_id)
|
||||
)
|
||||
`);
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_patch_outbox (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
cursor bigint not null,
|
||||
previous_cursor bigint not null,
|
||||
batch_id uuid not null references external_data_plane_batches(id) on delete cascade,
|
||||
operations jsonb not null,
|
||||
emitted_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, provider_id, data_product_id, cursor),
|
||||
check (jsonb_typeof(operations) = 'array' and jsonb_array_length(operations) > 0)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_patch_outbox_retention_idx on external_data_plane_patch_outbox (emitted_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_history (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
source_id text not null,
|
||||
semantic_type text not null,
|
||||
bucket_start timestamptz not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
attributes jsonb not null default '{}'::jsonb,
|
||||
geometry geography(Point, 4326),
|
||||
fingerprint text not null,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (
|
||||
tenant_id, connection_id, provider_id, data_product_id,
|
||||
source_id, semantic_type, bucket_start
|
||||
)
|
||||
)
|
||||
`);
|
||||
await pool.query("select create_hypertable('external_data_plane_history', 'bucket_start', if_not_exists => true, migrate_data => true)");
|
||||
await pool.query("create index if not exists external_data_plane_history_source_time_idx on external_data_plane_history (tenant_id, connection_id, data_product_id, source_id, bucket_start desc)");
|
||||
await pool.query("create index if not exists external_data_plane_history_geometry_idx on external_data_plane_history using gist (geometry)");
|
||||
}
|
||||
@@ -0,0 +1,822 @@
|
||||
import express from "express";
|
||||
import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { createServer } from "node:http";
|
||||
import { Pool } from "pg";
|
||||
import { validateDataProductPublish, validateIntakeBatch } from "@nodedc/external-provider-contract";
|
||||
import { readConfig } from "./config.mjs";
|
||||
import {
|
||||
loadDataProductDefinition,
|
||||
assertPublishMatchesDefinition,
|
||||
persistDataProductDefinition,
|
||||
persistDataProductPublish,
|
||||
pruneBatchReceipts,
|
||||
pruneDataProductHistory,
|
||||
prunePatchOutbox,
|
||||
readDataProductSnapshot,
|
||||
readPatchEvents,
|
||||
} from "./data-product-delivery.mjs";
|
||||
import { normalizeDataProductDefinition, safeDataProductDefinition } from "./data-product-policy.mjs";
|
||||
import { reconcileDataProductDefinitions } from "./definitions.mjs";
|
||||
import { assertBatchTimeBounds, rawRetentionExpiry } from "./intake-policy.mjs";
|
||||
import {
|
||||
assertReaderProduct,
|
||||
createReaderToken,
|
||||
hashReaderToken,
|
||||
normalizeReaderBindingRequest,
|
||||
safeReaderBinding,
|
||||
} from "./reader-binding.mjs";
|
||||
import { migrate } from "./schema.mjs";
|
||||
import {
|
||||
createWriterToken,
|
||||
hashWriterToken,
|
||||
materializeDataProductPublish,
|
||||
materializeWriterBoundBatch,
|
||||
normalizeWriterBindingRequest,
|
||||
safeWriterBinding,
|
||||
} from "./writer-binding.mjs";
|
||||
|
||||
const config = readConfig();
|
||||
const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize });
|
||||
const app = express();
|
||||
const httpServer = createServer(app);
|
||||
let retentionSweepTimer = null;
|
||||
let lastRetentionSweepAt = null;
|
||||
const activeReaderStreams = new Map();
|
||||
const activeStreamResponses = new Set();
|
||||
let shuttingDown = false;
|
||||
let shutdownPromise = null;
|
||||
|
||||
app.disable("x-powered-by");
|
||||
app.use(express.json({ limit: config.maxBatchBytes }));
|
||||
|
||||
app.get("/healthz", asyncRoute(async (_req, res) => {
|
||||
await pool.query("select 1");
|
||||
res.json({
|
||||
ok: true,
|
||||
service: "nodedc-external-data-plane",
|
||||
database: "ready",
|
||||
internalApiConfigured: Boolean(config.internalAccessToken),
|
||||
providerLogic: "absent",
|
||||
commandTransport: "absent",
|
||||
writerBindings: "supported",
|
||||
readerBindings: "supported",
|
||||
dataProductDelivery: "snapshot+durable-patch",
|
||||
legacyIntake: config.legacyIntakeEnabled ? "migration-only" : "disabled",
|
||||
writerBindingProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled",
|
||||
rawRetentionSweep: {
|
||||
mode: "server-scheduled",
|
||||
lastSweepAt: lastRetentionSweepAt,
|
||||
},
|
||||
});
|
||||
}));
|
||||
|
||||
app.put("/internal/data-plane/v1/data-products/:dataProductId", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const definition = normalizeDataProductDefinition({ ...req.body, id: req.params.dataProductId });
|
||||
const saved = await persistDataProductDefinition(pool, definition);
|
||||
res.json({ ok: true, dataProduct: safeDataProductDefinition(saved) });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/data-products/:dataProductId/publish", requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_publish_scope_headers_forbidden");
|
||||
const validation = validateDataProductPublish(req.body, {
|
||||
maxFacts: config.maxFactsPerPublish,
|
||||
maxAttributesBytes: config.maxAttributesBytesPerFact,
|
||||
});
|
||||
if (!validation.ok) throw httpError(422, validation.errors[0] || "invalid_data_product_publish");
|
||||
const dataProductId = requireIdentifier(req.params.dataProductId, "data_product_id_invalid");
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
const batch = materializeDataProductPublish(req.body, req.writerBinding, definition, dataProductId);
|
||||
const canonicalValidation = validateIntakeBatch(batch);
|
||||
if (!canonicalValidation.ok) throw httpError(422, "materialized_publish_invalid");
|
||||
assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
const receipt = await persistDataProductPublish(pool, batch, definition, {
|
||||
maxPatchOperations: config.maxPatchOperations,
|
||||
maxPatchBytes: config.maxPatchBytes,
|
||||
});
|
||||
res.status(receipt.idempotent ? 200 : 201).json({ ok: true, ...receipt });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/writer/data-products", requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
const dataProducts = await listGrantedProducts(req.writerBinding);
|
||||
res.json({ ok: true, dataProducts });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/reader/data-products", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
const dataProducts = await listGrantedProducts(req.readerBinding);
|
||||
res.json({ ok: true, dataProducts });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/intake", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const validation = validateIntakeBatch(req.body);
|
||||
if (!validation.ok) throw httpError(422, "invalid_intake_batch");
|
||||
assertBatchTimeBounds(req.body, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
await assertLegacyBatchProduct(req.body);
|
||||
const scope = requireScope(req);
|
||||
if (scope.tenantId !== req.body.source.tenantId || scope.connectionId !== req.body.source.connectionId) {
|
||||
throw httpError(403, "scope_mismatch");
|
||||
}
|
||||
|
||||
const result = await persistBatch(req.body);
|
||||
res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/intake/writer-bound", requireLegacyIntake, requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
const batch = materializeWriterBoundBatch(req.body, req.writerBinding, {
|
||||
hasScopeHeaders: hasScopeHeaders(req),
|
||||
});
|
||||
const validation = validateIntakeBatch(batch);
|
||||
if (!validation.ok) throw httpError(422, "invalid_intake_batch");
|
||||
assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
await assertLegacyBatchProduct(batch);
|
||||
|
||||
const result = await persistBatch(batch);
|
||||
res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const policy = normalizeWriterBindingRequest(req.body, {
|
||||
maxTtlDays: config.writerBindingMaxTtlDays,
|
||||
});
|
||||
await assertRegisteredProductIds(policy.allowedDataProductIds);
|
||||
const token = createWriterToken();
|
||||
const bindingId = randomUUID();
|
||||
const result = await pool.query(
|
||||
`insert into external_data_plane_writer_bindings (
|
||||
id, token_hash, tenant_id, connection_id, provider_id,
|
||||
allowed_data_product_ids, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[
|
||||
bindingId,
|
||||
hashWriterToken(token),
|
||||
policy.tenantId,
|
||||
policy.connectionId,
|
||||
policy.providerId,
|
||||
JSON.stringify(policy.allowedDataProductIds),
|
||||
policy.expiresAt,
|
||||
],
|
||||
);
|
||||
// The token is intentionally returned exactly once to a trusted provisioner.
|
||||
// It must be placed directly into an opaque Engine credential reference and
|
||||
// must never be logged, stored in L2 graph data or shown to a consumer.
|
||||
sendOneTimeCapability(res, 201, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid");
|
||||
const token = createWriterToken();
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_writer_bindings
|
||||
set token_hash = $2, rotated_at = now()
|
||||
where id = $1 and active = true and expires_at > now()
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId, hashWriterToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive");
|
||||
sendOneTimeCapability(res, 200, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid");
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_writer_bindings
|
||||
set active = false, revoked_at = now()
|
||||
where id = $1 and active = true
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive");
|
||||
res.json({ ok: true, writerBinding: safeWriterBinding(result.rows[0]) });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const policy = normalizeReaderBindingRequest(req.body, {
|
||||
maxTtlDays: config.writerBindingMaxTtlDays,
|
||||
});
|
||||
await assertRegisteredProductIds(policy.allowedDataProductIds);
|
||||
const token = createReaderToken();
|
||||
const bindingId = randomUUID();
|
||||
const result = await pool.query(
|
||||
`insert into external_data_plane_reader_bindings (
|
||||
id, token_hash, tenant_id, connection_id, provider_id,
|
||||
allowed_data_product_ids, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[
|
||||
bindingId,
|
||||
hashReaderToken(token),
|
||||
policy.tenantId,
|
||||
policy.connectionId,
|
||||
policy.providerId,
|
||||
JSON.stringify(policy.allowedDataProductIds),
|
||||
policy.expiresAt,
|
||||
],
|
||||
);
|
||||
sendOneTimeCapability(res, 201, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid");
|
||||
const token = createReaderToken();
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_reader_bindings
|
||||
set token_hash = $2, rotated_at = now()
|
||||
where id = $1 and active = true and expires_at > now()
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId, hashReaderToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive");
|
||||
sendOneTimeCapability(res, 200, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid");
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_reader_bindings
|
||||
set active = false, revoked_at = now()
|
||||
where id = $1 and active = true
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive");
|
||||
res.json({ ok: true, readerBinding: safeReaderBinding(result.rows[0]) });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/snapshot", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
|
||||
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
const limit = boundedLimit(req.query.limit, 5000, 1, 5000);
|
||||
const snapshot = await readDataProductSnapshot(pool, req.readerBinding, definition, { limit });
|
||||
res.json(snapshot);
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/stream", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
if (shuttingDown) throw httpError(503, "service_shutting_down");
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
|
||||
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
if (definition.deliveryMode !== "snapshot+patch") throw httpError(409, "data_product_stream_not_supported");
|
||||
const streamCount = activeReaderStreams.get(req.readerBinding.id) || 0;
|
||||
if (streamCount >= config.maxReaderStreams) throw httpError(429, "reader_stream_limit_exceeded");
|
||||
activeReaderStreams.set(req.readerBinding.id, streamCount + 1);
|
||||
activeStreamResponses.add(res);
|
||||
|
||||
let poll = null;
|
||||
let heartbeat = null;
|
||||
let polling = false;
|
||||
let heartbeatWriting = false;
|
||||
let cleanedUp = false;
|
||||
const cleanup = () => {
|
||||
if (cleanedUp) return;
|
||||
cleanedUp = true;
|
||||
if (poll) clearInterval(poll);
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
activeStreamResponses.delete(res);
|
||||
const remaining = Math.max(0, (activeReaderStreams.get(req.readerBinding.id) || 1) - 1);
|
||||
if (remaining) activeReaderStreams.set(req.readerBinding.id, remaining);
|
||||
else activeReaderStreams.delete(req.readerBinding.id);
|
||||
};
|
||||
req.once("close", cleanup);
|
||||
res.once("close", cleanup);
|
||||
|
||||
try {
|
||||
let cursor = parseCursor(req.get("last-event-id") || req.query.after || "0");
|
||||
const initialEvents = await readPatchEvents(pool, req.readerBinding, definition, cursor);
|
||||
if (res.destroyed || cleanedUp) return;
|
||||
res.status(200);
|
||||
res.set({
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"Content-Type": "text/event-stream",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
});
|
||||
res.flushHeaders();
|
||||
await writeSseFrame(res, ": nodedc-data-product-stream\n\n");
|
||||
await writeSseFrame(res, `event: nodedc.data-product.ready.v1\ndata: ${JSON.stringify({
|
||||
schemaVersion: "nodedc.data-product.ready/v1",
|
||||
dataProductId,
|
||||
cursor: cursor.toString(),
|
||||
emittedAt: new Date().toISOString(),
|
||||
})}\n\n`);
|
||||
for (const event of initialEvents) {
|
||||
await writePatchEvent(res, event);
|
||||
cursor = BigInt(event.cursor);
|
||||
}
|
||||
|
||||
const pump = async () => {
|
||||
if (polling || res.writableEnded || res.destroyed || shuttingDown) return;
|
||||
polling = true;
|
||||
try {
|
||||
await assertReaderStreamAccess(req.readerBinding, dataProductId);
|
||||
const events = await readPatchEvents(pool, req.readerBinding, definition, cursor);
|
||||
for (const event of events) {
|
||||
await writePatchEvent(res, event);
|
||||
cursor = BigInt(event.cursor);
|
||||
}
|
||||
} catch (error) {
|
||||
const code = safeErrorCode(error);
|
||||
if (!res.writableEnded && !res.destroyed) {
|
||||
await writeSseFrame(res, `event: error\ndata: ${JSON.stringify({ ok: false, error: code })}\n\n`).catch(() => {});
|
||||
res.end();
|
||||
}
|
||||
} finally {
|
||||
polling = false;
|
||||
}
|
||||
};
|
||||
poll = setInterval(() => { void pump(); }, config.streamPollMs);
|
||||
poll.unref();
|
||||
heartbeat = setInterval(() => {
|
||||
if (polling || heartbeatWriting || res.writableEnded || res.destroyed) return;
|
||||
heartbeatWriting = true;
|
||||
void writeSseFrame(res, `: heartbeat ${Date.now()}\n\n`)
|
||||
.catch(() => { if (!res.writableEnded) res.end(); })
|
||||
.finally(() => { heartbeatWriting = false; });
|
||||
}, config.streamHeartbeatMs);
|
||||
heartbeat.unref();
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
if (res.headersSent) {
|
||||
if (!res.writableEnded) res.end();
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/current", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireScope(req);
|
||||
const dataProductId = String(req.params.dataProductId || "");
|
||||
if (!isIdentifier(dataProductId)) throw httpError(400, "data_product_id_invalid");
|
||||
const limit = boundedLimit(req.query.limit, 200, 1, 1000);
|
||||
const rows = await pool.query(
|
||||
`select
|
||||
provider_id as "providerId", data_product_id as "dataProductId",
|
||||
source_id as "sourceId", semantic_type as "semanticType",
|
||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||
case when geometry is null then null
|
||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
||||
)) end as geometry
|
||||
from external_data_plane_current
|
||||
where tenant_id = $1 and connection_id = $2 and data_product_id = $3
|
||||
order by observed_at desc, source_id asc
|
||||
limit $4`,
|
||||
[scope.tenantId, scope.connectionId, dataProductId, limit],
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
dataProductId,
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
facts: rows.rows,
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/status", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireScope(req);
|
||||
const result = await pool.query(
|
||||
`select count(*)::integer as "currentFactCount", max(received_at) as "lastReceivedAt"
|
||||
from external_data_plane_current where tenant_id = $1 and connection_id = $2`,
|
||||
[scope.tenantId, scope.connectionId],
|
||||
);
|
||||
res.json({ ok: true, ...scope, ...result.rows[0], providerLogic: "absent", commandTransport: "absent" });
|
||||
}));
|
||||
|
||||
app.use((error, _req, res, _next) => {
|
||||
const status = Number(error?.status || 500);
|
||||
const publicStatus = status >= 400 && status < 600 ? status : 500;
|
||||
console.error(JSON.stringify({ event: "external_data_plane_error", error: safeErrorCode(error), status: publicStatus }));
|
||||
res.status(publicStatus).json({ ok: false, error: publicStatus >= 500 ? "internal_error" : safeErrorCode(error) });
|
||||
});
|
||||
|
||||
await migrate(pool);
|
||||
const bundledDataProducts = await reconcileDataProductDefinitions(pool);
|
||||
retentionSweepTimer = setInterval(() => {
|
||||
void sweepRetention().catch((error) => {
|
||||
console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) }));
|
||||
});
|
||||
}, config.retentionSweepMs);
|
||||
retentionSweepTimer.unref();
|
||||
|
||||
httpServer.listen(config.port, "0.0.0.0", () => {
|
||||
console.log(`NODE.DC External Data Plane listening on http://0.0.0.0:${config.port}`);
|
||||
console.log(JSON.stringify({ event: "external_data_plane_definitions_ready", count: bundledDataProducts.length }));
|
||||
setImmediate(() => {
|
||||
void sweepRetention().catch((error) => {
|
||||
console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) }));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
async function persistBatch(batch) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
const existing = await client.query(
|
||||
`select id, fact_count as "factCount", inserted_fact_count as "insertedFactCount"
|
||||
from external_data_plane_batches
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3
|
||||
and data_product_id = $4 and idempotency_key = $5
|
||||
for update`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.batch.idempotencyKey,
|
||||
],
|
||||
);
|
||||
if (existing.rowCount) {
|
||||
await client.query("commit");
|
||||
return { batchId: existing.rows[0].id, idempotent: true, ...existing.rows[0] };
|
||||
}
|
||||
|
||||
const batchId = randomUUID();
|
||||
await client.query(
|
||||
`insert into external_data_plane_batches (
|
||||
id, tenant_id, connection_id, provider_id, data_product_id, contract_version,
|
||||
ontology_revision, run_id, sequence, idempotency_key, received_at, fact_count
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.contract.version, batch.contract.ontologyRevision,
|
||||
batch.batch.runId, batch.batch.sequence, batch.batch.idempotencyKey,
|
||||
batch.batch.receivedAt, batch.facts.length,
|
||||
],
|
||||
);
|
||||
|
||||
if (batch.raw) await persistRawEnvelope(client, batchId, batch);
|
||||
|
||||
const records = batch.facts.map((fact) => ({
|
||||
id: randomUUID(),
|
||||
source_id: fact.sourceId,
|
||||
semantic_type: fact.semanticType,
|
||||
observed_at: fact.observedAt,
|
||||
attributes: fact.attributes || {},
|
||||
geometry: fact.geometry || null,
|
||||
fingerprint: hash(fact),
|
||||
}));
|
||||
const inserted = await client.query(
|
||||
`insert into external_data_plane_facts (
|
||||
id, batch_id, tenant_id, connection_id, provider_id, data_product_id,
|
||||
source_id, semantic_type, observed_at, received_at, attributes, geometry, fingerprint
|
||||
)
|
||||
select item.id::uuid, $1, $2, $3, $4, $5,
|
||||
item.source_id, item.semantic_type, item.observed_at, $6,
|
||||
coalesce(item.attributes, '{}'::jsonb),
|
||||
case when item.geometry is null then null
|
||||
else ST_SetSRID(ST_MakePoint(
|
||||
(item.geometry->'coordinates'->>0)::double precision,
|
||||
(item.geometry->'coordinates'->>1)::double precision
|
||||
), 4326)::geography end,
|
||||
item.fingerprint
|
||||
from jsonb_to_recordset($7::jsonb) as item(
|
||||
id text, source_id text, semantic_type text, observed_at timestamptz,
|
||||
attributes jsonb, geometry jsonb, fingerprint text
|
||||
)
|
||||
on conflict do nothing
|
||||
returning id`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.batch.receivedAt, JSON.stringify(records),
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`insert into external_data_plane_current (
|
||||
tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, updated_at
|
||||
)
|
||||
select tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, now()
|
||||
from external_data_plane_facts where batch_id = $1
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||
do update set
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
attributes = excluded.attributes,
|
||||
geometry = excluded.geometry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at >= external_data_plane_current.observed_at`,
|
||||
[batchId],
|
||||
);
|
||||
await client.query(
|
||||
"update external_data_plane_batches set inserted_fact_count = $2 where id = $1",
|
||||
[batchId, inserted.rowCount],
|
||||
);
|
||||
await client.query("commit");
|
||||
return { batchId, idempotent: false, factCount: batch.facts.length, insertedFactCount: inserted.rowCount };
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function persistRawEnvelope(client, batchId, batch) {
|
||||
const serialized = batch.raw.payload === undefined ? null : JSON.stringify(batch.raw.payload);
|
||||
const expiresAt = rawRetentionExpiry({ rawRetentionDays: config.rawRetentionDays });
|
||||
await client.query(
|
||||
`insert into external_data_plane_raw_envelopes (
|
||||
id, batch_id, payload_hash, content_type, payload, payload_ref, payload_bytes, received_at, expires_at
|
||||
) values ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9)`,
|
||||
[
|
||||
randomUUID(), batchId, batch.raw.hash || hash(batch.raw.payload), batch.raw.contentType, serialized, batch.raw.ref || null,
|
||||
serialized ? Buffer.byteLength(serialized) : null, batch.batch.receivedAt, expiresAt,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
function requireInternalApi(req, _res, next) {
|
||||
if (!config.internalAccessToken) return next(httpError(503, "internal_api_not_configured"));
|
||||
const value = bearerToken(req);
|
||||
if (!value || !safeEqual(value, config.internalAccessToken)) return next(httpError(401, "unauthorized"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireLegacyIntake(_req, _res, next) {
|
||||
if (!config.legacyIntakeEnabled) return next(httpError(410, "legacy_intake_disabled"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireProvisionerApi(req, _res, next) {
|
||||
if (!config.provisionerApiEnabled) return next(httpError(503, "provisioner_api_disabled"));
|
||||
if (!config.provisionerAccessToken) return next(httpError(503, "provisioner_api_not_configured"));
|
||||
const value = bearerToken(req);
|
||||
if (!value || !safeEqual(value, config.provisionerAccessToken)) return next(httpError(401, "provisioner_unauthorized"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireWriterBinding(req, _res, next) {
|
||||
return resolveWriterBinding(req)
|
||||
.then((binding) => {
|
||||
req.writerBinding = binding;
|
||||
next();
|
||||
})
|
||||
.catch(next);
|
||||
}
|
||||
|
||||
function requireReaderBinding(req, _res, next) {
|
||||
return resolveReaderBinding(req)
|
||||
.then((binding) => {
|
||||
req.readerBinding = binding;
|
||||
next();
|
||||
})
|
||||
.catch(next);
|
||||
}
|
||||
|
||||
async function resolveWriterBinding(req) {
|
||||
const token = bearerToken(req);
|
||||
if (!token) throw httpError(401, "writer_binding_unauthorized");
|
||||
const result = await pool.query(
|
||||
`select id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"
|
||||
from external_data_plane_writer_bindings
|
||||
where token_hash = $1 and active = true and expires_at > now()`,
|
||||
[hashWriterToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "writer_binding_unauthorized");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function resolveReaderBinding(req) {
|
||||
const token = bearerToken(req);
|
||||
if (!token) throw httpError(401, "reader_binding_unauthorized");
|
||||
const result = await pool.query(
|
||||
`select id, token_hash as "tokenHash", tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"
|
||||
from external_data_plane_reader_bindings
|
||||
where token_hash = $1 and active = true and expires_at > now()`,
|
||||
[hashReaderToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "reader_binding_unauthorized");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function assertReaderStreamAccess(binding, dataProductId) {
|
||||
const result = await pool.query(
|
||||
`select binding.id
|
||||
from external_data_plane_reader_bindings as binding
|
||||
join external_data_plane_products as product
|
||||
on product.id = $3 and product.active = true
|
||||
where binding.id = $1 and binding.token_hash = $2
|
||||
and binding.active = true and binding.expires_at > now()
|
||||
and binding.allowed_data_product_ids ? $3`,
|
||||
[binding.id, binding.tokenHash, dataProductId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "reader_stream_access_revoked");
|
||||
}
|
||||
|
||||
async function assertLegacyBatchProduct(batch) {
|
||||
const definition = await loadDataProductDefinition(pool, batch.contract.dataProductId);
|
||||
if (!definition) throw httpError(422, "legacy_data_product_not_registered");
|
||||
if (definition.version !== batch.contract.version || definition.ontologyRevision !== batch.contract.ontologyRevision) {
|
||||
throw httpError(422, "legacy_data_product_contract_mismatch");
|
||||
}
|
||||
assertPublishMatchesDefinition(batch, definition);
|
||||
}
|
||||
|
||||
async function listGrantedProducts(binding) {
|
||||
const allowed = Array.isArray(binding.allowedDataProductIds) ? binding.allowedDataProductIds : [];
|
||||
if (!allowed.length) return [];
|
||||
const result = await pool.query(
|
||||
`select id, version, ontology_revision as "ontologyRevision",
|
||||
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
|
||||
fields, history_policy as "historyPolicy", active,
|
||||
created_at as "createdAt", updated_at as "updatedAt"
|
||||
from external_data_plane_products
|
||||
where active = true and id = any($1::text[])
|
||||
order by id asc`,
|
||||
[allowed],
|
||||
);
|
||||
return result.rows.map(safeDataProductDefinition);
|
||||
}
|
||||
|
||||
async function assertRegisteredProductIds(dataProductIds) {
|
||||
const result = await pool.query(
|
||||
"select id from external_data_plane_products where active = true and id = any($1::text[])",
|
||||
[dataProductIds],
|
||||
);
|
||||
const registered = new Set(result.rows.map((row) => row.id));
|
||||
if (dataProductIds.some((id) => !registered.has(id))) throw httpError(422, "binding_data_product_not_registered");
|
||||
}
|
||||
|
||||
function requireScope(req) {
|
||||
const tenantId = String(req.get("x-nodedc-tenant-id") || "");
|
||||
const connectionId = String(req.get("x-nodedc-connection-id") || "");
|
||||
if (!isIdentifier(tenantId) || !isIdentifier(connectionId)) throw httpError(400, "scope_headers_required");
|
||||
return { tenantId, connectionId };
|
||||
}
|
||||
|
||||
function hasScopeHeaders(req) {
|
||||
return Object.hasOwn(req.headers, "x-nodedc-tenant-id") || Object.hasOwn(req.headers, "x-nodedc-connection-id");
|
||||
}
|
||||
|
||||
function bearerToken(req) {
|
||||
return String(req.get("authorization") || "").replace(/^Bearer\s+/i, "");
|
||||
}
|
||||
|
||||
function requireUuid(value, code) {
|
||||
const normalized = String(value || "");
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(normalized)) {
|
||||
throw httpError(400, code);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireIdentifier(value, code) {
|
||||
const normalized = String(value || "");
|
||||
if (!isIdentifier(normalized)) throw httpError(400, code);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseCursor(value) {
|
||||
const normalized = String(value ?? "");
|
||||
if (!/^(?:0|[1-9]\d*)$/.test(normalized)) throw httpError(400, "data_product_cursor_invalid");
|
||||
const cursor = BigInt(normalized);
|
||||
if (cursor > 9_223_372_036_854_775_807n) throw httpError(400, "data_product_cursor_invalid");
|
||||
return cursor;
|
||||
}
|
||||
|
||||
function writePatchEvent(res, event) {
|
||||
return writeSseFrame(res, `id: ${event.cursor}\nevent: nodedc.data-product.patch.v1\ndata: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
|
||||
function writeSseFrame(res, frame) {
|
||||
if (res.writableEnded || res.destroyed) return Promise.reject(httpError(499, "reader_stream_closed"));
|
||||
if (res.write(frame)) return Promise.resolve();
|
||||
return new Promise((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
res.off("drain", onDrain);
|
||||
res.off("close", onClose);
|
||||
res.off("error", onError);
|
||||
};
|
||||
const onDrain = () => { cleanup(); resolve(); };
|
||||
const onClose = () => { cleanup(); reject(httpError(499, "reader_stream_closed")); };
|
||||
const onError = (error) => { cleanup(); reject(error); };
|
||||
res.once("drain", onDrain);
|
||||
res.once("close", onClose);
|
||||
res.once("error", onError);
|
||||
});
|
||||
}
|
||||
|
||||
function sendOneTimeCapability(res, status, payload) {
|
||||
// Create/rotate is the only boundary where a capability exists in
|
||||
// plaintext. A trusted provisioner must consume the response in memory and
|
||||
// place it directly into its opaque destination; intermediaries must never
|
||||
// cache or persist it.
|
||||
res.set({
|
||||
"Cache-Control": "no-store, max-age=0",
|
||||
Pragma: "no-cache",
|
||||
Expires: "0",
|
||||
});
|
||||
return res.status(status).json(payload);
|
||||
}
|
||||
|
||||
function safeEqual(left, right) {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
|
||||
}
|
||||
|
||||
function boundedLimit(value, fallback, min, max) {
|
||||
const candidate = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isInteger(candidate)) return fallback;
|
||||
return Math.max(min, Math.min(max, candidate));
|
||||
}
|
||||
|
||||
function hash(value) {
|
||||
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
||||
}
|
||||
|
||||
function isIdentifier(value) {
|
||||
return /^[a-z][a-z0-9._:-]{2,127}$/i.test(value);
|
||||
}
|
||||
|
||||
function asyncRoute(handler) {
|
||||
return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next);
|
||||
}
|
||||
|
||||
function httpError(status, code) {
|
||||
return Object.assign(new Error(code), { status, code });
|
||||
}
|
||||
|
||||
function safeErrorCode(error) {
|
||||
return String(error?.code || error?.message || "internal_error").replace(/[^a-z0-9_.:-]/gi, "_").slice(0, 120);
|
||||
}
|
||||
|
||||
async function sweepExpiredRawEnvelopes() {
|
||||
const result = await pool.query(
|
||||
`with expired as (
|
||||
select id from external_data_plane_raw_envelopes
|
||||
where expires_at < now()
|
||||
order by expires_at asc
|
||||
limit $1
|
||||
)
|
||||
delete from external_data_plane_raw_envelopes as target
|
||||
using expired where target.id = expired.id`,
|
||||
[config.retentionDeleteLimit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
async function sweepRetention() {
|
||||
const rawEnvelopeCount = await sweepExpiredRawEnvelopes();
|
||||
const patchEventCount = await prunePatchOutbox(pool, {
|
||||
retentionMs: config.patchRetentionMs,
|
||||
limit: config.retentionDeleteLimit,
|
||||
});
|
||||
const historyFactCount = await pruneDataProductHistory(pool, { limit: config.retentionDeleteLimit });
|
||||
const batchReceiptCount = await pruneBatchReceipts(pool, {
|
||||
retentionMs: config.receiptRetentionMs,
|
||||
limit: config.retentionDeleteLimit,
|
||||
});
|
||||
lastRetentionSweepAt = new Date().toISOString();
|
||||
return { rawEnvelopeCount, patchEventCount, historyFactCount, batchReceiptCount };
|
||||
}
|
||||
|
||||
async function shutdown() {
|
||||
if (shutdownPromise) return shutdownPromise;
|
||||
shutdownPromise = (async () => {
|
||||
shuttingDown = true;
|
||||
if (retentionSweepTimer) clearInterval(retentionSweepTimer);
|
||||
for (const response of activeStreamResponses) {
|
||||
if (!response.writableEnded) response.end();
|
||||
}
|
||||
const closed = new Promise((resolve) => httpServer.close(resolve));
|
||||
httpServer.closeIdleConnections?.();
|
||||
const forceClose = setTimeout(() => httpServer.closeAllConnections?.(), 250);
|
||||
await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 2_000))]);
|
||||
clearTimeout(forceClose);
|
||||
httpServer.closeAllConnections?.();
|
||||
await pool.end();
|
||||
})();
|
||||
return shutdownPromise;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
|
||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||
const TOKEN_PREFIX = "ndc_edpwb_";
|
||||
const SECRET_LIKE_KEY = /(token|secret|password|access[_-]?token|refresh[_-]?token|api[_-]?key)/i;
|
||||
const BINDING_REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]);
|
||||
const BINDING_SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]);
|
||||
|
||||
/**
|
||||
* Produces an opaque, high-entropy capability. The plaintext is returned only
|
||||
* to the trusted provisioning caller; persistence uses its SHA-256 digest.
|
||||
*/
|
||||
export function createWriterToken() {
|
||||
return `${TOKEN_PREFIX}${randomBytes(32).toString("base64url")}`;
|
||||
}
|
||||
|
||||
export function hashWriterToken(token) {
|
||||
return createHash("sha256").update(String(token), "utf8").digest("hex");
|
||||
}
|
||||
|
||||
export function normalizeWriterBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(value.source)) {
|
||||
throw writerBindingError("writer_binding_request_invalid");
|
||||
}
|
||||
if (containsSecretLikeKey(value)) {
|
||||
throw writerBindingError("writer_binding_request_secret_material_forbidden");
|
||||
}
|
||||
if (value.endpoint !== undefined || value.url !== undefined || value.host !== undefined) {
|
||||
throw writerBindingError("writer_binding_request_transport_forbidden");
|
||||
}
|
||||
if (!hasOnlyKeys(value, BINDING_REQUEST_KEYS) || !hasOnlyKeys(value.source, BINDING_SOURCE_KEYS)) {
|
||||
throw writerBindingError("writer_binding_request_fields_invalid");
|
||||
}
|
||||
|
||||
const tenantId = normalizeIdentifier(value.source.tenantId);
|
||||
const connectionId = normalizeIdentifier(value.source.connectionId);
|
||||
const providerId = normalizeIdentifier(value.source.providerId);
|
||||
if (!tenantId || !connectionId || !providerId) {
|
||||
throw writerBindingError("writer_binding_scope_invalid");
|
||||
}
|
||||
|
||||
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
|
||||
if (!allowedDataProductIds.length) {
|
||||
throw writerBindingError("writer_binding_data_products_invalid");
|
||||
}
|
||||
|
||||
const expiresAt = new Date(String(value.expiresAt || ""));
|
||||
const maxExpiresAt = new Date(now.getTime() + maxTtlDays * 24 * 60 * 60 * 1000);
|
||||
if (Number.isNaN(expiresAt.getTime()) || expiresAt <= now || expiresAt > maxExpiresAt) {
|
||||
throw writerBindingError("writer_binding_expiry_invalid");
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
tenantId,
|
||||
connectionId,
|
||||
providerId,
|
||||
allowedDataProductIds,
|
||||
expiresAt: expiresAt.toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a caller-provided, deliberately unscoped intake envelope into the
|
||||
* canonical scoped form. Caller scope is rejected, never trusted or merged.
|
||||
*/
|
||||
export function materializeWriterBoundBatch(value, binding, { hasScopeHeaders = false, now = new Date() } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(value.source) || !isPlainObject(binding)) {
|
||||
throw writerBindingError("writer_bound_intake_invalid");
|
||||
}
|
||||
if (hasScopeHeaders || value.source.tenantId !== undefined || value.source.connectionId !== undefined) {
|
||||
throw writerBindingError("writer_bound_scope_forbidden");
|
||||
}
|
||||
if (binding.active !== true || !bindingIsCurrent(binding, now)) {
|
||||
throw writerBindingError("writer_binding_inactive");
|
||||
}
|
||||
|
||||
const providerId = normalizeIdentifier(value.source.providerId);
|
||||
const tenantId = normalizeIdentifier(binding.tenantId);
|
||||
const connectionId = normalizeIdentifier(binding.connectionId);
|
||||
if (!providerId || !tenantId || !connectionId || providerId !== binding.providerId) {
|
||||
throw writerBindingError("writer_binding_provider_forbidden");
|
||||
}
|
||||
|
||||
const dataProductId = normalizeIdentifier(value.contract?.dataProductId);
|
||||
const allowedDataProductIds = uniqueIdentifiers(binding.allowedDataProductIds);
|
||||
if (!dataProductId || !allowedDataProductIds.includes(dataProductId)) {
|
||||
throw writerBindingError("writer_binding_data_product_forbidden");
|
||||
}
|
||||
|
||||
return {
|
||||
...value,
|
||||
source: { providerId, tenantId, connectionId },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Materializes the provider-neutral Data Product publish wire form. Unlike the
|
||||
* legacy writer-bound intake, the caller cannot send provider identity,
|
||||
* contract metadata, receivedAt or any scope at all.
|
||||
*/
|
||||
export function materializeDataProductPublish(value, binding, definition, dataProductId, { now = new Date() } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(binding) || !isPlainObject(definition)) {
|
||||
throw writerBindingError("data_product_publish_invalid");
|
||||
}
|
||||
if (binding.active !== true || !bindingIsCurrent(binding, now)) {
|
||||
throw writerBindingError("writer_binding_inactive");
|
||||
}
|
||||
const normalizedProductId = normalizeIdentifier(dataProductId);
|
||||
const allowedDataProductIds = uniqueIdentifiers(binding.allowedDataProductIds);
|
||||
if (!normalizedProductId || normalizedProductId !== definition.id || !allowedDataProductIds.includes(normalizedProductId)) {
|
||||
throw writerBindingError("writer_binding_data_product_forbidden");
|
||||
}
|
||||
const tenantId = normalizeIdentifier(binding.tenantId);
|
||||
const connectionId = normalizeIdentifier(binding.connectionId);
|
||||
const providerId = normalizeIdentifier(binding.providerId);
|
||||
if (!tenantId || !connectionId || !providerId) throw writerBindingError("writer_binding_scope_invalid");
|
||||
|
||||
return {
|
||||
schemaVersion: "nodedc.external-provider-contract/v1",
|
||||
source: { tenantId, connectionId, providerId },
|
||||
contract: {
|
||||
dataProductId: normalizedProductId,
|
||||
ontologyRevision: definition.ontologyRevision,
|
||||
version: definition.version,
|
||||
},
|
||||
batch: {
|
||||
runId: value.batch?.runId,
|
||||
sequence: value.batch?.sequence,
|
||||
idempotencyKey: value.batch?.idempotencyKey,
|
||||
receivedAt: now.toISOString(),
|
||||
},
|
||||
facts: value.facts,
|
||||
};
|
||||
}
|
||||
|
||||
export function safeWriterBinding(binding) {
|
||||
return {
|
||||
id: binding.id,
|
||||
tenantId: binding.tenantId,
|
||||
connectionId: binding.connectionId,
|
||||
providerId: binding.providerId,
|
||||
allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds),
|
||||
active: binding.active === true,
|
||||
expiresAt: new Date(binding.expiresAt).toISOString(),
|
||||
createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined,
|
||||
rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined,
|
||||
revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function writerBindingError(code) {
|
||||
return Object.assign(new Error(code), { status: 400, code });
|
||||
}
|
||||
|
||||
function bindingIsCurrent(binding, now) {
|
||||
const expiresAt = new Date(binding.expiresAt);
|
||||
return !Number.isNaN(expiresAt.getTime()) && expiresAt > now;
|
||||
}
|
||||
|
||||
function normalizeIdentifier(value) {
|
||||
const normalized = typeof value === "string" ? value.trim() : "";
|
||||
return IDENTIFIER.test(normalized) ? normalized : "";
|
||||
}
|
||||
|
||||
function uniqueIdentifiers(value) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return [...new Set(value.map(normalizeIdentifier).filter(Boolean))];
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function containsSecretLikeKey(value) {
|
||||
if (Array.isArray(value)) return value.some(containsSecretLikeKey);
|
||||
if (!isPlainObject(value)) return false;
|
||||
return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeKey(child));
|
||||
}
|
||||
|
||||
function hasOnlyKeys(value, allowedKeys) {
|
||||
return Object.keys(value).every((key) => allowedKeys.has(key));
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import net from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract";
|
||||
|
||||
const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL;
|
||||
if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required");
|
||||
const port = await freePort();
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-api-test-"));
|
||||
const provisionerSecret = "nodedc_edp_integration_provisioner_secret_0123456789abcdef";
|
||||
const secretPath = join(directory, "provisioner-token");
|
||||
await writeFile(secretPath, `${provisionerSecret}\n`, { mode: 0o600 });
|
||||
|
||||
const child = spawn(process.execPath, ["src/server.mjs"], {
|
||||
cwd: new URL("..", import.meta.url),
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: String(port),
|
||||
EXTERNAL_DATA_PLANE_DATABASE_URL: databaseUrl,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath,
|
||||
EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000",
|
||||
EXTERNAL_DATA_PLANE_STREAM_POLL_MS: "250",
|
||||
EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS: "5000",
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let childOutput = "";
|
||||
child.stdout.on("data", (chunk) => { childOutput += chunk.toString(); });
|
||||
child.stderr.on("data", (chunk) => { childOutput += chunk.toString(); });
|
||||
|
||||
try {
|
||||
await waitForHealth();
|
||||
const productId = "api.test.positions.v1";
|
||||
const product = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}`, {
|
||||
method: "PUT",
|
||||
token: provisionerSecret,
|
||||
body: {
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.api.test.v1",
|
||||
deliveryMode: "snapshot+patch",
|
||||
semanticTypes: ["map.moving_object"],
|
||||
fields: ["source_id", "observed_at", "geometry", "status"],
|
||||
history: { mode: "sampled", intervalMs: 60_000, strategy: "latest-per-entity-per-bucket", retentionDays: 30 },
|
||||
},
|
||||
});
|
||||
assert.equal(product.dataProduct.id, productId);
|
||||
|
||||
const expiry = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString();
|
||||
const scope = { tenantId: "api-tenant", connectionId: "api-connection", providerId: "api-provider" };
|
||||
const writerIssuance = await rawJsonRequest("/internal/data-plane/v1/writer-bindings", {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry },
|
||||
});
|
||||
const readerIssuance = await rawJsonRequest("/internal/data-plane/v1/reader-bindings", {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry },
|
||||
});
|
||||
assertOneTimeCapabilityResponse(writerIssuance.response);
|
||||
assertOneTimeCapabilityResponse(readerIssuance.response);
|
||||
const writer = writerIssuance.value;
|
||||
const reader = readerIssuance.value;
|
||||
assert.equal(typeof writer.token, "string");
|
||||
assert.equal(typeof reader.token, "string");
|
||||
|
||||
const catalog = await jsonRequest("/internal/data-plane/v1/writer/data-products", { token: writer.token });
|
||||
assert.deepEqual(catalog.dataProducts.map((value) => value.id), [productId]);
|
||||
|
||||
const first = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61),
|
||||
});
|
||||
assert.equal(first.currentUpdatedCount, 1);
|
||||
const duplicate = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61),
|
||||
});
|
||||
assert.equal(duplicate.idempotent, true);
|
||||
const reusedKey = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${writer.token}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify(publish("api-run-01", "2026-07-15T12:00:00.000Z", 99.9)),
|
||||
});
|
||||
assert.equal(reusedKey.status, 409);
|
||||
assert.equal((await reusedKey.json()).error, "idempotency_key_reused");
|
||||
|
||||
const snapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
|
||||
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
|
||||
assert.equal(snapshot.facts.length, 1);
|
||||
|
||||
const controller = new AbortController();
|
||||
const response = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${snapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
signal: controller.signal,
|
||||
});
|
||||
assert.equal(response.status, 200);
|
||||
const patchPromise = readFirstPatch(response, controller);
|
||||
await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-02", "2026-07-15T12:00:10.000Z", 37.62),
|
||||
});
|
||||
const patch = await patchPromise;
|
||||
assert.equal(validateDataProductPatch(patch).ok, true);
|
||||
assert.equal(patch.previousCursor, snapshot.cursor);
|
||||
|
||||
const forbidden = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${reader.token}`,
|
||||
"X-NODEDC-Tenant-Id": scope.tenantId,
|
||||
"X-NODEDC-Connection-Id": scope.connectionId,
|
||||
},
|
||||
});
|
||||
assert.equal(forbidden.status, 400);
|
||||
|
||||
const revocationSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
|
||||
const revocationStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${revocationSnapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
});
|
||||
assert.equal(revocationStream.status, 200);
|
||||
const revokedEventPromise = readNamedEvent(revocationStream, "error");
|
||||
const rotatedReaderIssuance = await rawJsonRequest(`/internal/data-plane/v1/reader-bindings/${reader.readerBinding.id}/rotate`, {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
});
|
||||
assertOneTimeCapabilityResponse(rotatedReaderIssuance.response);
|
||||
const rotatedReader = rotatedReaderIssuance.value;
|
||||
const revokedEvent = await revokedEventPromise;
|
||||
assert.equal(revokedEvent.error, "reader_stream_access_revoked");
|
||||
const rejectedOldReader = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
});
|
||||
assert.equal(rejectedOldReader.status, 401);
|
||||
const rotatedSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: rotatedReader.token });
|
||||
assert.equal(rotatedSnapshot.dataProduct.id, productId);
|
||||
|
||||
const shutdownStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${rotatedSnapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${rotatedReader.token}` },
|
||||
});
|
||||
assert.equal(shutdownStream.status, 200);
|
||||
child.kill("SIGTERM");
|
||||
assert.equal(await waitForChildExit(child, 3_000), true);
|
||||
|
||||
console.log("external-data-plane API integration: ok");
|
||||
} catch (error) {
|
||||
error.message = `${error.message}\nserver_output=${childOutput.replace(/[A-Za-z0-9_-]{48,}/g, "[redacted]").slice(-4000)}`;
|
||||
throw error;
|
||||
} finally {
|
||||
child.kill("SIGTERM");
|
||||
await Promise.race([
|
||||
new Promise((resolve) => child.once("exit", resolve)),
|
||||
new Promise((resolve) => setTimeout(resolve, 3000)),
|
||||
]);
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function publish(runId, observedAt, longitude) {
|
||||
return {
|
||||
schemaVersion: "nodedc.data-product.publish/v1",
|
||||
batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0` },
|
||||
facts: [{
|
||||
sourceId: "unit-01",
|
||||
semanticType: "map.moving_object",
|
||||
observedAt,
|
||||
attributes: { status: "online" },
|
||||
geometry: { type: "Point", coordinates: [longitude, 55.75] },
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async function jsonRequest(path, { method = "GET", token, body } = {}) {
|
||||
const { response, value } = await rawJsonRequest(path, { method, token, body });
|
||||
if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
async function rawJsonRequest(path, { method = "GET", token, body } = {}) {
|
||||
const response = await fetch(`${baseUrl}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
...(body ? { "Content-Type": "application/json" } : {}),
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
const value = await response.json();
|
||||
if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`);
|
||||
return { response, value };
|
||||
}
|
||||
|
||||
function assertOneTimeCapabilityResponse(response) {
|
||||
assert.equal(response.headers.get("cache-control"), "no-store, max-age=0");
|
||||
assert.equal(response.headers.get("pragma"), "no-cache");
|
||||
assert.equal(response.headers.get("expires"), "0");
|
||||
}
|
||||
|
||||
async function waitForHealth() {
|
||||
const deadline = Date.now() + 15_000;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/healthz`);
|
||||
if (response.ok) return;
|
||||
} catch {}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
throw new Error("server_health_timeout");
|
||||
}
|
||||
|
||||
async function readFirstPatch(response, controller) {
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const timeout = setTimeout(() => controller.abort(), 10_000);
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) throw new Error("stream_closed_before_patch");
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
for (const frame of buffer.split("\n\n")) {
|
||||
const data = frame.split("\n").find((line) => line.startsWith("data: "));
|
||||
if (frame.includes("event: nodedc.data-product.patch.v1") && data) {
|
||||
controller.abort();
|
||||
return JSON.parse(data.slice(6));
|
||||
}
|
||||
}
|
||||
const boundary = buffer.lastIndexOf("\n\n");
|
||||
if (boundary >= 0) buffer = buffer.slice(boundary + 2);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function readNamedEvent(response, eventName) {
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline) {
|
||||
const next = await Promise.race([
|
||||
reader.read(),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("stream_event_timeout")), 10_000)),
|
||||
]);
|
||||
if (next.done) throw new Error(`stream_closed_before_${eventName}`);
|
||||
buffer += decoder.decode(next.value, { stream: true }).replace(/\r\n/g, "\n");
|
||||
let boundary;
|
||||
while ((boundary = buffer.indexOf("\n\n")) !== -1) {
|
||||
const frame = buffer.slice(0, boundary);
|
||||
buffer = buffer.slice(boundary + 2);
|
||||
if (!frame.includes(`event: ${eventName}`)) continue;
|
||||
const data = frame.split("\n").find((line) => line.startsWith("data: "));
|
||||
if (data) return JSON.parse(data.slice(6));
|
||||
}
|
||||
}
|
||||
throw new Error(`stream_event_timeout:${eventName}`);
|
||||
}
|
||||
|
||||
async function waitForChildExit(process, timeoutMs) {
|
||||
if (process.exitCode !== null) return true;
|
||||
return Promise.race([
|
||||
new Promise((resolve) => process.once("exit", () => resolve(true))),
|
||||
new Promise((resolve) => setTimeout(() => resolve(false), timeoutMs)),
|
||||
]);
|
||||
}
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject));
|
||||
const { port } = server.address();
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
return port;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { readConfig } from "../src/config.mjs";
|
||||
|
||||
const base = {
|
||||
EXTERNAL_DATA_PLANE_DATABASE_URL: "postgresql://user:pass@example.invalid/data_plane",
|
||||
NODEDC_INTERNAL_ACCESS_TOKEN: "legacy-internal-token",
|
||||
};
|
||||
|
||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-config-"));
|
||||
const secretPath = join(directory, "provisioner-token");
|
||||
const secret = "provisioner_secret_is_separate_from_legacy_internal_token_123456";
|
||||
try {
|
||||
await writeFile(secretPath, `${secret}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
const config = readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath,
|
||||
EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS: "120",
|
||||
EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000",
|
||||
});
|
||||
assert.equal(config.provisionerAccessToken, secret);
|
||||
assert.equal(config.maxFutureSkewSeconds, 120);
|
||||
assert.equal(config.retentionSweepMs, 60000);
|
||||
assert.equal(config.maxPatchBytes, 262144);
|
||||
assert.equal(config.receiptRetentionMs, 604800000);
|
||||
assert.equal(config.legacyIntakeEnabled, false);
|
||||
assert.equal(readConfig({ ...base, EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED: "true" }).legacyIntakeEnabled, true);
|
||||
assert.throws(() => readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath,
|
||||
NODEDC_INTERNAL_ACCESS_TOKEN: secret,
|
||||
}), /provisioner_token_must_differ_from_internal_token/);
|
||||
assert.throws(() => readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"),
|
||||
}), /external_data_plane_provisioner_token_file_unreadable/);
|
||||
assert.equal(readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "false",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"),
|
||||
}).provisionerAccessToken, "");
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
console.log("external-data-plane config: ok");
|
||||
@@ -0,0 +1,184 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { Pool } from "pg";
|
||||
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract";
|
||||
import {
|
||||
loadDataProductDefinition,
|
||||
persistDataProductDefinition,
|
||||
persistDataProductPublish,
|
||||
readDataProductSnapshot,
|
||||
readPatchEvents,
|
||||
} from "../src/data-product-delivery.mjs";
|
||||
import { normalizeDataProductDefinition } from "../src/data-product-policy.mjs";
|
||||
import { migrate } from "../src/schema.mjs";
|
||||
|
||||
const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL;
|
||||
if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required");
|
||||
const pool = new Pool({ connectionString: databaseUrl, max: 4 });
|
||||
|
||||
try {
|
||||
await migrate(pool);
|
||||
await pool.query(`truncate table
|
||||
external_data_plane_patch_outbox,
|
||||
external_data_plane_history,
|
||||
external_data_plane_current,
|
||||
external_data_plane_facts,
|
||||
external_data_plane_raw_envelopes,
|
||||
external_data_plane_batches,
|
||||
external_data_plane_delivery_state,
|
||||
external_data_plane_reader_bindings,
|
||||
external_data_plane_writer_bindings,
|
||||
external_data_plane_products
|
||||
restart identity cascade`);
|
||||
|
||||
const definition = normalizeDataProductDefinition({
|
||||
id: "test.positions.current.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.test.positions.v1",
|
||||
deliveryMode: "snapshot+patch",
|
||||
semanticTypes: ["map.moving_object"],
|
||||
fields: ["source_id", "observed_at", "geometry", "status"],
|
||||
history: {
|
||||
mode: "sampled",
|
||||
intervalMs: 60_000,
|
||||
strategy: "latest-per-entity-per-bucket",
|
||||
retentionDays: 30,
|
||||
},
|
||||
});
|
||||
await persistDataProductDefinition(pool, definition);
|
||||
const storedDefinition = await loadDataProductDefinition(pool, definition.id);
|
||||
const binding = {
|
||||
id: "reader-binding-test",
|
||||
tenantId: "tenant-test",
|
||||
connectionId: "connection-test",
|
||||
providerId: "provider-test",
|
||||
allowedDataProductIds: [definition.id],
|
||||
active: true,
|
||||
expiresAt: "2027-01-01T00:00:00.000Z",
|
||||
};
|
||||
|
||||
const firstBatch = batch("run-01", "2026-07-15T10:00:05.000Z", [
|
||||
fact("unit-01", 37.61, 55.75, "online"),
|
||||
fact("unit-02", 37.62, 55.76, "online"),
|
||||
]);
|
||||
const first = await persistDataProductPublish(pool, firstBatch, storedDefinition);
|
||||
assert.equal(first.idempotent, false);
|
||||
assert.equal(first.currentUpdatedCount, 2);
|
||||
assert.equal(first.historyInsertedCount, 2);
|
||||
assert.equal(first.cursor, "1");
|
||||
|
||||
const duplicate = await persistDataProductPublish(pool, firstBatch, storedDefinition);
|
||||
assert.equal(duplicate.idempotent, true);
|
||||
assert.equal(duplicate.cursor, "1");
|
||||
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, {
|
||||
...firstBatch,
|
||||
facts: [fact("unit-01", 99.9, 55.75, "changed")],
|
||||
}, storedDefinition),
|
||||
(error) => error?.status === 409 && error?.code === "idempotency_key_reused",
|
||||
);
|
||||
|
||||
const concurrentBatch = batch("run-concurrent", "2026-07-15T10:00:08.000Z", [
|
||||
fact("unit-03", 37.63, 55.77, "online", "2026-07-15T10:00:08.000Z"),
|
||||
]);
|
||||
const concurrent = await Promise.all([
|
||||
persistDataProductPublish(pool, concurrentBatch, storedDefinition),
|
||||
persistDataProductPublish(pool, concurrentBatch, storedDefinition),
|
||||
]);
|
||||
assert.deepEqual(concurrent.map((value) => value.idempotent).sort(), [false, true]);
|
||||
|
||||
const second = await persistDataProductPublish(pool, batch(
|
||||
"run-02",
|
||||
"2026-07-15T10:00:15.000Z",
|
||||
[fact("unit-01", 37.611, 55.751, "moving", "2026-07-15T10:00:15.000Z")],
|
||||
), storedDefinition);
|
||||
assert.equal(second.currentUpdatedCount, 1);
|
||||
assert.equal(second.cursor, "3");
|
||||
|
||||
const latest = await persistDataProductPublish(pool, batch(
|
||||
"run-latest",
|
||||
"2026-07-15T10:02:05.000Z",
|
||||
[fact("unit-01", 37.7, 55.8, "latest", "2026-07-15T10:02:00.000Z")],
|
||||
), storedDefinition);
|
||||
assert.equal(latest.currentUpdatedCount, 1);
|
||||
const late = await persistDataProductPublish(pool, batch(
|
||||
"run-late",
|
||||
"2026-07-15T10:03:00.000Z",
|
||||
[fact("unit-01", 37.65, 55.78, "late", "2026-07-15T10:01:30.000Z")],
|
||||
), storedDefinition);
|
||||
assert.equal(late.currentUpdatedCount, 0);
|
||||
assert.equal(late.historyInsertedCount, 1);
|
||||
assert.equal(late.patchOperationCount, 0);
|
||||
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, batch("run-bad-type", "2026-07-15T10:03:10.000Z", [{
|
||||
...fact("unit-04", 37.6, 55.7, "online"),
|
||||
semanticType: "map.forbidden",
|
||||
}]), storedDefinition),
|
||||
(error) => error?.status === 422 && error?.code === "data_product_semantic_type_forbidden",
|
||||
);
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, batch("run-bad-field", "2026-07-15T10:03:11.000Z", [{
|
||||
...fact("unit-04", 37.6, 55.7, "online"),
|
||||
attributes: { undeclared: true },
|
||||
}]), storedDefinition),
|
||||
(error) => error?.status === 422 && error?.code === "data_product_field_forbidden",
|
||||
);
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, batch("run-duplicate-fact", "2026-07-15T10:03:12.000Z", [
|
||||
fact("unit-04", 37.6, 55.7, "online"),
|
||||
fact("unit-04", 37.61, 55.71, "moving"),
|
||||
]), storedDefinition),
|
||||
(error) => error?.status === 422 && error?.code === "data_product_duplicate_entity_key",
|
||||
);
|
||||
|
||||
const snapshot = await readDataProductSnapshot(pool, binding, storedDefinition);
|
||||
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
|
||||
assert.equal(snapshot.cursor, "4");
|
||||
assert.equal(snapshot.facts.length, 3);
|
||||
assert.equal(snapshot.facts.find((value) => value.sourceId === "unit-01").attributes.status, "latest");
|
||||
|
||||
const patches = await readPatchEvents(pool, binding, storedDefinition, 0n);
|
||||
assert.equal(patches.length, 4);
|
||||
assert.equal(patches.every((patch) => validateDataProductPatch(patch).ok), true);
|
||||
assert.deepEqual(patches.map((patch) => patch.cursor), ["1", "2", "3", "4"]);
|
||||
|
||||
await assert.rejects(
|
||||
readPatchEvents(pool, binding, storedDefinition, 999n),
|
||||
(error) => error?.status === 409 && error?.code === "resync_required",
|
||||
);
|
||||
|
||||
const history = await pool.query("select source_id, observed_at from external_data_plane_history order by source_id");
|
||||
assert.equal(history.rowCount, 5);
|
||||
const unitOneHistory = history.rows.filter((row) => row.source_id === "unit-01");
|
||||
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:01:30.000Z"), true);
|
||||
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:02:00.000Z"), true);
|
||||
|
||||
console.log("external-data-plane delivery integration: ok");
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
function batch(runId, receivedAt, facts) {
|
||||
return {
|
||||
schemaVersion: "nodedc.external-provider-contract/v1",
|
||||
source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" },
|
||||
contract: {
|
||||
dataProductId: "test.positions.current.v1",
|
||||
ontologyRevision: "ontology.test.positions.v1",
|
||||
version: "1.0.0",
|
||||
},
|
||||
batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0`, receivedAt },
|
||||
facts,
|
||||
};
|
||||
}
|
||||
|
||||
function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10:00:00.000Z") {
|
||||
return {
|
||||
sourceId,
|
||||
semanticType: "map.moving_object",
|
||||
observedAt,
|
||||
attributes: { status },
|
||||
geometry: { type: "Point", coordinates: [longitude, latitude] },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { normalizeDataProductDefinition } from "../src/data-product-policy.mjs";
|
||||
|
||||
const input = {
|
||||
id: "fleet.positions.current.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.map.v1",
|
||||
deliveryMode: "snapshot+patch",
|
||||
semanticTypes: ["vehicle.trike", "map.moving_object"],
|
||||
fields: ["status", "source_id", "observed_at", "geometry"],
|
||||
history: {
|
||||
mode: "sampled",
|
||||
intervalMs: 60_000,
|
||||
strategy: "latest-per-entity-per-bucket",
|
||||
retentionDays: 365,
|
||||
},
|
||||
};
|
||||
const definition = normalizeDataProductDefinition(input);
|
||||
assert.equal(definition.history.mode, "sampled");
|
||||
assert.equal(definition.history.intervalMs, 60_000);
|
||||
assert.deepEqual(definition.semanticTypes, ["map.moving_object", "vehicle.trike"]);
|
||||
assert.deepEqual(definition.fields, ["geometry", "observed_at", "source_id", "status"]);
|
||||
assert.deepEqual(input.semanticTypes, ["vehicle.trike", "map.moving_object"]);
|
||||
assert.deepEqual(input.fields, ["status", "source_id", "observed_at", "geometry"]);
|
||||
assert.equal(Object.isFrozen(definition.semanticTypes), true);
|
||||
assert.equal(Object.isFrozen(definition.fields), true);
|
||||
|
||||
const reordered = normalizeDataProductDefinition({
|
||||
...input,
|
||||
semanticTypes: [...input.semanticTypes].reverse(),
|
||||
fields: [...input.fields].reverse(),
|
||||
});
|
||||
assert.deepEqual(reordered.semanticTypes, definition.semanticTypes);
|
||||
assert.deepEqual(reordered.fields, definition.fields);
|
||||
|
||||
assert.throws(() => normalizeDataProductDefinition({ ...definition, providerId: "gelios" }), /data_product_definition_invalid/);
|
||||
assert.throws(() => normalizeDataProductDefinition({ ...definition, history: { mode: "none", intervalMs: 1000 } }), /history_policy_none_has_sampling_fields/);
|
||||
|
||||
for (const semanticTypes of [
|
||||
undefined,
|
||||
"map.moving_object",
|
||||
[],
|
||||
["map.moving_object", null],
|
||||
["map.moving_object", "Map.invalid"],
|
||||
]) {
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, semanticTypes }),
|
||||
/data_product_definition_(semantic_types_invalid|shape_invalid)/,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, semanticTypes: ["map.moving_object", "map.moving_object"] }),
|
||||
/data_product_definition_semantic_types_duplicate/,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, semanticTypes: ["map.moving_object", " map.moving_object "] }),
|
||||
/data_product_definition_semantic_types_duplicate/,
|
||||
);
|
||||
|
||||
for (const fields of [
|
||||
undefined,
|
||||
"source_id",
|
||||
[],
|
||||
["source_id", null],
|
||||
["source_id", "Invalid"],
|
||||
]) {
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, fields }),
|
||||
/data_product_definition_(fields_invalid|shape_invalid)/,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, fields: ["source_id", "source_id"] }),
|
||||
/data_product_definition_fields_duplicate/,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, fields: ["source_id", " source_id "] }),
|
||||
/data_product_definition_fields_duplicate/,
|
||||
);
|
||||
|
||||
console.log("external-data-plane data product policy: ok");
|
||||
@@ -0,0 +1,32 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { loadDataProductDefinitions } from "../src/definitions.mjs";
|
||||
|
||||
const bundled = await loadDataProductDefinitions();
|
||||
assert.deepEqual(bundled.map((definition) => definition.id), ["fleet.positions.current.v1"]);
|
||||
assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]);
|
||||
assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1");
|
||||
assert.equal(bundled[0].history.mode, "sampled");
|
||||
assert.equal(bundled[0].history.intervalMs, 60_000);
|
||||
assert.equal(bundled[0].history.retentionDays, 90);
|
||||
assert.equal(bundled[0].fields.includes("geometry"), true);
|
||||
assert.equal(bundled[0].fields.includes("providerUnitId"), false);
|
||||
assert.equal(bundled[0].fields.includes("provider_unit_id"), false);
|
||||
|
||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-"));
|
||||
try {
|
||||
await writeFile(join(directory, "wrong-name.json"), JSON.stringify({
|
||||
...bundled[0],
|
||||
id: "another.product.v1",
|
||||
}));
|
||||
await assert.rejects(loadDataProductDefinitions(directory), /data_product_definition_filename_mismatch/);
|
||||
await mkdir(join(directory, "ignored.json"));
|
||||
await assert.rejects(loadDataProductDefinitions(directory), /data_product_definition_file_invalid/);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
console.log("external-data-plane definitions: ok");
|
||||
@@ -0,0 +1,26 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { assertBatchTimeBounds, rawRetentionExpiry } from "../src/intake-policy.mjs";
|
||||
|
||||
const now = new Date("2026-07-15T12:00:00.000Z");
|
||||
const batch = {
|
||||
batch: { receivedAt: "2026-07-15T12:04:59.000Z" },
|
||||
facts: [{ observedAt: "2026-07-15T12:05:00.000Z" }],
|
||||
};
|
||||
|
||||
assert.doesNotThrow(() => assertBatchTimeBounds(batch, { now, maxFutureSkewSeconds: 300 }));
|
||||
assert.throws(() => assertBatchTimeBounds({
|
||||
...batch,
|
||||
batch: { receivedAt: "2026-07-15T12:05:01.000Z" },
|
||||
}, { now, maxFutureSkewSeconds: 300 }), /batch_received_at_too_far_in_future/);
|
||||
assert.throws(() => assertBatchTimeBounds({
|
||||
...batch,
|
||||
facts: [{ observedAt: "2026-07-15T12:05:01.000Z" }],
|
||||
}, { now, maxFutureSkewSeconds: 300 }), /fact_observed_at_too_far_in_future/);
|
||||
|
||||
assert.equal(
|
||||
rawRetentionExpiry({ now, rawRetentionDays: 14 }).toISOString(),
|
||||
"2026-07-29T12:00:00.000Z",
|
||||
);
|
||||
assert.throws(() => rawRetentionExpiry({ now, rawRetentionDays: 0 }), /raw_retention_days_invalid/);
|
||||
|
||||
console.log("external-data-plane intake policy: ok");
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
assertReaderProduct,
|
||||
createReaderToken,
|
||||
hashReaderToken,
|
||||
normalizeReaderBindingRequest,
|
||||
safeReaderBinding,
|
||||
} from "../src/reader-binding.mjs";
|
||||
|
||||
const now = new Date("2026-07-15T12:00:00.000Z");
|
||||
const policy = normalizeReaderBindingRequest({
|
||||
source: { tenantId: "tenant-01", connectionId: "connection-01", providerId: "example-provider" },
|
||||
allowedDataProductIds: ["fleet.positions.current.v1"],
|
||||
expiresAt: "2026-08-01T12:00:00.000Z",
|
||||
}, { now });
|
||||
const token = createReaderToken();
|
||||
assert.match(token, /^ndc_edprb_[A-Za-z0-9_-]{40,}$/);
|
||||
assert.equal(hashReaderToken(token), hashReaderToken(token));
|
||||
assert.equal(assertReaderProduct({ ...policy, active: true }, "fleet.positions.current.v1", now), "fleet.positions.current.v1");
|
||||
assert.throws(() => assertReaderProduct({ ...policy, active: true }, "other.product.v1", now), /reader_binding_data_product_forbidden/);
|
||||
const safe = safeReaderBinding({ id: "reader-01", ...policy, active: true, token, tokenHash: hashReaderToken(token) });
|
||||
assert.equal("token" in safe, false);
|
||||
assert.equal("tokenHash" in safe, false);
|
||||
|
||||
console.log("external-data-plane reader bindings: ok");
|
||||
@@ -0,0 +1,117 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { validateIntakeBatch } from "../../../packages/external-provider-contract/src/index.mjs";
|
||||
import {
|
||||
createWriterToken,
|
||||
hashWriterToken,
|
||||
materializeDataProductPublish,
|
||||
materializeWriterBoundBatch,
|
||||
normalizeWriterBindingRequest,
|
||||
safeWriterBinding,
|
||||
} from "../src/writer-binding.mjs";
|
||||
|
||||
const now = new Date("2026-07-15T12:00:00.000Z");
|
||||
const request = {
|
||||
source: {
|
||||
tenantId: "tenant-01",
|
||||
connectionId: "connection-01",
|
||||
providerId: "example-provider",
|
||||
},
|
||||
allowedDataProductIds: ["fleet.positions.current.v1"],
|
||||
expiresAt: "2026-08-01T12:00:00.000Z",
|
||||
};
|
||||
|
||||
const bindingPolicy = normalizeWriterBindingRequest(request, { now, maxTtlDays: 90 });
|
||||
assert.deepEqual(bindingPolicy, {
|
||||
tenantId: "tenant-01",
|
||||
connectionId: "connection-01",
|
||||
providerId: "example-provider",
|
||||
allowedDataProductIds: ["fleet.positions.current.v1"],
|
||||
expiresAt: "2026-08-01T12:00:00.000Z",
|
||||
});
|
||||
|
||||
const token = createWriterToken();
|
||||
assert.match(token, /^ndc_edpwb_[A-Za-z0-9_-]{40,}$/);
|
||||
assert.equal(hashWriterToken(token), hashWriterToken(token));
|
||||
assert.notEqual(hashWriterToken(token), hashWriterToken(`${token}x`));
|
||||
|
||||
const unscopedBatch = {
|
||||
schemaVersion: "nodedc.external-provider-contract/v1",
|
||||
source: { providerId: "example-provider" },
|
||||
contract: { dataProductId: "fleet.positions.current.v1", ontologyRevision: "example.v1", version: "1.0.0" },
|
||||
batch: { runId: "run-01", sequence: 0, idempotencyKey: "run-01.batch-0", receivedAt: "2026-07-15T12:00:00.000Z" },
|
||||
facts: [{ sourceId: "unit-01", semanticType: "map.moving_object", observedAt: "2026-07-15T12:00:00.000Z" }],
|
||||
};
|
||||
const bound = materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: true }, { now });
|
||||
assert.deepEqual(bound.source, { providerId: "example-provider", tenantId: "tenant-01", connectionId: "connection-01" });
|
||||
assert.equal(validateIntakeBatch(bound).ok, true);
|
||||
assert.throws(() => materializeWriterBoundBatch({
|
||||
...unscopedBatch,
|
||||
source: { ...unscopedBatch.source, tenantId: "forged-tenant" },
|
||||
}, { ...bindingPolicy, active: true }, { now }), /writer_bound_scope_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: true }, {
|
||||
now,
|
||||
hasScopeHeaders: true,
|
||||
}), /writer_bound_scope_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch(unscopedBatch, {
|
||||
...bindingPolicy,
|
||||
providerId: "other-provider",
|
||||
active: true,
|
||||
}, { now }), /writer_binding_provider_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch({
|
||||
...unscopedBatch,
|
||||
contract: { ...unscopedBatch.contract, dataProductId: "other.product.v1" },
|
||||
}, { ...bindingPolicy, active: true }, { now }), /writer_binding_data_product_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: false }, { now }), /writer_binding_inactive/);
|
||||
|
||||
const materializedPublish = materializeDataProductPublish({
|
||||
schemaVersion: "nodedc.data-product.publish/v1",
|
||||
batch: { runId: "run-02", sequence: 0, idempotencyKey: "run-02.batch-0" },
|
||||
facts: unscopedBatch.facts,
|
||||
}, { ...bindingPolicy, active: true }, {
|
||||
id: "fleet.positions.current.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.example-fleet.v1",
|
||||
}, "fleet.positions.current.v1", { now });
|
||||
assert.deepEqual(materializedPublish.source, {
|
||||
tenantId: "tenant-01",
|
||||
connectionId: "connection-01",
|
||||
providerId: "example-provider",
|
||||
});
|
||||
assert.equal(materializedPublish.batch.receivedAt, now.toISOString());
|
||||
assert.equal(materializedPublish.contract.version, "1.0.0");
|
||||
assert.equal(validateIntakeBatch(materializedPublish).ok, true);
|
||||
assert.throws(() => materializeDataProductPublish({
|
||||
schemaVersion: "nodedc.data-product.publish/v1",
|
||||
batch: { runId: "run-02", sequence: 0, idempotencyKey: "run-02.batch-0" },
|
||||
facts: unscopedBatch.facts,
|
||||
}, { ...bindingPolicy, active: true }, {
|
||||
id: "other.product.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.example-fleet.v1",
|
||||
}, "other.product.v1", { now }), /writer_binding_data_product_forbidden/);
|
||||
|
||||
assert.throws(() => normalizeWriterBindingRequest({
|
||||
...request,
|
||||
expiresAt: "2027-01-01T00:00:00.000Z",
|
||||
}, { now, maxTtlDays: 90 }), /writer_binding_expiry_invalid/);
|
||||
assert.throws(() => normalizeWriterBindingRequest({
|
||||
...request,
|
||||
apiToken: "must-never-be-here",
|
||||
}, { now, maxTtlDays: 90 }), /writer_binding_request_secret_material_forbidden/);
|
||||
assert.throws(() => normalizeWriterBindingRequest({
|
||||
...request,
|
||||
arbitraryRuntimeSetting: "must-not-be-accepted",
|
||||
}, { now, maxTtlDays: 90 }), /writer_binding_request_fields_invalid/);
|
||||
|
||||
const safeBinding = safeWriterBinding({
|
||||
id: "binding-01",
|
||||
...bindingPolicy,
|
||||
active: true,
|
||||
createdAt: now,
|
||||
token: "must-not-leak",
|
||||
tokenHash: "must-not-leak",
|
||||
});
|
||||
assert.equal("token" in safeBinding, false);
|
||||
assert.equal("tokenHash" in safeBinding, false);
|
||||
|
||||
console.log("external-data-plane writer bindings: ok");
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM node:20-alpine AS deps
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci --omit=dev
|
||||
|
||||
FROM node:20-alpine AS runner
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=18105
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=deps /app/package.json /app/package-lock.json ./
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY src ./src
|
||||
|
||||
USER node
|
||||
|
||||
EXPOSE 18105
|
||||
|
||||
CMD ["node", "src/server.mjs"]
|
||||
@@ -0,0 +1,70 @@
|
||||
# NODE.DC Gelios Gateway
|
||||
|
||||
Storage and read-model service for the Gelios provider domain. It has no
|
||||
provider credential, no provider HTTP client, no scheduler and no command
|
||||
transport.
|
||||
|
||||
## Boundary
|
||||
|
||||
The protected Engine L2 Gelios Collector is the only component that reaches
|
||||
the provider and uses the Engine Credential. It submits authenticated unit
|
||||
intake to this Gateway. The Gateway enforces the connection scope and approved
|
||||
unit allowlist, persists current state/history, and exposes scoped read/SSE
|
||||
contracts to other Engine workflows and map consumers.
|
||||
|
||||
Neither a provider token nor a credential reference is accepted, stored or
|
||||
returned by this service.
|
||||
|
||||
## Runtime contracts
|
||||
|
||||
`GET /healthz` is unprotected and contains no secret material.
|
||||
|
||||
All `/internal/gelios/v1/*` routes require the internal service credential:
|
||||
|
||||
```text
|
||||
Authorization: Bearer <NODEDC_INTERNAL_ACCESS_TOKEN>
|
||||
X-NODEDC-Tenant-Id: <approved-tenant-id>
|
||||
X-NODEDC-Connection-Id: <approved-connection-id>
|
||||
```
|
||||
|
||||
```text
|
||||
POST /internal/gelios/v1/intake/units
|
||||
GET /internal/gelios/v1/status
|
||||
GET /internal/gelios/v1/capabilities
|
||||
GET /internal/gelios/v1/units/current
|
||||
GET /internal/gelios/v1/stream
|
||||
GET /internal/gelios/v1/data-products/fleet.positions.current.v1/snapshot
|
||||
GET /internal/gelios/v1/data-products/fleet.positions.current.v1/stream
|
||||
```
|
||||
|
||||
`intake/units` is disabled by default. It accepts a bounded payload with a
|
||||
`units` array or an equivalent safe unit-list envelope; it does not accept a
|
||||
URL, method, token or command. The command catalog remains visible only as
|
||||
disabled ontology metadata.
|
||||
|
||||
The connection policy is explicit: `GELIOS_UNIT_SCOPE=allowlist` uses fixed
|
||||
`GELIOS_ALLOWED_UNIT_IDS`; `GELIOS_UNIT_SCOPE=all` accepts every unit returned
|
||||
by that already-approved tenant + connection, including units added later by
|
||||
the provider. Missing units are never deleted from storage: their stable
|
||||
provider IDs and last-seen history remain available to the presentation layer.
|
||||
|
||||
`fleet.positions.current.v1` is the first provider-neutral product. Its
|
||||
snapshot and SSE patch stream contain semantic moving objects, position,
|
||||
motion, quality and `active` / `stale` / `no-position` state. They deliberately
|
||||
exclude raw telemetry and provider transport payloads. A consumer first fetches
|
||||
the snapshot and then attaches the stream; the stream `ready` event never
|
||||
pretends to be a complete current-state snapshot.
|
||||
|
||||
## Storage
|
||||
|
||||
The database requires TimescaleDB and PostGIS. It keeps a current projection,
|
||||
append-only time-series snapshots, bounded restricted intake envelopes and
|
||||
collection audit. Raw envelopes have no browser/API route and expire according
|
||||
to `GELIOS_RAW_RETENTION_DAYS`.
|
||||
|
||||
## Local checks
|
||||
|
||||
```bash
|
||||
npm ci
|
||||
npm run smoke
|
||||
```
|
||||
+1008
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "@nodedc/gelios-gateway",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/server.mjs",
|
||||
"dev": "node --watch src/server.mjs",
|
||||
"smoke": "node src/scripts/smoke.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^5.2.1",
|
||||
"pg": "^8.18.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
export const GELIOS_CAPABILITIES = Object.freeze([
|
||||
{
|
||||
id: "gelios.units.current_intake",
|
||||
class: "internal-safe-intake",
|
||||
enabled: true,
|
||||
summary: "Authenticated normalized current-unit intake from the protected Engine Collector.",
|
||||
},
|
||||
{
|
||||
id: "fleet.positions.current.v1",
|
||||
class: "internal-data-product",
|
||||
enabled: true,
|
||||
summary: "Provider-neutral current fleet positions snapshot and patch stream; no raw telemetry fields.",
|
||||
},
|
||||
{
|
||||
id: "gelios.commands.catalog",
|
||||
class: "red-read-catalogue",
|
||||
enabled: false,
|
||||
summary: "Catalogued for ontology only; not connected to the collector or Gateway.",
|
||||
},
|
||||
{
|
||||
id: "gelios.commands.send",
|
||||
class: "red-write",
|
||||
enabled: false,
|
||||
summary: "No transport is implemented.",
|
||||
},
|
||||
]);
|
||||
@@ -0,0 +1,80 @@
|
||||
export function readConfig(env = process.env) {
|
||||
const rawRetentionDays = integer(env.GELIOS_RAW_RETENTION_DAYS, 14, 1, 3650);
|
||||
|
||||
return {
|
||||
nodeEnv: string(env.NODE_ENV, "development"),
|
||||
port: integer(env.PORT, 18105, 1, 65535),
|
||||
databaseUrl: required(env.DATABASE_URL, "DATABASE_URL"),
|
||||
databasePoolSize: integer(env.GELIOS_DATABASE_POOL_SIZE, 10, 1, 50),
|
||||
internalAccessToken: optional(env.NODEDC_INTERNAL_ACCESS_TOKEN),
|
||||
tenantId: slug(required(env.GELIOS_TENANT_ID, "GELIOS_TENANT_ID"), "GELIOS_TENANT_ID"),
|
||||
connectionId: slug(required(env.GELIOS_CONNECTION_ID, "GELIOS_CONNECTION_ID"), "GELIOS_CONNECTION_ID"),
|
||||
unitScope: collectionScope(env.GELIOS_UNIT_SCOPE),
|
||||
allowedUnitIds: intList(env.GELIOS_ALLOWED_UNIT_IDS),
|
||||
intakeEnabled: boolean(env.GELIOS_INTAKE_ENABLED, false),
|
||||
rawRetentionDays,
|
||||
positionStaleAfterMs: integer(env.GELIOS_POSITION_STALE_AFTER_MS, 300_000, 1_000, 86_400_000),
|
||||
};
|
||||
}
|
||||
|
||||
function collectionScope(value) {
|
||||
const candidate = optional(value).toLowerCase() || "allowlist";
|
||||
if (["allowlist", "all"].includes(candidate)) return candidate;
|
||||
throw new Error("GELIOS_UNIT_SCOPE_must_be_allowlist_or_all");
|
||||
}
|
||||
|
||||
export function unitIsInScope(config, sourceUnitId) {
|
||||
return config?.unitScope === "all" || config?.allowedUnitIds?.includes(sourceUnitId) === true;
|
||||
}
|
||||
|
||||
function required(value, name) {
|
||||
const normalized = optional(value);
|
||||
if (!normalized) throw new Error(`${name}_required`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function optional(value) {
|
||||
const normalized = String(value ?? "").trim();
|
||||
return normalized || "";
|
||||
}
|
||||
|
||||
function string(value, fallback) {
|
||||
return optional(value) || fallback;
|
||||
}
|
||||
|
||||
function integer(value, fallback, min, max) {
|
||||
const candidate = optional(value);
|
||||
if (!candidate) return fallback;
|
||||
const number = Number.parseInt(candidate, 10);
|
||||
if (!Number.isInteger(number) || number < min || number > max) {
|
||||
throw new Error(`invalid_integer:${candidate}`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function boolean(value, fallback) {
|
||||
const candidate = optional(value).toLowerCase();
|
||||
if (!candidate) return fallback;
|
||||
if (["1", "true", "yes", "on"].includes(candidate)) return true;
|
||||
if (["0", "false", "no", "off"].includes(candidate)) return false;
|
||||
throw new Error(`invalid_boolean:${candidate}`);
|
||||
}
|
||||
|
||||
function intList(value) {
|
||||
const values = optional(value)
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean)
|
||||
.map((item) => Number.parseInt(item, 10));
|
||||
if (values.some((item) => !Number.isInteger(item) || item <= 0)) {
|
||||
throw new Error("GELIOS_ALLOWED_UNIT_IDS_must_contain_positive_integers");
|
||||
}
|
||||
return [...new Set(values)].sort((left, right) => left - right);
|
||||
}
|
||||
|
||||
function slug(value, name) {
|
||||
if (!/^[a-z0-9][a-z0-9_-]{0,95}$/i.test(value)) {
|
||||
throw new Error(`${name}_invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
export const FLEET_POSITIONS_CURRENT = Object.freeze({
|
||||
id: "fleet.positions.current.v1",
|
||||
version: "1.0.0",
|
||||
delivery: "snapshot+patch",
|
||||
semanticTypes: Object.freeze(["map.moving_object", "geo.position"]),
|
||||
});
|
||||
|
||||
export function toFleetPositionsSnapshot(rows, { tenantId, connectionId, generatedAt = new Date(), staleAfterMs }) {
|
||||
return {
|
||||
schemaVersion: "nodedc.data-product.snapshot.v1",
|
||||
dataProduct: FLEET_POSITIONS_CURRENT,
|
||||
scope: { tenantId, connectionId },
|
||||
generatedAt: iso(generatedAt),
|
||||
entities: rows.map((row) => toFleetPositionEntity(row, { generatedAt, staleAfterMs })),
|
||||
};
|
||||
}
|
||||
|
||||
export function toFleetPositionPatch(row, { generatedAt = new Date(), staleAfterMs }) {
|
||||
return {
|
||||
schemaVersion: "nodedc.data-product.patch.v1",
|
||||
dataProductId: FLEET_POSITIONS_CURRENT.id,
|
||||
operation: "upsert",
|
||||
emittedAt: iso(generatedAt),
|
||||
entity: toFleetPositionEntity(row, { generatedAt, staleAfterMs }),
|
||||
};
|
||||
}
|
||||
|
||||
export function toFleetPositionEntity(row, { generatedAt, staleAfterMs }) {
|
||||
const observedAt = date(row.observedAt);
|
||||
const receivedAt = date(row.receivedAt);
|
||||
const latitude = finiteNumber(row.latitude);
|
||||
const longitude = finiteNumber(row.longitude);
|
||||
const hasPosition = latitude !== null && longitude !== null;
|
||||
const ageMs = observedAt ? Math.max(0, generatedAt.getTime() - observedAt.getTime()) : null;
|
||||
const status = !hasPosition ? "no-position" : ageMs !== null && ageMs > staleAfterMs ? "stale" : "active";
|
||||
|
||||
return {
|
||||
subjectId: String(row.subjectId),
|
||||
semanticType: "map.moving_object",
|
||||
label: String(row.name || row.subjectId),
|
||||
status,
|
||||
observedAt: observedAt ? observedAt.toISOString() : null,
|
||||
receivedAt: receivedAt ? receivedAt.toISOString() : null,
|
||||
position: hasPosition ? { latitude, longitude } : null,
|
||||
motion: {
|
||||
speed: finiteNumber(row.speed),
|
||||
course: finiteNumber(row.course),
|
||||
},
|
||||
quality: {
|
||||
gpsValid: booleanOrNull(row.gpsValid),
|
||||
satellites: finiteNumber(row.satellites),
|
||||
hdop: finiteNumber(row.hdop),
|
||||
accuracyM: finiteNumber(row.accuracyM),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function iso(value) {
|
||||
const parsed = date(value);
|
||||
if (!parsed) throw new Error("data_product_timestamp_invalid");
|
||||
return parsed.toISOString();
|
||||
}
|
||||
|
||||
function date(value) {
|
||||
if (value instanceof Date && !Number.isNaN(value.getTime())) return value;
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
|
||||
function finiteNumber(value) {
|
||||
if (value === null || value === undefined || value === "") return null;
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function booleanOrNull(value) {
|
||||
if (value === true || value === 1 || value === "1" || value === "true") return true;
|
||||
if (value === false || value === 0 || value === "0" || value === "false") return false;
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export function normalizeUnit(unit, { tenantId, connectionId, receivedAt = new Date() }) {
|
||||
const sourceUnitId = positiveInteger(first(unit?.id, unit?.unit_id, unit?.unitId));
|
||||
if (!sourceUnitId) return null;
|
||||
|
||||
const lastMessage = firstObject(unit?.lmsg, unit?.last_msg, unit?.lastMsg, unit?.lastMessage) || {};
|
||||
const latitude = finiteNumber(first(lastMessage.lat, lastMessage.latitude, unit?.lat, unit?.latitude));
|
||||
const longitude = finiteNumber(first(lastMessage.lon, lastMessage.lng, lastMessage.longitude, unit?.lon, unit?.lng, unit?.longitude));
|
||||
const observedAt = timestamp(first(lastMessage.time, lastMessage.ts, lastMessage.timestamp, unit?.last_msg_time, unit?.lmsg_time), receivedAt);
|
||||
const telemetry = {
|
||||
sourceUnitId,
|
||||
name: optionalString(first(unit?.name, unit?.unit_name)),
|
||||
observedAt: observedAt.toISOString(),
|
||||
latitude,
|
||||
longitude,
|
||||
speed: finiteNumber(first(lastMessage.speed, unit?.speed)),
|
||||
course: finiteNumber(first(lastMessage.course, lastMessage.angle, unit?.course)),
|
||||
gpsValid: booleanOrNull(first(lastMessage.gps_valid, lastMessage.gpsValid, unit?.gps_valid)),
|
||||
satellites: finiteNumber(first(lastMessage.sats, lastMessage.satellites, unit?.sats)),
|
||||
hdop: finiteNumber(first(lastMessage.hdop, unit?.hdop)),
|
||||
accuracyM: finiteNumber(first(lastMessage.accuracy, lastMessage.accuracy_m, unit?.accuracy_m)),
|
||||
};
|
||||
|
||||
return {
|
||||
tenantId,
|
||||
connectionId,
|
||||
sourceUnitId,
|
||||
stableSubjectId: `gelios.unit:${connectionId}:${sourceUnitId}`,
|
||||
name: telemetry.name || `Gelios unit ${sourceUnitId}`,
|
||||
observedAt,
|
||||
receivedAt,
|
||||
latitude,
|
||||
longitude,
|
||||
speed: telemetry.speed,
|
||||
course: telemetry.course,
|
||||
gpsValid: telemetry.gpsValid,
|
||||
satellites: telemetry.satellites,
|
||||
hdop: telemetry.hdop,
|
||||
accuracyM: telemetry.accuracyM,
|
||||
telemetry,
|
||||
fingerprint: hash(telemetry),
|
||||
};
|
||||
}
|
||||
|
||||
export function extractUnitItems(payload) {
|
||||
if (Array.isArray(payload?.items)) return payload.items;
|
||||
if (Array.isArray(payload?.units)) return payload.units;
|
||||
if (Array.isArray(payload?.data)) return payload.data;
|
||||
return [];
|
||||
}
|
||||
|
||||
export function hash(value) {
|
||||
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
||||
}
|
||||
|
||||
function first(...values) {
|
||||
return values.find((value) => value !== undefined && value !== null && value !== "");
|
||||
}
|
||||
|
||||
function firstObject(...values) {
|
||||
return values.find((value) => value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function positiveInteger(value) {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
|
||||
function finiteNumber(value) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function optionalString(value) {
|
||||
const normalized = String(value ?? "").trim();
|
||||
return normalized || null;
|
||||
}
|
||||
|
||||
function booleanOrNull(value) {
|
||||
if (value === true || value === 1 || value === "1" || value === "true") return true;
|
||||
if (value === false || value === 0 || value === "0" || value === "false") return false;
|
||||
return null;
|
||||
}
|
||||
|
||||
function timestamp(value, fallback) {
|
||||
const numeric = Number(value);
|
||||
if (!Number.isFinite(numeric) || numeric <= 0) return fallback;
|
||||
const milliseconds = numeric < 1_000_000_000_000 ? numeric * 1000 : numeric;
|
||||
const parsed = new Date(milliseconds);
|
||||
return Number.isNaN(parsed.getTime()) ? fallback : parsed;
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
export async function migrate(pool) {
|
||||
await pool.query("create extension if not exists timescaledb");
|
||||
await pool.query("create extension if not exists postgis");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists gelios_connections (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null default 'gelios',
|
||||
unit_scope text not null default 'allowlist',
|
||||
allowed_unit_ids jsonb not null default '[]'::jsonb,
|
||||
collection_enabled boolean not null default false,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id)
|
||||
)
|
||||
`);
|
||||
await pool.query("alter table gelios_connections add column if not exists unit_scope text not null default 'allowlist'");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists gelios_collection_runs (
|
||||
id uuid primary key,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
capability_id text not null,
|
||||
status text not null,
|
||||
unit_count integer not null default 0,
|
||||
inserted_snapshots integer not null default 0,
|
||||
error_code text,
|
||||
started_at timestamptz not null default now(),
|
||||
completed_at timestamptz,
|
||||
foreign key (tenant_id, connection_id)
|
||||
references gelios_connections (tenant_id, connection_id)
|
||||
on delete cascade
|
||||
)
|
||||
`);
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists gelios_raw_envelopes (
|
||||
id uuid primary key,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
collection_run_id uuid references gelios_collection_runs(id) on delete set null,
|
||||
capability_id text not null,
|
||||
payload_hash text not null,
|
||||
payload jsonb not null,
|
||||
payload_bytes integer not null,
|
||||
received_at timestamptz not null,
|
||||
expires_at timestamptz not null,
|
||||
unique (tenant_id, connection_id, capability_id, payload_hash)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists gelios_raw_envelopes_expiry_idx on gelios_raw_envelopes (expires_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists gelios_units (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
source_unit_id bigint not null,
|
||||
stable_subject_id text not null,
|
||||
name text not null,
|
||||
first_seen_at timestamptz not null default now(),
|
||||
last_seen_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, source_unit_id),
|
||||
unique (tenant_id, connection_id, stable_subject_id),
|
||||
foreign key (tenant_id, connection_id)
|
||||
references gelios_connections (tenant_id, connection_id)
|
||||
on delete cascade
|
||||
)
|
||||
`);
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists gelios_unit_current (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
source_unit_id bigint not null,
|
||||
stable_subject_id text not null,
|
||||
name text not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
latitude double precision,
|
||||
longitude double precision,
|
||||
position geography(Point, 4326),
|
||||
speed double precision,
|
||||
course double precision,
|
||||
gps_valid boolean,
|
||||
satellites double precision,
|
||||
hdop double precision,
|
||||
accuracy_m double precision,
|
||||
telemetry jsonb not null,
|
||||
fingerprint text not null,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, source_unit_id),
|
||||
foreign key (tenant_id, connection_id, source_unit_id)
|
||||
references gelios_units (tenant_id, connection_id, source_unit_id)
|
||||
on delete cascade
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists gelios_unit_current_position_idx on gelios_unit_current using gist (position)");
|
||||
await pool.query("create index if not exists gelios_unit_current_observed_idx on gelios_unit_current (tenant_id, connection_id, observed_at desc)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists gelios_telemetry_snapshots (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
source_unit_id bigint not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
fingerprint text not null,
|
||||
telemetry jsonb not null,
|
||||
primary key (tenant_id, connection_id, source_unit_id, observed_at, fingerprint)
|
||||
)
|
||||
`);
|
||||
await pool.query("select create_hypertable('gelios_telemetry_snapshots', 'observed_at', if_not_exists => true, migrate_data => true)");
|
||||
await pool.query("create index if not exists gelios_snapshots_unit_time_idx on gelios_telemetry_snapshots (tenant_id, connection_id, source_unit_id, observed_at desc)");
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { GELIOS_CAPABILITIES } from "../capabilities.mjs";
|
||||
import { readConfig, unitIsInScope } from "../config.mjs";
|
||||
import { toFleetPositionPatch, toFleetPositionsSnapshot } from "../data-products.mjs";
|
||||
import { extractUnitItems, normalizeUnit } from "../normalize.mjs";
|
||||
|
||||
const unit = normalizeUnit({
|
||||
id: 42,
|
||||
name: "Sample fleet fixture",
|
||||
lmsg: { time: 1_789_123_456, lat: 55.75, lon: 37.61, speed: 12.5, gps_valid: true, sats: 11 },
|
||||
}, { tenantId: "sample-tenant", connectionId: "gelios-sample", receivedAt: new Date("2026-07-13T00:00:00.000Z") });
|
||||
assert.equal(unit.stableSubjectId, "gelios.unit:gelios-sample:42");
|
||||
assert.equal(unit.latitude, 55.75);
|
||||
assert.equal(unit.longitude, 37.61);
|
||||
assert.equal(unit.gpsValid, true);
|
||||
assert.equal(extractUnitItems({ items: [{ id: 1 }] }).length, 1);
|
||||
assert.equal(extractUnitItems({ units: [{ id: 1 }] }).length, 1);
|
||||
assert.equal(GELIOS_CAPABILITIES.some((item) => item.enabled === true && item.class.includes("write")), false);
|
||||
assert.equal(GELIOS_CAPABILITIES.some((item) => item.id === "gelios.commands.send" && item.enabled === false), true);
|
||||
const snapshot = toFleetPositionsSnapshot([unit], {
|
||||
tenantId: "sample-tenant",
|
||||
connectionId: "gelios-sample",
|
||||
generatedAt: new Date("2027-09-01T00:00:00.000Z"),
|
||||
staleAfterMs: 300_000,
|
||||
});
|
||||
assert.equal(snapshot.dataProduct.id, "fleet.positions.current.v1");
|
||||
assert.equal(snapshot.entities[0].semanticType, "map.moving_object");
|
||||
assert.equal(snapshot.entities[0].status, "stale");
|
||||
assert.equal(Object.hasOwn(snapshot.entities[0], "telemetry"), false);
|
||||
assert.equal(toFleetPositionPatch({ ...unit, latitude: null, longitude: null }, {
|
||||
generatedAt: new Date("2026-07-13T00:00:00.000Z"),
|
||||
staleAfterMs: 300_000,
|
||||
}).entity.status, "no-position");
|
||||
|
||||
const configBase = {
|
||||
DATABASE_URL: "postgresql://fixture",
|
||||
GELIOS_TENANT_ID: "fixture-tenant",
|
||||
GELIOS_CONNECTION_ID: "fixture-connection",
|
||||
GELIOS_INTAKE_ENABLED: "true",
|
||||
};
|
||||
const allUnitsConfig = readConfig({ ...configBase, GELIOS_UNIT_SCOPE: "all" });
|
||||
assert.equal(allUnitsConfig.unitScope, "all");
|
||||
assert.equal(unitIsInScope(allUnitsConfig, 1), true);
|
||||
assert.equal(unitIsInScope(allUnitsConfig, 99_999), true);
|
||||
const allowlistConfig = readConfig({ ...configBase, GELIOS_UNIT_SCOPE: "allowlist", GELIOS_ALLOWED_UNIT_IDS: "7,11" });
|
||||
assert.equal(unitIsInScope(allowlistConfig, 7), true);
|
||||
assert.equal(unitIsInScope(allowlistConfig, 8), false);
|
||||
assert.throws(() => readConfig({ ...configBase, GELIOS_UNIT_SCOPE: "everything" }), /GELIOS_UNIT_SCOPE/);
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
checks: [
|
||||
"unit_normalization",
|
||||
"safe_intake_registry",
|
||||
"provider_neutral_data_product",
|
||||
"all_units_connection_scope",
|
||||
"command_transport_absent",
|
||||
],
|
||||
}));
|
||||
@@ -0,0 +1,406 @@
|
||||
import express from "express";
|
||||
import { randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { createServer } from "node:http";
|
||||
import { Pool } from "pg";
|
||||
import { GELIOS_CAPABILITIES } from "./capabilities.mjs";
|
||||
import { readConfig, unitIsInScope } from "./config.mjs";
|
||||
import { FLEET_POSITIONS_CURRENT, toFleetPositionPatch, toFleetPositionsSnapshot } from "./data-products.mjs";
|
||||
import { extractUnitItems, hash, normalizeUnit } from "./normalize.mjs";
|
||||
import { migrate } from "./schema.mjs";
|
||||
|
||||
const config = readConfig();
|
||||
const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize });
|
||||
const app = express();
|
||||
const httpServer = createServer(app);
|
||||
const streamClients = new Set();
|
||||
let intakeInFlight = null;
|
||||
|
||||
app.disable("x-powered-by");
|
||||
app.use(express.json({ limit: "1mb" }));
|
||||
|
||||
app.get("/healthz", asyncRoute(async (_req, res) => {
|
||||
await pool.query("select 1");
|
||||
res.json({
|
||||
ok: true,
|
||||
service: "nodedc-gelios-gateway",
|
||||
database: "ready",
|
||||
internalApiConfigured: Boolean(config.internalAccessToken),
|
||||
intakeEnabled: config.intakeEnabled,
|
||||
unitScope: config.unitScope,
|
||||
credentialOwner: "engine",
|
||||
providerTransport: "absent",
|
||||
commandTransport: "absent",
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/status", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const latestRun = await pool.query(
|
||||
`select id, capability_id, status, unit_count, inserted_snapshots, error_code, started_at, completed_at
|
||||
from gelios_collection_runs
|
||||
where tenant_id = $1 and connection_id = $2
|
||||
order by started_at desc
|
||||
limit 1`,
|
||||
[scope.tenantId, scope.connectionId],
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
provider: "gelios",
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
unitScope: config.unitScope,
|
||||
approvedUnitCount: config.unitScope === "all" ? null : config.allowedUnitIds.length,
|
||||
intakeEnabled: config.intakeEnabled,
|
||||
credentialOwner: "engine",
|
||||
providerTransport: "absent",
|
||||
commandTransport: "absent",
|
||||
lastRun: latestRun.rows[0] || null,
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/capabilities", requireInternalApi, (_req, res) => {
|
||||
res.json({ ok: true, provider: "gelios", capabilities: GELIOS_CAPABILITIES, commandTransport: "absent" });
|
||||
});
|
||||
|
||||
app.post("/internal/gelios/v1/intake/units", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const result = await runIntake({
|
||||
scope,
|
||||
payload: req.body,
|
||||
receivedAt: parseReceivedAt(req.body?.receivedAt),
|
||||
});
|
||||
res.status(result.status === "completed" ? 200 : 409).json(result);
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/units/current", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const limit = boundedLimit(req.query.limit, 200, 1, 1000);
|
||||
const rows = await listCurrentRows(scope, limit);
|
||||
res.json({
|
||||
ok: true,
|
||||
contract: "gelios.current-position.v1",
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
units: rows,
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/data-products/fleet.positions.current.v1/snapshot", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const limit = boundedLimit(req.query.limit, 200, 1, 1000);
|
||||
const rows = await listCurrentRows(scope, limit);
|
||||
res.json({
|
||||
ok: true,
|
||||
...toFleetPositionsSnapshot(rows, { ...scope, staleAfterMs: config.positionStaleAfterMs }),
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/data-products/fleet.positions.current.v1/stream", requireInternalApi, (req, res) => {
|
||||
let scope;
|
||||
try {
|
||||
scope = requireConnectionScope(req);
|
||||
} catch (error) {
|
||||
res.status(Number(error.status || 400)).json({ ok: false, error: error.message });
|
||||
return;
|
||||
}
|
||||
const client = openSseClient(req, res, scope, "data-product");
|
||||
writeSse(res, "nodedc.data-product.ready.v1", {
|
||||
dataProduct: FLEET_POSITIONS_CURRENT,
|
||||
scope,
|
||||
bootstrap: "fetch-snapshot",
|
||||
});
|
||||
return client;
|
||||
});
|
||||
|
||||
app.get("/internal/gelios/v1/stream", requireInternalApi, (req, res) => {
|
||||
let scope;
|
||||
try {
|
||||
scope = requireConnectionScope(req);
|
||||
} catch (error) {
|
||||
res.status(Number(error.status || 400)).json({ ok: false, error: error.message });
|
||||
return;
|
||||
}
|
||||
const client = openSseClient(req, res, scope, "legacy");
|
||||
writeSse(res, "gelios.ready", {
|
||||
ok: true,
|
||||
contract: "gelios.current-position.v1",
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
commandTransport: "absent",
|
||||
});
|
||||
return client;
|
||||
});
|
||||
|
||||
async function listCurrentRows(scope, limit) {
|
||||
const rows = await pool.query(
|
||||
`select
|
||||
stable_subject_id as "subjectId", source_unit_id as "sourceUnitId", name,
|
||||
observed_at as "observedAt", received_at as "receivedAt", latitude, longitude,
|
||||
speed, course, gps_valid as "gpsValid", satellites, hdop,
|
||||
accuracy_m as "accuracyM", telemetry
|
||||
from gelios_unit_current
|
||||
where tenant_id = $1 and connection_id = $2
|
||||
order by observed_at desc, source_unit_id asc
|
||||
limit $3`,
|
||||
[scope.tenantId, scope.connectionId, limit],
|
||||
);
|
||||
return rows.rows;
|
||||
}
|
||||
|
||||
app.use((error, _req, res, _next) => {
|
||||
const status = Number(error?.status || 500);
|
||||
const publicStatus = status >= 400 && status < 600 ? status : 500;
|
||||
console.error(JSON.stringify({ event: "gelios_gateway_error", error: safeErrorCode(error), status: publicStatus }));
|
||||
res.status(publicStatus).json({ ok: false, error: publicStatus >= 500 ? "internal_error" : safeErrorCode(error) });
|
||||
});
|
||||
|
||||
await migrate(pool);
|
||||
await upsertConnection();
|
||||
|
||||
httpServer.listen(config.port, "0.0.0.0", () => {
|
||||
console.log(`NODE.DC Gelios Gateway listening on http://0.0.0.0:${config.port}`);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
async function upsertConnection() {
|
||||
await pool.query(
|
||||
`insert into gelios_connections (
|
||||
tenant_id, connection_id, unit_scope, allowed_unit_ids, collection_enabled, updated_at
|
||||
) values ($1, $2, $3, $4::jsonb, $5, now())
|
||||
on conflict (tenant_id, connection_id) do update
|
||||
set unit_scope = excluded.unit_scope,
|
||||
allowed_unit_ids = excluded.allowed_unit_ids,
|
||||
collection_enabled = excluded.collection_enabled,
|
||||
updated_at = now()`,
|
||||
[config.tenantId, config.connectionId, config.unitScope, JSON.stringify(config.allowedUnitIds), config.intakeEnabled],
|
||||
);
|
||||
}
|
||||
|
||||
async function runIntake({ scope, payload, receivedAt }) {
|
||||
if (!config.intakeEnabled) return { ok: false, status: "disabled" };
|
||||
if (config.unitScope === "allowlist" && !config.allowedUnitIds.length) {
|
||||
return { ok: false, status: "scope_missing" };
|
||||
}
|
||||
if (intakeInFlight) return { ok: false, status: "busy" };
|
||||
intakeInFlight = ingestCurrentUnits({ scope, payload, receivedAt }).finally(() => {
|
||||
intakeInFlight = null;
|
||||
});
|
||||
return intakeInFlight;
|
||||
}
|
||||
|
||||
async function ingestCurrentUnits({ scope, payload, receivedAt }) {
|
||||
const runId = randomUUID();
|
||||
await pool.query(
|
||||
`insert into gelios_collection_runs (id, tenant_id, connection_id, capability_id, status)
|
||||
values ($1, $2, $3, 'gelios.units.current_intake', 'running')`,
|
||||
[runId, scope.tenantId, scope.connectionId],
|
||||
);
|
||||
|
||||
try {
|
||||
const sourcePayload = payload?.units ? { units: payload.units } : payload;
|
||||
const items = extractUnitItems(sourcePayload);
|
||||
if (!items.length) throw Object.assign(new Error("intake_units_required"), { status: 422 });
|
||||
await storeRawEnvelope({ runId, scope, payload: sourcePayload, receivedAt });
|
||||
const normalized = items
|
||||
.map((item) => normalizeUnit(item, { ...scope, receivedAt }))
|
||||
.filter(Boolean)
|
||||
.filter((item) => unitIsInScope(config, item.sourceUnitId));
|
||||
|
||||
let insertedSnapshots = 0;
|
||||
for (const unit of normalized) {
|
||||
const result = await persistCurrentUnit(unit);
|
||||
insertedSnapshots += result.insertedSnapshot ? 1 : 0;
|
||||
if (result.current) broadcastCurrentUnit(result.current, scope.tenantId, scope.connectionId);
|
||||
}
|
||||
await pruneExpiredRawEnvelopes();
|
||||
await pool.query(
|
||||
`update gelios_collection_runs
|
||||
set status = 'completed', unit_count = $2, inserted_snapshots = $3, completed_at = now()
|
||||
where id = $1`,
|
||||
[runId, normalized.length, insertedSnapshots],
|
||||
);
|
||||
return { ok: true, status: "completed", runId, unitCount: normalized.length, insertedSnapshots };
|
||||
} catch (error) {
|
||||
await pool.query(
|
||||
`update gelios_collection_runs
|
||||
set status = 'failed', error_code = $2, completed_at = now()
|
||||
where id = $1`,
|
||||
[runId, safeErrorCode(error)],
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function storeRawEnvelope({ runId, scope, payload, receivedAt }) {
|
||||
const serialized = JSON.stringify(payload);
|
||||
await pool.query(
|
||||
`insert into gelios_raw_envelopes (
|
||||
id, tenant_id, connection_id, collection_run_id, capability_id,
|
||||
payload_hash, payload, payload_bytes, received_at, expires_at
|
||||
) values ($1, $2, $3, $4, 'gelios.units.current_intake', $5, $6::jsonb, $7, $8, $9)
|
||||
on conflict (tenant_id, connection_id, capability_id, payload_hash) do nothing`,
|
||||
[
|
||||
randomUUID(), scope.tenantId, scope.connectionId, runId, hash(payload), serialized,
|
||||
Buffer.byteLength(serialized), receivedAt,
|
||||
new Date(receivedAt.getTime() + config.rawRetentionDays * 24 * 60 * 60 * 1000),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
async function persistCurrentUnit(unit) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
await client.query(
|
||||
`insert into gelios_units (
|
||||
tenant_id, connection_id, source_unit_id, stable_subject_id, name, last_seen_at
|
||||
) values ($1, $2, $3, $4, $5, $6)
|
||||
on conflict (tenant_id, connection_id, source_unit_id) do update
|
||||
set stable_subject_id = excluded.stable_subject_id,
|
||||
name = excluded.name,
|
||||
last_seen_at = excluded.last_seen_at`,
|
||||
[unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.stableSubjectId, unit.name, unit.receivedAt],
|
||||
);
|
||||
const current = await client.query(
|
||||
`insert into gelios_unit_current (
|
||||
tenant_id, connection_id, source_unit_id, stable_subject_id, name,
|
||||
observed_at, received_at, latitude, longitude, position, speed, course,
|
||||
gps_valid, satellites, hdop, accuracy_m, telemetry, fingerprint
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
case when $8::double precision is not null and $9::double precision is not null
|
||||
then ST_SetSRID(ST_MakePoint($9, $8), 4326)::geography else null end,
|
||||
$10, $11, $12, $13, $14, $15, $16::jsonb, $17
|
||||
)
|
||||
on conflict (tenant_id, connection_id, source_unit_id) do update
|
||||
set stable_subject_id = excluded.stable_subject_id,
|
||||
name = excluded.name,
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
latitude = excluded.latitude,
|
||||
longitude = excluded.longitude,
|
||||
position = excluded.position,
|
||||
speed = excluded.speed,
|
||||
course = excluded.course,
|
||||
gps_valid = excluded.gps_valid,
|
||||
satellites = excluded.satellites,
|
||||
hdop = excluded.hdop,
|
||||
accuracy_m = excluded.accuracy_m,
|
||||
telemetry = excluded.telemetry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at >= gelios_unit_current.observed_at
|
||||
returning stable_subject_id as "subjectId", source_unit_id as "sourceUnitId", name,
|
||||
observed_at as "observedAt", received_at as "receivedAt", latitude, longitude,
|
||||
speed, course, gps_valid as "gpsValid", satellites, hdop, accuracy_m as "accuracyM", telemetry`,
|
||||
[
|
||||
unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.stableSubjectId, unit.name,
|
||||
unit.observedAt, unit.receivedAt, unit.latitude, unit.longitude, unit.speed, unit.course,
|
||||
unit.gpsValid, unit.satellites, unit.hdop, unit.accuracyM, JSON.stringify(unit.telemetry), unit.fingerprint,
|
||||
],
|
||||
);
|
||||
const snapshot = await client.query(
|
||||
`insert into gelios_telemetry_snapshots (
|
||||
tenant_id, connection_id, source_unit_id, observed_at, received_at, fingerprint, telemetry
|
||||
) values ($1, $2, $3, $4, $5, $6, $7::jsonb)
|
||||
on conflict do nothing`,
|
||||
[unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.observedAt, unit.receivedAt, unit.fingerprint, JSON.stringify(unit.telemetry)],
|
||||
);
|
||||
await client.query("commit");
|
||||
return { current: current.rows[0] || null, insertedSnapshot: snapshot.rowCount > 0 };
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function pruneExpiredRawEnvelopes() {
|
||||
await pool.query("delete from gelios_raw_envelopes where expires_at < now()");
|
||||
}
|
||||
|
||||
function requireInternalApi(req, res, next) {
|
||||
const expected = Buffer.from(config.internalAccessToken || "");
|
||||
const received = Buffer.from(String(req.get("authorization") || "").replace(/^Bearer\s+/i, "").trim());
|
||||
if (!expected.length) {
|
||||
res.status(503).json({ ok: false, error: "internal_api_not_configured" });
|
||||
return;
|
||||
}
|
||||
if (expected.length !== received.length || !timingSafeEqual(expected, received)) {
|
||||
res.status(401).json({ ok: false, error: "internal_authorization_required" });
|
||||
return;
|
||||
}
|
||||
next();
|
||||
}
|
||||
|
||||
function requireConnectionScope(req) {
|
||||
const tenantId = String(req.get("x-nodedc-tenant-id") || "").trim();
|
||||
const connectionId = String(req.query.connectionId || req.get("x-nodedc-connection-id") || config.connectionId).trim();
|
||||
if (tenantId !== config.tenantId) throw Object.assign(new Error("tenant_scope_required"), { status: 403 });
|
||||
if (connectionId !== config.connectionId) throw Object.assign(new Error("connection_scope_denied"), { status: 403 });
|
||||
return { tenantId, connectionId };
|
||||
}
|
||||
|
||||
function broadcastCurrentUnit(unit, tenantId, connectionId) {
|
||||
for (const client of streamClients) {
|
||||
if (client.tenantId === tenantId && client.connectionId === connectionId) {
|
||||
if (client.kind === "data-product") {
|
||||
writeSse(client.res, "nodedc.data-product.patch.v1", toFleetPositionPatch(unit, {
|
||||
staleAfterMs: config.positionStaleAfterMs,
|
||||
}));
|
||||
} else {
|
||||
writeSse(client.res, "gelios.current.updated", unit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function openSseClient(req, res, scope, kind) {
|
||||
const client = { id: randomUUID(), res, kind, ...scope };
|
||||
res.setHeader("Content-Type", "text/event-stream");
|
||||
res.setHeader("Cache-Control", "no-cache, no-transform");
|
||||
res.setHeader("Connection", "keep-alive");
|
||||
res.setHeader("X-Accel-Buffering", "no");
|
||||
res.flushHeaders?.();
|
||||
const keepAlive = setInterval(() => res.write(": keep-alive\n\n"), 30000);
|
||||
streamClients.add(client);
|
||||
req.on("close", () => {
|
||||
clearInterval(keepAlive);
|
||||
streamClients.delete(client);
|
||||
});
|
||||
return client;
|
||||
}
|
||||
|
||||
function writeSse(res, event, payload) {
|
||||
res.write(`event: ${event}\ndata: ${JSON.stringify(payload)}\n\n`);
|
||||
}
|
||||
|
||||
function parseReceivedAt(value) {
|
||||
if (value === undefined || value === null || value === "") return new Date();
|
||||
const parsed = new Date(value);
|
||||
if (Number.isNaN(parsed.getTime())) throw Object.assign(new Error("intake_received_at_invalid"), { status: 422 });
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function boundedLimit(value, fallback, min, max) {
|
||||
const parsed = Number.parseInt(String(value || ""), 10);
|
||||
if (!Number.isInteger(parsed)) return fallback;
|
||||
return Math.max(min, Math.min(max, parsed));
|
||||
}
|
||||
|
||||
function safeErrorCode(error) {
|
||||
return String(error?.message || "internal_error").replace(/[^a-zA-Z0-9_:.-]+/g, "_").slice(0, 160);
|
||||
}
|
||||
|
||||
function asyncRoute(handler) {
|
||||
return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next);
|
||||
}
|
||||
|
||||
async function shutdown() {
|
||||
await new Promise((resolve) => httpServer.close(resolve));
|
||||
await pool.end();
|
||||
process.exit(0);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
FROM node:22-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json ./
|
||||
COPY catalog ./catalog
|
||||
COPY docs ./docs
|
||||
COPY examples ./examples
|
||||
COPY src ./src
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=18104
|
||||
|
||||
EXPOSE 18104
|
||||
|
||||
CMD ["node", "src/mcp-server.mjs"]
|
||||
@@ -4,6 +4,9 @@ Docs-first ontology service/module for NODE.DC.
|
||||
|
||||
Current status: `v0.4-pre` implementation slice.
|
||||
|
||||
The read-only dynamic MCP integration for AI Workspace is documented in
|
||||
[`../../docs/AI_WORKSPACE_ONTOLOGY_MCP.md`](../../docs/AI_WORKSPACE_ONTOLOGY_MCP.md).
|
||||
|
||||
This module owns:
|
||||
|
||||
- canonical entity catalog;
|
||||
@@ -62,6 +65,7 @@ src/adapters/hub-launcher-admin.mjs
|
||||
|
||||
```text
|
||||
npm run validate
|
||||
npm run smoke:mcp
|
||||
```
|
||||
|
||||
## Resolver Smoke
|
||||
@@ -159,8 +163,10 @@ Each package may contribute entities, relations, aliases, guardrails, evidence,
|
||||
|
||||
Current package:
|
||||
|
||||
- `integration` - provider-neutral external provider connection, capability, collection, read-model and red command-domain ontology used by all adapter services.
|
||||
- `seo` - NDC SEO mod domain ontology for site scans, project ontology instances, scope contracts, semantic analysis, market evidence, optimization planning, validation, changesets, and future app-owned SEO assistant actions.
|
||||
- `map` - provider-neutral NDC Module Studio map ontology for spatial subjects, layers, routes, zones, shared labels, visibility rules, selection, and replaceable renderer adapters.
|
||||
- `gelios` - provider-neutral Gelios fleet and telemetry ontology, bound to map.moving_object, map.zone and map.place_target without exposing provider credentials or renderer objects.
|
||||
|
||||
The package loader merges domain packages into the base catalog before validation. Domain packages extend core meanings; they do not own app data or execute mutations.
|
||||
|
||||
|
||||
@@ -16,9 +16,8 @@
|
||||
{ "alias": "app", "canonicalId": "hub.application" },
|
||||
{ "alias": "application tile", "canonicalId": "hub.application_card" },
|
||||
{ "alias": "app card", "canonicalId": "hub.application_card" },
|
||||
{ "alias": "n8n workflow id", "canonicalId": "engine.workflow_l2_runtime_id" },
|
||||
{ "alias": "runtime workflow id", "canonicalId": "engine.workflow_l2_runtime_id" },
|
||||
{ "alias": "n8n execution", "canonicalId": "engine.l2_execution" },
|
||||
{ "alias": "l2 execution", "canonicalId": "engine.l2_execution" },
|
||||
{ "alias": "Engine-side OPS", "canonicalId": "engine.tech_debt.ops_layer" },
|
||||
{ "alias": "Agent Monitor OPS", "canonicalId": "engine.tech_debt.ops_layer" },
|
||||
{ "alias": "Tender Agent Monitor UI", "canonicalId": "engine.tech_debt.tender_domain_ui" },
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"aliases": [
|
||||
{ "alias": "gelios", "canonicalId": "gelios.integration" },
|
||||
{ "alias": "gelius", "canonicalId": "gelios.integration" },
|
||||
{ "alias": "helius", "canonicalId": "gelios.integration" },
|
||||
{ "alias": "hel i os", "canonicalId": "gelios.integration" },
|
||||
{ "alias": "гелиос", "canonicalId": "gelios.integration" },
|
||||
{ "alias": "трайк robot2b", "canonicalId": "gelios.unit" },
|
||||
{ "alias": "юнит гелиос", "canonicalId": "gelios.unit" },
|
||||
{ "alias": "группа юнитов гелиос", "canonicalId": "gelios.unit_group" },
|
||||
{ "alias": "последняя телеметрия гелиос", "canonicalId": "gelios.telemetry_snapshot" },
|
||||
{ "alias": "сырая телеметрия гелиос", "canonicalId": "gelios.raw_telemetry_message" },
|
||||
{ "alias": "позиция трайка", "canonicalId": "gelios.position_fix" },
|
||||
{ "alias": "статус трайка", "canonicalId": "gelios.operational_status" },
|
||||
{ "alias": "датчик гелиос", "canonicalId": "gelios.sensor_definition" },
|
||||
{ "alias": "показание датчика", "canonicalId": "gelios.sensor_reading" },
|
||||
{ "alias": "калибровка датчика", "canonicalId": "gelios.sensor_conversion" },
|
||||
{ "alias": "топливный профиль", "canonicalId": "gelios.fuel_profile" },
|
||||
{ "alias": "геозона гелиос", "canonicalId": "gelios.geozone" },
|
||||
{ "alias": "геоточка гелиос", "canonicalId": "gelios.geopoint" },
|
||||
{ "alias": "шаблон команды гелиос", "canonicalId": "gelios.command_template" },
|
||||
{ "alias": "диспетчеризация команды гелиос", "canonicalId": "gelios.command_dispatch" },
|
||||
{ "alias": "аудит команды гелиос", "canonicalId": "gelios.command_audit" },
|
||||
{ "alias": "контур robot2b", "canonicalId": "gelios.access_scope" },
|
||||
{ "alias": "курсор загрузки гелиос", "canonicalId": "gelios.ingestion_cursor" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"entities": [
|
||||
{ "id": "gelios.integration", "name": "Gelios Integration", "surface": "platform", "status": ["product-required", "source-evidenced"], "authority": "Gelios Gateway", "summary": "Server-side provider integration boundary for Gelios REST OAuth and legacy compatibility. It owns no UI, map renderer or workflow secret." },
|
||||
{ "id": "gelios.access_scope", "name": "Gelios Access Scope", "surface": "platform", "status": ["product-required", "source-evidenced"], "authority": "Robot2B owner + Gelios Gateway", "summary": "Approved collection boundary expressed as an allowlist or owner rule for units, groups and allowed capabilities; distinct from broad provider-account visibility." },
|
||||
{ "id": "gelios.unit", "name": "Gelios Unit", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Tracked operational object, including a Robot2B trike when its stable source unit ID is inside the approved scope." },
|
||||
{ "id": "gelios.unit_group", "name": "Gelios Unit Group", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider-managed grouping of units. It is evidence for navigation and access, not an automatic definition of the Robot2B business scope." },
|
||||
{ "id": "gelios.tracker_device", "name": "Tracker Device", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Hardware tracker associated with a unit, carrying device type/manufacturer and restricted hardware identity fields." },
|
||||
{ "id": "gelios.telemetry_snapshot", "name": "Telemetry Snapshot", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Normalized current telemetry state for one unit at observed and received time; derived from the Gelios pure last message contract." },
|
||||
{ "id": "gelios.raw_telemetry_message", "name": "Raw Telemetry Message", "surface": "telemetry", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider hardware message preserved only behind restricted raw-data policy; it is not the default Studio or analytics contract." },
|
||||
{ "id": "gelios.position_fix", "name": "Position Fix", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Time-qualified geographic position with latitude, longitude, height, course, speed, satellite and quality attributes for one unit." },
|
||||
{ "id": "gelios.telemetry_parameter", "name": "Telemetry Parameter", "surface": "telemetry", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Name/value parameter received with a message. Parameters are dynamic and require a namespaced registry and whitelist before product exposure." },
|
||||
{ "id": "gelios.operational_status", "name": "Operational Status", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Derived unit state such as online, offline, moving, parked, no-position or low-GPS; distinct from raw provider message fields." },
|
||||
{ "id": "gelios.sensor_definition", "name": "Sensor Definition", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Declared sensor on a unit with message parameter, type, unit of measure, visibility and optional fuel semantics." },
|
||||
{ "id": "gelios.sensor_reading", "name": "Sensor Reading", "surface": "telemetry", "status": ["source-evidenced", "product-required"], "authority": "Gelios Gateway", "summary": "Time-qualified normalized reading of a defined sensor, retaining both numeric value and human-readable representation when permitted." },
|
||||
{ "id": "gelios.sensor_conversion", "name": "Sensor Conversion", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Calibration/conversion rule or row for translating raw sensor values into operational measures." },
|
||||
{ "id": "gelios.fuel_profile", "name": "Fuel Profile", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Fuel and consumption configuration associated with a unit or fuel sensor; use only after sensor semantics are approved." },
|
||||
{ "id": "gelios.maintenance_plan", "name": "Maintenance Plan", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Planned maintenance configuration for a unit, separate from historical maintenance events." },
|
||||
{ "id": "gelios.custom_field", "name": "Custom Field", "surface": "domain", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider-configurable unit field. Its value is restricted until explicitly classified because semantics and PII risk are dynamic." },
|
||||
{ "id": "gelios.geozone", "name": "Gelios Geozone", "surface": "spatial", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider geofence or zone geometry. Large geometry collections require a separate loading and LOD strategy." },
|
||||
{ "id": "gelios.geozone_group", "name": "Gelios Geozone Group", "surface": "spatial", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Provider grouping of geozones." },
|
||||
{ "id": "gelios.geopoint", "name": "Gelios Geopoint", "surface": "spatial", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Named provider spatial point; it must remain distinct from a live unit position." },
|
||||
{ "id": "gelios.report_template", "name": "Gelios Report Template", "surface": "analytics", "status": ["source-evidenced"], "authority": "Gelios Pro", "summary": "Reusable provider report definition available to the account or scope." },
|
||||
{ "id": "gelios.report_result", "name": "Gelios Report Result", "surface": "analytics", "status": ["source-evidenced"], "authority": "Gelios Pro + Gelios Gateway", "summary": "Bounded result, table, graphic or map output produced from an approved report request; not a default realtime stream." },
|
||||
{ "id": "gelios.collection_run", "name": "Gelios Collection Run", "surface": "platform", "status": ["product-required"], "authority": "Gelios Gateway", "summary": "Auditable read-only collection attempt with scope, endpoint family, volume controls and outcome; it contains no token or raw payload." },
|
||||
{ "id": "gelios.ingestion_cursor", "name": "Gelios Ingestion Cursor", "surface": "platform", "status": ["product-required"], "authority": "Gelios Gateway + telemetry storage", "summary": "Checkpoint and watermark controlling resumable historical or incremental ingestion for an approved unit and data family." },
|
||||
{ "id": "gelios.command_template", "name": "Gelios Command Template", "surface": "command", "status": ["source-evidenced", "product-required"], "authority": "Gelios Pro", "summary": "Read-only catalogued command template assigned to a unit. Template content is red-domain configuration, not a telemetry field." },
|
||||
{ "id": "gelios.command_group", "name": "Gelios Command Group", "surface": "command", "status": ["source-evidenced", "product-required"], "authority": "Gelios Pro", "summary": "Provider grouping of command templates, managed separately from operational unit groups." },
|
||||
{ "id": "gelios.command_dispatch", "name": "Gelios Command Dispatch", "surface": "command", "status": ["product-required"], "authority": "Future Command Gateway", "summary": "Explicit, user-confirmed request to send a command to one approved unit or approved group. It must never be created by background ingestion or map rendering." },
|
||||
{ "id": "gelios.command_delivery", "name": "Gelios Command Delivery", "surface": "command", "status": ["product-required", "source-evidenced"], "authority": "Gelios Pro + Future Command Gateway", "summary": "Provider delivery/task status associated with a command dispatch." },
|
||||
{ "id": "gelios.command_audit", "name": "Gelios Command Audit", "surface": "command", "status": ["product-required"], "authority": "Future Command Gateway", "summary": "Immutable governance record of an explicit command decision, confirmation, target, outcome and actor; separate from provider command history." }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"sourceRoots": [
|
||||
{
|
||||
"surface": "gelios-rest",
|
||||
"path": "https://api.geliospro.com/docs",
|
||||
"mode": "OpenAPI inspection and safe read-only runtime audit; no write routes executed"
|
||||
},
|
||||
{
|
||||
"surface": "engine",
|
||||
"path": "/Users/dcconstructions/Downloads/mnt/NODEDC/NODEDC_ENGINE_INFRA/nodedc-source",
|
||||
"mode": "Read-only MMAP workflow, Gelios node and historical snapshot donor inspection"
|
||||
},
|
||||
{
|
||||
"surface": "module-studio",
|
||||
"path": "/Users/dcconstructions/Downloads/mnt/NODEDC/NODEDC_DESIGN_GUIDELINE",
|
||||
"mode": "Future provider-neutral map target surface"
|
||||
}
|
||||
],
|
||||
"ledgers": [
|
||||
{
|
||||
"id": "ledger.gelios_domain_v0",
|
||||
"path": "docs/GELIOS_DOMAIN_ONTOLOGY.md",
|
||||
"entityIds": [
|
||||
"gelios.integration",
|
||||
"gelios.access_scope",
|
||||
"gelios.unit",
|
||||
"gelios.telemetry_snapshot",
|
||||
"gelios.position_fix",
|
||||
"gelios.sensor_definition",
|
||||
"gelios.sensor_reading",
|
||||
"gelios.geozone",
|
||||
"gelios.report_template",
|
||||
"gelios.command_template",
|
||||
"gelios.command_dispatch",
|
||||
"gelios.ingestion_cursor"
|
||||
]
|
||||
}
|
||||
],
|
||||
"baselineDocs": ["docs/GELIOS_DOMAIN_ONTOLOGY.md"],
|
||||
"restrictions": [
|
||||
"Do not copy access/refresh tokens, passwords, hardware decrypt keys or runtime snapshots into Ontology Core.",
|
||||
"Do not make the current Engine Gelios node, Cesium entities or provider group names canonical domain identities.",
|
||||
"Do not invoke command send, create, update, delete or purge routes as ontology evidence.",
|
||||
"Do not treat the 107 currently visible provider-account units or the 95 legacy snapshot units as the final Robot2B scope without owner approval.",
|
||||
"Do not bulk-download history, media or full geozone geometry before collection policy and storage architecture are approved."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"rules": [
|
||||
{
|
||||
"id": "guardrail.gelios.ontology_not_runtime_store",
|
||||
"severity": "error",
|
||||
"summary": "Ontology Core describes Gelios meanings and contracts only. It must not contain access/refresh tokens, raw messages, live positions, provider command text, hardware decrypt keys or bulk geometry.",
|
||||
"entityIds": ["gelios.integration", "gelios.telemetry_snapshot", "gelios.raw_telemetry_message", "gelios.position_fix", "gelios.command_template", "gelios.geozone"]
|
||||
},
|
||||
{
|
||||
"id": "guardrail.gelios.scope_precedes_collection",
|
||||
"severity": "error",
|
||||
"summary": "A Gelios collection run must resolve an approved Robot2B access scope before addressing units. Provider-account visibility and provider group membership are not sufficient product scope on their own.",
|
||||
"entityIds": ["gelios.integration", "gelios.access_scope", "gelios.collection_run", "gelios.unit", "gelios.unit_group"]
|
||||
},
|
||||
{
|
||||
"id": "guardrail.gelios.commands_are_red_domain",
|
||||
"severity": "error",
|
||||
"summary": "Command template catalogue is read-only data. Command dispatch, command group mutation, history purge and every send route are separate red-domain actions requiring explicit human confirmation and audit; ingestion and map rendering may never create them.",
|
||||
"entityIds": ["gelios.command_template", "gelios.command_group", "gelios.command_dispatch", "gelios.command_delivery", "gelios.command_audit"]
|
||||
},
|
||||
{
|
||||
"id": "guardrail.gelios.map_uses_stable_subjects",
|
||||
"severity": "error",
|
||||
"summary": "Cesium pins and labels must consume stable gelios.unit and normalized gelios.position_fix through map.moving_object. They must not use raw message IDs, transient Cesium entity IDs or provider credentials as map identity.",
|
||||
"entityIds": ["gelios.unit", "gelios.position_fix", "gelios.telemetry_snapshot", "map.moving_object", "map.pin", "map.label"]
|
||||
},
|
||||
{
|
||||
"id": "guardrail.gelios.raw_and_pii_are_restricted",
|
||||
"severity": "warning",
|
||||
"summary": "IMEI/hardware identifiers, phone fields, precise addresses, raw params, custom fields and unclassified sensor values require explicit field policy before analytics, Studio or external exposure.",
|
||||
"entityIds": ["gelios.tracker_device", "gelios.position_fix", "gelios.telemetry_parameter", "gelios.custom_field", "gelios.sensor_reading"]
|
||||
},
|
||||
{
|
||||
"id": "guardrail.gelios.geometry_requires_bounded_loading",
|
||||
"severity": "warning",
|
||||
"summary": "Geozone and report-map loading must use explicit paging, filtering, volume limits, LOD and retention policy. The unpaged provider geozone response is not an acceptable realtime renderer contract.",
|
||||
"entityIds": ["gelios.geozone", "gelios.geozone_group", "gelios.collection_run", "map.zone", "map.visibility_rule"]
|
||||
},
|
||||
{
|
||||
"id": "guardrail.gelios.history_requires_cursor",
|
||||
"severity": "warning",
|
||||
"summary": "Any future history ingestion must be bounded by unit scope, time window, field whitelist and durable cursor. It must not turn a historical API family into an uncontrolled account dump.",
|
||||
"entityIds": ["gelios.collection_run", "gelios.ingestion_cursor", "gelios.raw_telemetry_message", "gelios.report_result"]
|
||||
}
|
||||
],
|
||||
"blockedConflations": [
|
||||
["gelios.integration", "gelios.access_scope"],
|
||||
["gelios.unit", "gelios.tracker_device"],
|
||||
["gelios.telemetry_snapshot", "gelios.raw_telemetry_message"],
|
||||
["gelios.position_fix", "map.moving_object"],
|
||||
["gelios.command_template", "gelios.command_dispatch"],
|
||||
["gelios.command_dispatch", "gelios.command_delivery"],
|
||||
["gelios.geozone", "map.zone"]
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"id": "gelios",
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"status": "product-required/source-evidenced",
|
||||
"summary": "Gelios Pro telemetry, fleet, spatial, reporting and command-domain ontology for the Robot2B client context. The package describes canonical meanings and contracts; it does not store credentials or runtime telemetry."
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"relations": [
|
||||
{ "id": "gelios.integration.is_provider_connection", "from": ["gelios.integration"], "to": ["integration.connection"], "status": "product-required", "summary": "A Gelios integration is a tenant-scoped instance of the common external provider connection contract." },
|
||||
{ "id": "gelios.access_scope.specializes_integration_scope", "from": ["gelios.access_scope"], "to": ["integration.access_scope"], "status": "product-required", "summary": "Robot2B/Gelios object approval specializes the common provider access-scope contract." },
|
||||
{ "id": "gelios.collection_run.is_integration_collection_run", "from": ["gelios.collection_run"], "to": ["integration.collection_run"], "status": "product-required", "summary": "Gelios collection runs inherit the bounded collection and audit lifecycle of external integrations." },
|
||||
{ "id": "gelios.raw_message.is_integration_raw_envelope", "from": ["gelios.raw_telemetry_message"], "to": ["integration.raw_envelope"], "status": "product-required", "summary": "A Gelios raw message is a restricted provider envelope, not a UI read model." },
|
||||
{ "id": "gelios.unit.is_integration_canonical_subject", "from": ["gelios.unit"], "to": ["integration.canonical_subject"], "status": "product-required", "summary": "A Gelios unit is a stable canonical subject produced by the provider adapter." },
|
||||
{ "id": "gelios.command_template.is_integration_command_capability", "from": ["gelios.command_template"], "to": ["integration.command_capability"], "status": "product-required", "summary": "Gelios command templates are catalogued provider command capabilities only." },
|
||||
{ "id": "gelios.command_dispatch.is_integration_command_intent", "from": ["gelios.command_dispatch"], "to": ["integration.command_intent"], "status": "future-concept", "summary": "Any future Gelios command dispatch must become an explicitly governed generic command intent." },
|
||||
{ "id": "gelios.command_audit.is_integration_command_audit", "from": ["gelios.command_audit"], "to": ["integration.command_audit"], "status": "future-concept", "summary": "Gelios command audit specializes the generic red-domain audit contract." },
|
||||
{ "id": "gelios.integration.enforces_scope", "from": ["gelios.integration"], "to": ["gelios.access_scope"], "status": "product-required", "summary": "The Gateway applies an approved Robot2B scope before reading or publishing data." },
|
||||
{ "id": "gelios.access_scope.includes_unit", "from": ["gelios.access_scope"], "to": ["gelios.unit", "gelios.unit_group"], "status": "product-required", "summary": "An approved scope can include explicit units and approved group criteria, but group membership alone is not authoritative." },
|
||||
{ "id": "gelios.unit.belongs_to_group", "from": ["gelios.unit"], "to": ["gelios.unit_group"], "status": "source-evidenced", "summary": "A provider unit can belong to one or more provider unit groups." },
|
||||
{ "id": "gelios.unit.uses_tracker_device", "from": ["gelios.unit"], "to": ["gelios.tracker_device"], "status": "source-evidenced", "summary": "A tracked unit is associated with a hardware tracker device." },
|
||||
{ "id": "gelios.unit.has_telemetry_snapshot", "from": ["gelios.unit"], "to": ["gelios.telemetry_snapshot"], "status": "product-required", "summary": "The Gateway maintains normalized current telemetry for an approved unit." },
|
||||
{ "id": "gelios.raw_message.normalizes_to_snapshot", "from": ["gelios.raw_telemetry_message"], "to": ["gelios.telemetry_snapshot"], "status": "product-required", "summary": "Restricted raw provider telemetry can be normalized into the stable snapshot contract." },
|
||||
{ "id": "gelios.telemetry_snapshot.has_position_fix", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.position_fix"], "status": "product-required", "summary": "A current telemetry snapshot can carry one time-qualified position fix." },
|
||||
{ "id": "gelios.telemetry_snapshot.has_operational_status", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.operational_status"], "status": "product-required", "summary": "Gateway derives normalized operational state from telemetry and data-quality policy." },
|
||||
{ "id": "gelios.telemetry_snapshot.has_parameter", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.telemetry_parameter"], "status": "source-evidenced", "summary": "A provider snapshot can contain dynamic telemetry parameters." },
|
||||
{ "id": "gelios.unit.has_sensor", "from": ["gelios.unit"], "to": ["gelios.sensor_definition"], "status": "source-evidenced", "summary": "A unit exposes zero or more sensor definitions." },
|
||||
{ "id": "gelios.sensor_definition.has_conversion", "from": ["gelios.sensor_definition"], "to": ["gelios.sensor_conversion"], "status": "source-evidenced", "summary": "A sensor can define conversion/calibration rows." },
|
||||
{ "id": "gelios.sensor_reading.observes_sensor", "from": ["gelios.sensor_reading"], "to": ["gelios.sensor_definition"], "status": "product-required", "summary": "Each normalized reading identifies the sensor definition it observes." },
|
||||
{ "id": "gelios.telemetry_snapshot.has_sensor_reading", "from": ["gelios.telemetry_snapshot"], "to": ["gelios.sensor_reading"], "status": "product-required", "summary": "Current telemetry can expose normalized readings for available sensors." },
|
||||
{ "id": "gelios.unit.has_fuel_profile", "from": ["gelios.unit"], "to": ["gelios.fuel_profile"], "status": "source-evidenced", "summary": "Fuel configuration belongs to a unit and is interpreted through approved sensor semantics." },
|
||||
{ "id": "gelios.unit.has_maintenance_plan", "from": ["gelios.unit"], "to": ["gelios.maintenance_plan"], "status": "source-evidenced", "summary": "A unit can have zero or more planned maintenance configurations." },
|
||||
{ "id": "gelios.unit.has_custom_field", "from": ["gelios.unit"], "to": ["gelios.custom_field"], "status": "source-evidenced", "summary": "A unit can have provider-defined custom fields." },
|
||||
{ "id": "gelios.geozone.belongs_to_group", "from": ["gelios.geozone"], "to": ["gelios.geozone_group"], "status": "source-evidenced", "summary": "A provider geozone can belong to a provider geozone group." },
|
||||
{ "id": "gelios.report_result.uses_template", "from": ["gelios.report_result"], "to": ["gelios.report_template"], "status": "source-evidenced", "summary": "A report result is produced from an approved report template and bounded request." },
|
||||
{ "id": "gelios.collection_run.checkpoints_with_cursor", "from": ["gelios.collection_run"], "to": ["gelios.ingestion_cursor"], "status": "product-required", "summary": "Collection runs advance resumable ingestion only through a durable cursor." },
|
||||
{ "id": "gelios.collection_run.collects_unit", "from": ["gelios.collection_run"], "to": ["gelios.unit"], "status": "product-required", "summary": "A read-only collection run records the approved unit scope it addressed." },
|
||||
{ "id": "gelios.unit.has_command_template", "from": ["gelios.unit"], "to": ["gelios.command_template"], "status": "source-evidenced", "summary": "Command templates are catalogued per unit through read-only routes." },
|
||||
{ "id": "gelios.command_template.belongs_to_group", "from": ["gelios.command_template"], "to": ["gelios.command_group"], "status": "source-evidenced", "summary": "A command template can belong to a provider command group." },
|
||||
{ "id": "gelios.command_dispatch.uses_template", "from": ["gelios.command_dispatch"], "to": ["gelios.command_template"], "status": "product-required", "summary": "A dispatch may reference a reviewed command template." },
|
||||
{ "id": "gelios.command_dispatch.targets_scope", "from": ["gelios.command_dispatch"], "to": ["gelios.unit", "gelios.unit_group", "gelios.access_scope"], "status": "product-required", "summary": "A dispatch target must resolve to an approved unit or explicitly approved group in scope." },
|
||||
{ "id": "gelios.command_dispatch.has_delivery", "from": ["gelios.command_dispatch"], "to": ["gelios.command_delivery"], "status": "product-required", "summary": "A dispatch is associated with delivery/task state rather than assumed successful on request creation." },
|
||||
{ "id": "gelios.command_dispatch.has_audit", "from": ["gelios.command_dispatch"], "to": ["gelios.command_audit"], "status": "product-required", "summary": "Every dispatch requires immutable user-confirmation and outcome audit." },
|
||||
{ "id": "gelios.unit.is_map_moving_object", "from": ["gelios.unit"], "to": ["map.moving_object"], "status": "product-required", "summary": "An approved Gelios unit is rendered through the provider-neutral moving-object contract, never as a Cesium entity identity." },
|
||||
{ "id": "gelios.position_fix.positions_map_subject", "from": ["gelios.position_fix"], "to": ["map.moving_object"], "status": "product-required", "summary": "A normalized position fix updates the stable map subject bound to the Gelios unit." },
|
||||
{ "id": "gelios.geozone.is_map_zone", "from": ["gelios.geozone"], "to": ["map.zone"], "status": "product-required", "summary": "An approved Gelios geozone is exposed to the map through the generic zone contract." },
|
||||
{ "id": "gelios.geopoint.is_map_place_target", "from": ["gelios.geopoint"], "to": ["map.place_target"], "status": "product-required", "summary": "An approved Gelios geopoint is exposed through a provider-neutral place target." }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"aliases": [
|
||||
{"alias":"коннектор","canonicalId":"integration.connection","language":"ru","status":"product-required"},
|
||||
{"alias":"connector","canonicalId":"integration.connection","language":"en","status":"product-required"},
|
||||
{"alias":"внешний провайдер","canonicalId":"integration.provider","language":"ru","status":"product-required"},
|
||||
{"alias":"provider","canonicalId":"integration.provider","language":"en","status":"product-required"},
|
||||
{"alias":"источник данных","canonicalId":"integration.provider","language":"ru","status":"product-required"}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"entities": [
|
||||
{"id":"integration.provider","name":"External Provider","surface":"integration","status":["product-required"],"authority":"Platform External Provider Data Plane","summary":"A third-party product or API family integrated through an app-owned adapter, not a tenant or a renderer."},
|
||||
{"id":"integration.connection","name":"Provider Connection","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"A tenant-scoped configured connection to one provider account or endpoint; it contains policy and secret references, never secret values."},
|
||||
{"id":"integration.credential_reference","name":"Provider Credential Reference","surface":"integration","status":["product-required"],"authority":"Provider Gateway / secret store","summary":"Opaque reference to a server-side secret or token lifecycle, not a token, login or password."},
|
||||
{"id":"integration.access_scope","name":"Integration Access Scope","surface":"integration","status":["product-required"],"authority":"Client owner + Provider Gateway","summary":"Approved product boundary for source accounts, objects, fields and permitted operations; provider visibility alone is insufficient."},
|
||||
{"id":"integration.capability","name":"Provider Capability","surface":"integration","status":["product-required"],"authority":"Provider Adapter","summary":"Versioned provider operation or feature classified as read, metadata, write, destructive or unknown."},
|
||||
{"id":"integration.field_definition","name":"Provider Field Definition","surface":"integration","status":["product-required"],"authority":"Provider Adapter + Ontology Core","summary":"A documented source field, its semantics, sensitivity, units and normalisation mapping; it is not a live value."},
|
||||
{"id":"integration.collection_profile","name":"Collection Profile","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Approved rate, paging, cursor, capability and field selection for a connection; prevents uncontrolled account-wide polling."},
|
||||
{"id":"integration.retention_policy","name":"Integration Retention Policy","surface":"integration","status":["product-required"],"authority":"Client owner + Provider Gateway","summary":"Lifecycle rule for current state, raw envelope, normalised history, aggregates and backups."},
|
||||
{"id":"integration.collection_run","name":"Integration Collection Run","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Auditable bounded execution of a safe-read collection profile with cursor, response metrics and outcome."},
|
||||
{"id":"integration.raw_envelope","name":"Raw Provider Envelope","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Restricted provenance-preserving representation of a safe-read provider response, stored in a bounded cold layer rather than exposed to UI."},
|
||||
{"id":"integration.canonical_subject","name":"Canonical Integrated Subject","surface":"integration","status":["product-required"],"authority":"Provider Domain Adapter","summary":"Stable domain subject normalised from a provider object and suitable for relations, analytics and interfaces."},
|
||||
{"id":"integration.read_model","name":"Integration Read Model","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Scoped, versioned current, history or aggregate projection delivered to approved consumers without raw provider bypass."},
|
||||
{"id":"integration.realtime_channel","name":"Integration Realtime Channel","surface":"integration","status":["product-required"],"authority":"Provider Gateway","summary":"Internal bounded event or subscription contract for projection changes; its emission rate is independent from provider collection rate."},
|
||||
{"id":"integration.command_capability","name":"Provider Command Capability","surface":"integration","status":["product-required"],"authority":"Provider Adapter","summary":"Catalogued command metadata, parameters and risk classification. It is not permission or a send route."},
|
||||
{"id":"integration.command_intent","name":"Provider Command Intent","surface":"integration","status":["future-concept"],"authority":"Future Provider Command Gateway","summary":"A future explicitly confirmed, scoped and idempotent request to execute a catalogued command; no read collector may create it."},
|
||||
{"id":"integration.command_audit","name":"Provider Command Audit","surface":"integration","status":["future-concept"],"authority":"Future Provider Command Gateway","summary":"Immutable approval, execution and delivery audit for a red-domain command lifecycle."}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"sourceRoots": ["platform/docs", "platform/packages", "platform/services"],
|
||||
"ledgers": [
|
||||
{
|
||||
"id": "external_provider_data_plane_adr",
|
||||
"path": "../../docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md",
|
||||
"entityIds": ["integration.provider", "integration.connection", "integration.access_scope", "integration.capability", "integration.collection_profile", "integration.read_model", "integration.command_capability"]
|
||||
}
|
||||
],
|
||||
"baselineDocs": ["../../docs/ADR_EXTERNAL_PROVIDER_DATA_PLANE.md", "../../packages/external-provider-contract/README.md"],
|
||||
"restrictions": ["The package is a semantic contract. Provider secrets, runtime data and command transport remain outside Ontology Core."]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"rules": [
|
||||
{"id":"guardrail.integration.ontology_is_not_data_plane","severity":"error","summary":"Ontology Core may describe providers and contracts but must not store secret values, live payloads, tenant telemetry, raw envelopes or provider execution state.","entityIds":["integration.provider","integration.connection","integration.credential_reference","integration.raw_envelope","integration.read_model"]},
|
||||
{"id":"guardrail.integration.scope_precedes_collection","severity":"error","summary":"A collection run must resolve tenant scope, field policy and bounded collection profile before it addresses a provider. Account visibility does not define product scope.","entityIds":["integration.connection","integration.access_scope","integration.collection_profile","integration.collection_run"]},
|
||||
{"id":"guardrail.integration.read_model_not_raw_bypass","severity":"error","summary":"L2 workflows, interfaces and renderer adapters consume scoped read models or realtime channels, never provider credentials, raw envelopes or direct adapter database access.","entityIds":["integration.credential_reference","integration.raw_envelope","integration.read_model","integration.realtime_channel"]},
|
||||
{"id":"guardrail.integration.commands_are_separate_red_domain","severity":"error","summary":"Cataloguing a command does not activate it. Read collectors, L2 workflows, maps and AI Workspace may not create command intent or delivery. A later command gateway requires explicit human confirmation, scope, idempotency and audit.","entityIds":["integration.command_capability","integration.command_intent","integration.command_audit","integration.collection_run"]},
|
||||
{"id":"guardrail.integration.full_catalog_not_full_rate","severity":"warning","summary":"Every provider field may be catalogued, but enabling continuous collection requires an explicit collection, retention and rate policy. Do not turn a capability catalog into unbounded account polling.","entityIds":["integration.capability","integration.field_definition","integration.collection_profile","integration.retention_policy"]}
|
||||
],
|
||||
"blockedConflations": [
|
||||
["integration.provider","integration.connection"],
|
||||
["integration.connection","integration.credential_reference"],
|
||||
["integration.raw_envelope","integration.read_model"],
|
||||
["integration.command_capability","integration.command_intent"],
|
||||
["integration.collection_run","integration.command_intent"]
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"id": "integration",
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"status": "product-required/source-evidenced",
|
||||
"summary": "Provider-neutral external integration ontology: tenant-scoped connections, capabilities, collection, controlled storage projections and a separated red command domain."
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"version": "0.1.0",
|
||||
"updatedAt": "2026-07-13",
|
||||
"relations": [
|
||||
{"id":"integration.provider.offers_connection","from":["integration.provider"],"to":["integration.connection"],"status":"product-required","summary":"A provider is connected through one or more tenant-scoped connection instances."},
|
||||
{"id":"integration.connection.references_credential","from":["integration.connection"],"to":["integration.credential_reference"],"status":"product-required","summary":"A connection uses an opaque server-side credential reference rather than exposing a secret."},
|
||||
{"id":"integration.connection.enforces_scope","from":["integration.connection"],"to":["integration.access_scope"],"status":"product-required","summary":"Every source request is bounded by an approved product scope."},
|
||||
{"id":"integration.provider.declares_capability","from":["integration.provider"],"to":["integration.capability","integration.command_capability"],"status":"product-required","summary":"Adapter manifests catalogue source read and command capabilities independently from their activation."},
|
||||
{"id":"integration.capability.exposes_field","from":["integration.capability"],"to":["integration.field_definition"],"status":"product-required","summary":"A safe-read capability documents the fields it may return before a collection profile enables them."},
|
||||
{"id":"integration.collection_profile.selects_capability","from":["integration.collection_profile"],"to":["integration.capability","integration.field_definition"],"status":"product-required","summary":"A profile selects bounded capabilities and fields rather than polling every API feature continuously."},
|
||||
{"id":"integration.collection_profile.applies_retention","from":["integration.collection_profile"],"to":["integration.retention_policy"],"status":"product-required","summary":"Collection and storage lifecycle are explicit per connection profile."},
|
||||
{"id":"integration.collection_run.uses_connection","from":["integration.collection_run"],"to":["integration.connection","integration.collection_profile"],"status":"product-required","summary":"A collection run records which connection and bounded policy it used."},
|
||||
{"id":"integration.collection_run.records_raw_envelope","from":["integration.collection_run"],"to":["integration.raw_envelope"],"status":"product-required","summary":"Safe-read response provenance is retained only according to the active policy."},
|
||||
{"id":"integration.raw_envelope.normalizes_subject","from":["integration.raw_envelope"],"to":["integration.canonical_subject"],"status":"product-required","summary":"Provider payloads are normalised into stable domain subjects before consumption."},
|
||||
{"id":"integration.read_model.projects_subject","from":["integration.read_model"],"to":["integration.canonical_subject"],"status":"product-required","summary":"Consumers receive a scoped projection rather than a provider payload or direct database access."},
|
||||
{"id":"integration.realtime_channel.emits_read_model","from":["integration.realtime_channel"],"to":["integration.read_model"],"status":"product-required","summary":"Realtime delivery emits approved projection changes with independently controlled sampling."},
|
||||
{"id":"integration.command_intent.uses_capability","from":["integration.command_intent"],"to":["integration.command_capability"],"status":"future-concept","summary":"A future command intent may select only a catalogued command capability."},
|
||||
{"id":"integration.command_intent.has_audit","from":["integration.command_intent"],"to":["integration.command_audit"],"status":"future-concept","summary":"Every future command lifecycle requires immutable approval and delivery audit."}
|
||||
]
|
||||
}
|
||||
@@ -9,7 +9,7 @@
|
||||
{ "id": "map.grid_layer", "name": "Map Grid Layer", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Planetary or situational grid with configurable modes, levels of detail, step, radius and visual primitives." },
|
||||
{ "id": "map.place_target", "name": "Map Place Target", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Geographic place target such as a city or territory with location, pulse, radius and distance-dependent presentation." },
|
||||
{ "id": "map.moving_object", "name": "Map Moving Object", "surface": "map", "status": ["source-evidenced"], "authority": "NDC domain source", "summary": "Dynamic spatial object such as transport, robot or vehicle with identity, position, movement and status." },
|
||||
{ "id": "map.pin", "name": "Map Pin", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Reusable spatial pin presentation with stem, point, height, color, status and visibility rules." },
|
||||
{ "id": "map.pin", "name": "Map Pin", "surface": "map", "status": ["source-evidenced"], "authority": "NDC Module Studio", "summary": "Reusable provider-neutral elevated-spike presentation with ground anchor, stem, head, label anchor, semantic colour/status and camera-height visibility rules." },
|
||||
{ "id": "map.label", "name": "Map Label", "surface": "map", "status": ["source-evidenced", "product-required"], "authority": "NDC Module Studio", "summary": "Reusable information plate attached to a spatial subject with style, size variant, anchor, offset and visibility rules." },
|
||||
{ "id": "map.zone", "name": "Map Zone", "surface": "map", "status": ["source-evidenced"], "authority": "NDC domain source", "summary": "Polygon or multipolygon zone, sector or geofence with optional height, extrusion and level-dependent style." },
|
||||
{ "id": "map.route", "name": "Map Route", "surface": "map", "status": ["source-evidenced"], "authority": "NDC domain source", "summary": "Logical ordered path or service route independent of renderer geometry." },
|
||||
|
||||
@@ -81,14 +81,14 @@
|
||||
{ "id": "engine.workflow_owner", "name": "ENGINE Workflow Owner", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE", "summary": "Owner field in ACL." },
|
||||
{ "id": "engine.workflow_share", "name": "ENGINE Workflow Share", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE", "summary": "Share/invite/user access routes." },
|
||||
{ "id": "engine.workflow_access_request", "name": "ENGINE Workflow Access Request", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE", "summary": "Access request route/model for workflow access." },
|
||||
{ "id": "engine.workflow_l2", "name": "ENGINE L2 Workflow", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE/n8n bridge", "summary": "n8n subworkflow attached to L1 node." },
|
||||
{ "id": "engine.node_l2", "name": "ENGINE L2 Node", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE/n8n bridge", "summary": "Node inside compiled/deployed n8n workflow." },
|
||||
{ "id": "engine.l2_runtime_core", "name": "ENGINE L2 Runtime Core", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE/n8n bridge", "summary": "Target n8n runtime/core instance." },
|
||||
{ "id": "engine.workflow_l2_runtime_id", "name": "ENGINE L2 Runtime Workflow ID", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE/n8n bridge", "summary": "n8n runtime workflow id, separate from NodeDC workflow id." },
|
||||
{ "id": "engine.l2_execution", "name": "ENGINE L2 Execution", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE/n8n bridge", "summary": "n8n execution id mapped to run/execution records." },
|
||||
{ "id": "engine.workflow_l2", "name": "ENGINE L2 Workflow", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE execution bridge", "summary": "L2 execution workflow attached to an L1 node." },
|
||||
{ "id": "engine.node_l2", "name": "ENGINE L2 Node", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE execution bridge", "summary": "Node inside a compiled/deployed L2 workflow." },
|
||||
{ "id": "engine.l2_runtime_core", "name": "ENGINE L2 Runtime Core", "surface": "engine", "status": ["source-evidenced"], "authority": "ENGINE execution bridge", "summary": "Target Engine execution core." },
|
||||
{ "id": "engine.workflow_l2_runtime_id", "name": "ENGINE L2 Runtime Workflow ID", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE execution bridge", "summary": "L2 runtime workflow id, separate from the L1 workflow id." },
|
||||
{ "id": "engine.l2_execution", "name": "ENGINE L2 Execution", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE execution bridge", "summary": "L2 execution id mapped to run/execution records." },
|
||||
{ "id": "engine.l2_run", "name": "ENGINE L2 Run", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE-side OpsLayer", "summary": "Runtime run id; tech-debt-adjacent." },
|
||||
{ "id": "engine.l2_session", "name": "ENGINE L2 Session", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE-side OpsLayer", "summary": "Runtime session anchor." },
|
||||
{ "id": "engine.l2_runtime_event", "name": "ENGINE L2 Runtime Event", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE/n8n bridge", "summary": "Runtime event: workflow/node start/finish/success/error." },
|
||||
{ "id": "engine.l2_runtime_event", "name": "ENGINE L2 Runtime Event", "surface": "engine", "status": ["source-confirmed"], "authority": "ENGINE execution bridge", "summary": "Runtime event: workflow/node start/finish/success/error." },
|
||||
{ "id": "engine.source_stamp", "name": "ENGINE Source Stamp", "surface": "engine", "status": ["pending"], "authority": "ENGINE", "summary": "Future source/version traceability candidate." },
|
||||
|
||||
{ "id": "assistant.assistant", "name": "Assistant", "surface": "assistant", "status": ["product-required", "pending"], "authority": "AI Workspace", "summary": "User-facing assistant capability/persona." },
|
||||
|
||||
@@ -49,10 +49,10 @@
|
||||
{ "id": "engine.workflow_acl.has_owner", "from": ["engine.workflow_acl"], "to": ["engine.workflow_owner"], "status": "source-confirmed", "summary": "ACL owns owner field." },
|
||||
{ "id": "engine.workflow_l1.has_share", "from": ["engine.workflow_l1"], "to": ["engine.workflow_share"], "status": "source-confirmed", "summary": "Workflow can be shared/invited." },
|
||||
{ "id": "engine.workflow_l1.has_access_request", "from": ["engine.workflow_l1"], "to": ["engine.workflow_access_request"], "status": "source-confirmed", "summary": "Access requests target workflow." },
|
||||
{ "id": "engine.node_l1.embeds_l2_workflow", "from": ["engine.node_l1"], "to": ["engine.workflow_l2"], "status": "source-evidenced", "summary": "n8n subworkflow is attached to L1 node." },
|
||||
{ "id": "engine.workflow_l2.deployed_as_runtime_workflow", "from": ["engine.workflow_l2"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "Compiled n8n workflow deploy returns runtime workflow id." },
|
||||
{ "id": "engine.workflow_l2.runs_on_runtime_core", "from": ["engine.workflow_l2"], "to": ["engine.l2_runtime_core"], "status": "source-evidenced", "summary": "L2 workflow targets n8n instance/core." },
|
||||
{ "id": "engine.l2_execution.belongs_to_runtime_workflow", "from": ["engine.l2_execution"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "Execution is emitted for n8n workflow id." },
|
||||
{ "id": "engine.node_l1.embeds_l2_workflow", "from": ["engine.node_l1"], "to": ["engine.workflow_l2"], "status": "source-evidenced", "summary": "L2 execution workflow is attached to an L1 node." },
|
||||
{ "id": "engine.workflow_l2.deployed_as_runtime_workflow", "from": ["engine.workflow_l2"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "L2 deploy returns a runtime workflow id." },
|
||||
{ "id": "engine.workflow_l2.runs_on_runtime_core", "from": ["engine.workflow_l2"], "to": ["engine.l2_runtime_core"], "status": "source-evidenced", "summary": "L2 workflow targets the Engine execution core." },
|
||||
{ "id": "engine.l2_execution.belongs_to_runtime_workflow", "from": ["engine.l2_execution"], "to": ["engine.workflow_l2_runtime_id"], "status": "source-confirmed", "summary": "Execution is emitted for the L2 runtime workflow id." },
|
||||
{ "id": "engine.l2_runtime_event.describes_execution", "from": ["engine.l2_runtime_event"], "to": ["engine.l2_execution"], "status": "source-confirmed", "summary": "Runtime events carry execution id." },
|
||||
{ "id": "engine.l2_runtime_event.binds_run", "from": ["engine.l2_runtime_event"], "to": ["engine.l2_run"], "status": "source-confirmed", "summary": "Event binds to run id. Current storage is ENGINE-side OpsLayer." },
|
||||
{ "id": "engine.l2_run.has_session", "from": ["engine.l2_run"], "to": ["engine.l2_session"], "status": "source-confirmed", "summary": "Session anchors related runtime events/runs." },
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"relationId": "ontology.resolves_ops_card_to_engine_context",
|
||||
"fromSurface": "ops",
|
||||
"inputEntityIds": ["ops.card", "ops.project"],
|
||||
"intentHints": ["engine", "workflow", "l1", "l2", "n8n", "inspect", "switch", "runtime", "докодим", "воркфлоу"],
|
||||
"intentHints": ["engine", "workflow", "l1", "l2", "inspect", "switch", "runtime", "докодим", "воркфлоу"],
|
||||
"outputSurface": "engine",
|
||||
"outputEntityIds": ["engine.workflow_l1", "engine.node_l1", "engine.workflow_l2", "engine.workflow_l2_runtime_id"],
|
||||
"requiredBindings": [
|
||||
|
||||
@@ -79,7 +79,7 @@ If L2 runtime is known, add:
|
||||
```json
|
||||
{
|
||||
"runtime_workflow_id": "...",
|
||||
"n8n_instance_id": "..."
|
||||
"runtime_instance_id": "..."
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
# Gelios Domain Ontology
|
||||
|
||||
Package: `catalog/domain-packages/gelios`
|
||||
|
||||
Status: `v0.1.0` source-evidenced and product-required slice for the Robot2B client context and the Gelios provider.
|
||||
|
||||
## Purpose and boundary
|
||||
|
||||
The package describes the meaning of Gelios fleet, telemetry, sensor, spatial, report and command concepts. It is not a Gelios client, token store, telemetry database, Cesium implementation or Engine workflow.
|
||||
|
||||
The practical audit established a wide REST read surface: 107 currently visible units, 105 with `lastMsg`, a visible administrative account, unit groups, telemetry variants, sensor catalogues, command-template catalogues, geozone groups and report templates. The REST OpenAPI documents 295 operations. This ontology therefore separates three things that must not be conflated:
|
||||
|
||||
1. documented provider capability;
|
||||
2. actual runtime read evidence for the current account;
|
||||
3. approved Robot2B product scope.
|
||||
|
||||
The 95-unit legacy Engine snapshot is a temporary compatibility reference only. It is not the final business scope. The provider's current groups also do not resolve that scope automatically. A Robot2B owner must approve an allowlist or equivalent rule represented by `gelios.access_scope`.
|
||||
|
||||
## Canonical value contracts
|
||||
|
||||
### Unit and device
|
||||
|
||||
`gelios.unit` is the stable business subject. The durable key is a namespaced provider identity such as `gelios.unit:<sourceUnitId>`, never a renderer entity ID or a display name. A unit can use a `gelios.tracker_device`, belong to multiple `gelios.unit_group` records and expose sensor, fuel, maintenance and custom-field configurations.
|
||||
|
||||
Hardware IDs, IMEI, phones and decrypt-related fields are restricted or secret-class data. They are not part of the default Studio contract.
|
||||
|
||||
### Current telemetry
|
||||
|
||||
`gelios.telemetry_snapshot` is the normalized current state produced by the Gateway, with at least:
|
||||
|
||||
```text
|
||||
unitSubjectId
|
||||
observedAt
|
||||
receivedAt
|
||||
position: { latitude, longitude, height?, course?, speed?, satellites? }
|
||||
operationalStatus
|
||||
sensorReadings[]
|
||||
counterValues[]
|
||||
qualityFlags[]
|
||||
source: gelios-rest
|
||||
```
|
||||
|
||||
The source supports a raw and a pure last-message variant. Pure data may include sensors, counters and accumulated fields. Raw messages and dynamic `params` remain behind restricted policy and may be retained separately for diagnostics only. `gelios.position_fix` is the time-qualified position portion used by spatial consumers; it is not itself a pin.
|
||||
|
||||
### Sensors and operational semantics
|
||||
|
||||
`gelios.sensor_definition`, `gelios.sensor_conversion` and `gelios.sensor_reading` separate stable device configuration, calibration and time-series readings. Fuel profile, maintenance plan and custom field are independent configuration concepts. No assumed sensor meaning should be added to analytics until its message parameter, measure and conversion semantics are approved.
|
||||
|
||||
### Spatial binding and Cesium pins
|
||||
|
||||
Gelios is a source domain; Cesium is a renderer adapter. The binding is deliberately provider-neutral:
|
||||
|
||||
```text
|
||||
gelios.unit
|
||||
-> map.moving_object (stable spatial subject)
|
||||
gelios.position_fix
|
||||
-> map.moving_object (current position update)
|
||||
map.moving_object
|
||||
-> map.pin + map.label + map.visibility_rule
|
||||
-> map.renderer_adapter (Cesium today, replaceable later)
|
||||
```
|
||||
|
||||
The future map payload selects a stable `subjectId`, position, normalized operational status, observed time and approved label fields. It must not contain Gelios credentials, raw payloads or Cesium transient entity references. This is the replacement boundary for the current M-map node-to-Cesium coupling.
|
||||
|
||||
The Map Page resolves the selected Studio profile to the shared map.pin contract. The initial accepted presentation is an elevated spike — ground anchor, stem, outlined head and label — but that visual configuration remains on the Studio side. Gelios publishes data for a stable subject; it never publishes Cesium primitive settings, pin pixels or a renderer entity ID.
|
||||
|
||||
The proposed renderer-neutral payload is in [`examples/gelios-map-moving-object.fixture.json`](../examples/gelios-map-moving-object.fixture.json). It uses synthetic coordinates only: it specifies the interface between Gateway and Map View, not a live trike position and not a Cesium entity.
|
||||
|
||||
### Commands
|
||||
|
||||
`gelios.command_template` and `gelios.command_group` are read-only catalogue concepts. `gelios.command_dispatch`, `gelios.command_delivery` and `gelios.command_audit` are a separate red domain.
|
||||
|
||||
No collection run, workflow, map click or autonomous agent may create a dispatch. A future dispatch requires an explicit human action, approved scope, selected unit/group and template/parameters, confirmation, an audit record and delivery-state reconciliation. This ontology package does not grant or test a write permission.
|
||||
|
||||
## Future data plane
|
||||
|
||||
The intended flow is:
|
||||
|
||||
```text
|
||||
Gelios REST OAuth
|
||||
-> server-side Gelios Gateway
|
||||
-> scope filter + field policy + normalizer
|
||||
-> durable telemetry/configuration storage
|
||||
-> NDC level-2 workflow / internal data contract
|
||||
-> Map View binding / Cesium renderer adapter
|
||||
```
|
||||
|
||||
`gelios.collection_run` and `gelios.ingestion_cursor` define collection governance independently of the transport implementation. They support bounded incremental history ingestion later: one approved unit scope, limited time window, field whitelist, volume limit and durable checkpoint. They do not imply that all history should be collected now.
|
||||
|
||||
Database technology is intentionally not chosen by Ontology Core. The proposed platform data-plane boundary and its validation gates are documented in [`platform/docs/ADR_GELIOS_DATA_PLANE.md`](../../../docs/ADR_GELIOS_DATA_PLANE.md). The ontology supplies the record boundaries needed by that ADR.
|
||||
|
||||
## Guardrails
|
||||
|
||||
- Ontology Core never stores secrets, runtime snapshots, full raw telemetry or large geometry datasets.
|
||||
- Current provider-account visibility is not the approved Robot2B scope; collection must enforce an owner-approved scope before every run.
|
||||
- GET alone does not make a route safe: login-as, temporary tokens, configuration and WLN download are excluded from telemetry collection.
|
||||
- Geozones require paging, filtering, volume controls and map LOD.
|
||||
- Command send and mutation routes remain red even if the account has documented access.
|
||||
- Map selection and pins target stable domain subjects, not provider payload IDs or renderer objects.
|
||||
|
||||
## Evidence and next implementation decisions
|
||||
|
||||
Evidence is limited to official REST OpenAPI, safe GET results, historical Engine donor inspection and the MAP package. The next decisions outside this ontology package are:
|
||||
|
||||
1. owner-approved Robot2B allowlist/rule;
|
||||
2. field policy for Studio, analytics and raw retention;
|
||||
3. data-plane ADR for realtime telemetry and history;
|
||||
4. Gelios Gateway implementation boundary and deployment location;
|
||||
5. fixture and renderer-adapter contract for Map View pins and labels.
|
||||
@@ -30,6 +30,33 @@ NDC domain source
|
||||
|
||||
Cesium is one possible renderer adapter. A future Cesium version or another provider can replace it without changing Map View, application manifests or domain subject identifiers.
|
||||
|
||||
## Interface semantic ownership
|
||||
|
||||
Interface entities belong in this shared Ontology Core, not in a parallel Studio-specific ontology:
|
||||
|
||||
| Layer | Owns |
|
||||
| --- | --- |
|
||||
| Ontology Core | Canonical meanings and relations: future.interface_view, future.interface_widget, future.interface_binding, map.view, map.moving_object, map.pin, map.label, map.visibility_rule and map.renderer_adapter. |
|
||||
| NDC Module Studio | React implementation, saved visual profiles, page templates, application manifests and the concrete layout chosen by an application owner. |
|
||||
| Engine / NDC workflow | Source integration, normalization and emission of stable subject IDs, current positions, statuses and approved display fields. |
|
||||
| Platform Map Gateway | Provider tokens, cache, proxying and the renderer runtime boundary. |
|
||||
|
||||
Ontology Core never becomes a UI database. It does not store a user's live page layout, camera snapshot, provider token, renderer object ID, telemetry feed or tile cache.
|
||||
|
||||
## Pin presentation contract v0.1
|
||||
|
||||
map.pin is a reusable interface semantic, not a Cesium entity. The initial canonical variant is elevated-spike, based on the accepted Gelios donor:
|
||||
|
||||
~~~text
|
||||
ground anchor
|
||||
-> coloured stem at a semantic height
|
||||
-> outlined point/head
|
||||
-> shared map.label with explicit anchor and offset
|
||||
-> separate pin and label visibility thresholds
|
||||
~~~
|
||||
|
||||
Studio resolves a selected map.pin style profile into this contract. The renderer adapter decides whether its concrete implementation is a polyline and point, a mesh, a sprite or another primitive. A source payload only identifies the stable moving subject, its position/status and approved label fields; it never specifies renderer primitives or provider-specific IDs.
|
||||
|
||||
## First acceptance slice
|
||||
|
||||
The first fixture-backed Map Page must cover:
|
||||
|
||||
@@ -112,14 +112,14 @@ ENGINE remains workflow/dev environment. It may reference OPS and assistants, bu
|
||||
| `engine.workflow_owner` | ENGINE Workflow Owner | source-confirmed | ENGINE | Owner field in ACL. |
|
||||
| `engine.workflow_share` | ENGINE Workflow Share | source-confirmed | ENGINE | Share/invite/user access routes. |
|
||||
| `engine.workflow_access_request` | ENGINE Workflow Access Request | source-confirmed | ENGINE | Access request route/model for workflow access. |
|
||||
| `engine.workflow_l2` | ENGINE L2 Workflow | source-evidenced | ENGINE / n8n bridge | n8n subworkflow attached to L1 node by `workflowId` + `nodeId`. |
|
||||
| `engine.node_l2` | ENGINE L2 Node | source-evidenced | ENGINE / n8n bridge | Node inside compiled/deployed n8n workflow. |
|
||||
| `engine.l2_runtime_core` | ENGINE L2 Runtime Core | source-evidenced | ENGINE / n8n bridge | Target n8n runtime/core instance. |
|
||||
| `engine.workflow_l2_runtime_id` | ENGINE L2 Runtime Workflow ID | source-confirmed | ENGINE / n8n bridge | n8n runtime workflow id, separate from NodeDC workflow id. |
|
||||
| `engine.l2_execution` | ENGINE L2 Execution | source-confirmed | ENGINE / n8n bridge | n8n execution id mapped to run/execution records. |
|
||||
| `engine.workflow_l2` | ENGINE L2 Workflow | source-evidenced | ENGINE execution bridge | L2 workflow attached to L1 node by `workflowId` + `nodeId`. |
|
||||
| `engine.node_l2` | ENGINE L2 Node | source-evidenced | ENGINE execution bridge | Node inside a compiled/deployed L2 workflow. |
|
||||
| `engine.l2_runtime_core` | ENGINE L2 Runtime Core | source-evidenced | ENGINE execution bridge | Target Engine execution core. |
|
||||
| `engine.workflow_l2_runtime_id` | ENGINE L2 Runtime Workflow ID | source-confirmed | ENGINE execution bridge | L2 runtime workflow id, separate from NodeDC L1 workflow id. |
|
||||
| `engine.l2_execution` | ENGINE L2 Execution | source-confirmed | ENGINE execution bridge | L2 execution id mapped to run/execution records. |
|
||||
| `engine.l2_run` | ENGINE L2 Run | source-confirmed | ENGINE-side OpsLayer | Runtime run id used to bind execution/session. Current implementation is tech-debt-adjacent. |
|
||||
| `engine.l2_session` | ENGINE L2 Session | source-confirmed | ENGINE-side OpsLayer | Runtime session anchor, often derived from run id if upstream omits session id. |
|
||||
| `engine.l2_runtime_event` | ENGINE L2 Runtime Event | source-confirmed | ENGINE / n8n bridge | `workflow:start`, `workflow:finish`, `node:start`, `node:success`, `node:error`. |
|
||||
| `engine.l2_runtime_event` | ENGINE L2 Runtime Event | source-confirmed | ENGINE execution bridge | `workflow:start`, `workflow:finish`, `node:start`, `node:success`, `node:error`. |
|
||||
| `engine.source_stamp` | ENGINE Source Stamp | pending | ENGINE | Candidate for future source/version traceability. |
|
||||
|
||||
## Assistant / AI Workspace
|
||||
@@ -161,4 +161,3 @@ These names identify current working areas that must not leak into canonical cor
|
||||
| `engine.tech_debt.tender_storage` | Tender Storage | tech-debt-noncanonical | ENGINE | Current domain storage in ENGINE-side implementation. |
|
||||
| `engine.tech_debt.tender_ai_review` | Tender AI Review | tech-debt-noncanonical | ENGINE | Current AI review implementation for tender domain. |
|
||||
| `engine.tech_debt.ops_tabs_control` | ENGINE OPS Tabs Control | tech-debt-noncanonical | ENGINE | UI/control surface tied to current OpsLayer experiment. |
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ runtime/data/storage/env/secrets/logs/dumps/node_modules/build outputs
|
||||
.env / .env.*
|
||||
credentials / keys / pem / ssh
|
||||
DB dumps / archives / backup contents
|
||||
postgres/n8n runtime data
|
||||
postgres/Engine L2 runtime data
|
||||
```
|
||||
|
||||
Не менять:
|
||||
@@ -600,7 +600,7 @@ search: /api/workflows/:id/share
|
||||
search: workflow-access-requests
|
||||
search: engine-role-access-requests
|
||||
|
||||
NODEDC/NODEDC_ENGINE_INFRA/nodedc-source/server/routes/n8n.js
|
||||
Engine execution deployment route
|
||||
search: runtimeWorkflowId
|
||||
search: resolveNodeDcByRuntimeWorkflowId
|
||||
search: resolveWorkflowIdByNodeDcBindingRest
|
||||
@@ -608,12 +608,12 @@ search: executionId
|
||||
|
||||
NODEDC/NODEDC_ENGINE_INFRA/nodedc-source/server/ops/runtimeTap.js
|
||||
search: runtimeWorkflowId
|
||||
search: n8nCoreId
|
||||
search: runtime core id
|
||||
search: executionId
|
||||
search: runId
|
||||
search: sessionId
|
||||
|
||||
NODEDC/NODEDC_ENGINE_INFRA/nodedc-source/services/n8n/runtime-plugin/hooks.js
|
||||
Engine execution runtime hooks
|
||||
search: workflow:start
|
||||
search: workflow:finish
|
||||
search: node:start
|
||||
@@ -641,7 +641,7 @@ source-evidenced by previous ENGINE pass; direct links pending.
|
||||
Риск смешения терминов:
|
||||
|
||||
```text
|
||||
n8n names are implementation aliases for Engine L2 runtime, not canonical product names.
|
||||
The Engine L2 runtime is the only canonical product term in ontology and documentation.
|
||||
```
|
||||
|
||||
### 3.9. ENGINE / Assistant P1 — Assistant, Provider, Executor, Bridge
|
||||
|
||||
@@ -13,7 +13,7 @@ This file indexes the source-backed evidence ledgers. It is not a replacement fo
|
||||
| `EVIDENCE_LEDGER_OPS_P0.md` | OPS Product | source-backed P0 | Confirms workspace/project/card model and `ops.card` as canonical work object. |
|
||||
| `EVIDENCE_LEDGER_OPS_GATEWAY_P1.md` | OPS Gateway / MCP access | source-backed P1 | Confirms scoped agent identity/token/grant/scope/tool/idempotency/audit boundary. |
|
||||
| `EVIDENCE_LEDGER_ENGINE_DIRTY_BOUNDARY_P1.md` | ENGINE-side OpsLayer / Agent Monitor / tender tech debt | source-backed P1 | Confirms this is non-canonical tech debt and must not define OPS Product. |
|
||||
| `EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md` | ENGINE L1/L2 workflow and runtime evidence | source-backed P1 | Confirms L1 graph, workflow ACL/share, L2 n8n runtime ids/events/run/session bridge. |
|
||||
| `EVIDENCE_LEDGER_ENGINE_WORKFLOW_P1.md` | ENGINE L1/L2 workflow and runtime evidence | source-backed P1 | Confirms L1 graph, workflow ACL/share, L2 runtime ids/events/run/session bridge. |
|
||||
|
||||
## Source Roots Used
|
||||
|
||||
@@ -36,7 +36,7 @@ This file indexes the source-backed evidence ledgers. It is not a replacement fo
|
||||
- ENGINE-side Agent Monitor/tender UI is working tech debt, not target architecture.
|
||||
- ENGINE L1/L2 split is real enough for first ontology implementation:
|
||||
- L1 = NodeDC canvas/workflow graph;
|
||||
- L2 = n8n/subworkflow runtime attached to L1 node.
|
||||
- L2 = Engine execution workflow attached to an L1 node.
|
||||
- Future interface layer is a planned platform service/layer, motivated by current ENGINE-side custom UI debt.
|
||||
|
||||
## Evidence Gaps That Remain Non-Blocking
|
||||
@@ -46,4 +46,3 @@ This file indexes the source-backed evidence ledgers. It is not a replacement fo
|
||||
- `ops.card_type` needs taxonomy.
|
||||
- Gateway pairing/entitlement flow needs route-level hardening.
|
||||
- Domain ontology packages need separate passes; core must not absorb domain-specific tender/ecology/transport concepts.
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ OPS product
|
||||
!= ENGINE-side OpsLayer / tender-agent / Agent Monitor
|
||||
```
|
||||
|
||||
The ENGINE repository contains a substantial embedded `OpsLayer` around n8n workflows, tender search/review, monitor profiles, agent runs, trace events, and Agent Monitor UI nodes.
|
||||
The ENGINE repository contains a substantial embedded `OpsLayer` around L2 execution workflows, tender search/review, monitor profiles, agent runs, trace events, and Agent Monitor UI nodes.
|
||||
|
||||
This code is working product/legacy infrastructure, but it must be treated as:
|
||||
|
||||
@@ -43,14 +43,14 @@ It should not be promoted into canonical OPS product truth and should not define
|
||||
| Boundary ID | Canonical treatment | Evidence status | Evidence file:line | Evidence explanation | Notes |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| `engine.tech_debt.ops_layer` | Engine-side legacy OpsLayer | source-confirmed tech debt | `server/index.js:25`, `server/index.js:333`, `server/routes/ops.js:1-34`, `server/routes/ops.js:6412-11855` | ENGINE imports `opsRouter` and mounts it at `/api/ops`; `routes/ops.js` implements a large set of run/result/tender/AI review/monitor endpoints. | This is not the separate OPS product / Tasker model. |
|
||||
| `engine.tech_debt.ops_agent_instance` | Legacy Engine agent instance | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:89-137`, `src/store.ts:444-480`, `src/utils/opsApi.ts:759-855` | OpsLayer has `agents`, `agent_instances`, `agent_runs`; frontend prevents copied n8n nodes from inheriting `opsAgentInstanceId`; API can start/stop agent instances and record runs. | Do not merge with OPS Gateway `agent.identity`. |
|
||||
| `engine.tech_debt.ops_agent_instance` | Legacy Engine agent instance | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:89-137`, `src/store.ts:444-480`, `src/utils/opsApi.ts:759-855` | OpsLayer has `agents`, `agent_instances`, `agent_runs`; frontend prevents copied L2 nodes from inheriting `opsAgentInstanceId`; API can start/stop agent instances and record runs. | Do not merge with OPS Gateway `agent.identity`. |
|
||||
| `engine.tech_debt.agent_monitor_node` | Agent Monitor canvas node | source-confirmed tech debt | `src/nodes/AgentMonitorNode.tsx:10-14`, `src/nodes/AgentMonitorNode.tsx:21-38`, `src/driveinspector/nodes/AgentMonitor.definition.ts:1-13` | ENGINE has a visible `agentMonitor` React Flow node and inspector definition that binds monitor profiles/agents through `opsApi`. | This is current UI embedding, not final interface layer. |
|
||||
| `engine.tech_debt.monitor_profile` | Engine-side monitor profile | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:240-299`, `src/utils/opsApi.ts:260-360`, `src/utils/opsApi.ts:460-520` | OpsLayer stores monitor profiles, profile assignments, bound agent instances, n8n cores, and monitor nodes; frontend can list/create/update/bind profiles and save monitor layouts. | Useful evidence for future work-view requirements. |
|
||||
| `engine.tech_debt.tender_domain_ui` | Tender-specific embedded UI/domain slice | source-confirmed tech debt | `server/routes/ops.js:45-63`, `server/routes/ops.js:75-145`, `src/n8n/N8nAgentMonitorPanel.tsx:1-44`, `src/n8n/N8nAgentMonitorPanel.tsx:1160-1245`, `src/n8n/tenderElectronicPlaceCountries.ts:1-12` | ENGINE includes Kontur tender URLs, tender categories/electronic places, tender search settings, and a large Agent Monitor panel. | Treat as procurement/tender domain evidence, not core ontology. |
|
||||
| `engine.tech_debt.monitor_profile` | Engine-side monitor profile | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:240-299`, `src/utils/opsApi.ts:260-360`, `src/utils/opsApi.ts:460-520` | OpsLayer stores monitor profiles, profile assignments, bound agent instances, execution cores, and monitor nodes; frontend can list/create/update/bind profiles and save monitor layouts. | Useful evidence for future work-view requirements. |
|
||||
| `engine.tech_debt.tender_domain_ui` | Tender-specific embedded UI/domain slice | source-confirmed tech debt | Engine internal monitor implementation | ENGINE includes Kontur tender URLs, tender categories/electronic places, tender search settings, and a large Agent Monitor panel. | Treat as procurement/tender domain evidence, not core ontology. |
|
||||
| `engine.tech_debt.tender_storage` | Tender storage inside Engine OpsLayer | source-confirmed tech debt | `server/ops/migrations/0001_init_opslayer.sql:210-239`, `server/ops/migrations/0006_tender_dedup_registry.sql:1-17`, `server/ops/migrations/0007_tender_details.sql:1-24`, `server/ops/migrations/0008_tender_documents.sql:1-37` | OpsLayer stores tender results, dedup registry, details, and downloaded documents. | Do not confuse with future domain ontology persistence. |
|
||||
| `engine.tech_debt.tender_ai_review` | Tender AI review workflow state | source-confirmed tech debt | `server/ops/migrations/0009_tender_ai_review.sql:1-90`, `src/n8n/N8nAgentMonitorPanel.tsx:1214-1238`, `src/utils/opsApi.ts:577-688` | OpsLayer stores AI review tasks/latest state; UI has default manual AI criteria/response format; API enqueues/retries review tasks. | Useful sample for domain-specific assistant review flows. |
|
||||
| `engine.tech_debt.tender_ai_review` | Tender AI review workflow state | source-confirmed tech debt | Engine internal monitor implementation | OpsLayer stores AI review tasks/latest state; UI has default manual AI criteria/response format; API enqueues/retries review tasks. | Useful sample for domain-specific assistant review flows. |
|
||||
| `engine.tech_debt.ops_tabs_control` | Engine inspector ops control | source-confirmed tech debt | `src/driveinspector/ControlRegistry.tsx:27-50` | Drive inspector registers `opsTabs` control. | Another marker that Ops UI is embedded in ENGINE editor controls. |
|
||||
| `future.interface_layer` | Future platform UI/work-view layer | product-required / source-informed | `src/n8n/N8nAgentMonitorPanel.tsx:46-120`, `src/utils/opsApi.ts:460-520`, `server/ops/migrations/0001_init_opslayer.sql:240-299` | Current monitor panel/layout/profile code shows the kind of configurable work views the future interface layer must support. | Future home should be separate platform service/app, not Engine core. |
|
||||
| `future.interface_layer` | Future platform UI/work-view layer | product-required / source-informed | Engine internal monitor implementation; `src/utils/opsApi.ts:460-520`, `server/ops/migrations/0001_init_opslayer.sql:240-299` | Current monitor panel/layout/profile code shows the kind of configurable work views the future interface layer must support. | Future home should be separate platform service/app, not Engine core. |
|
||||
|
||||
## Route / Surface Evidence
|
||||
|
||||
@@ -58,9 +58,9 @@ It should not be promoted into canonical OPS product truth and should not define
|
||||
| --- | --- | --- | --- |
|
||||
| `/api/ops` is mounted inside ENGINE server | source-confirmed | `server/index.js:25`, `server/index.js:333` | ENGINE server imports and mounts `opsRouter`. |
|
||||
| Ops router owns runs/results/tender/AI review/monitor APIs | source-confirmed | `server/routes/ops.js:6412-11855` | Router exposes health, runs, tender results/details/docs, AI review, trace, monitor profiles/layout/admin, agent instance start/stop, actions, contour limits/search. |
|
||||
| Agent Monitor is an ENGINE canvas node | source-confirmed | `src/nodes/AgentMonitorNode.tsx:10-14`, `src/nodes/AgentMonitorNode.tsx:73-170` | Node type is `agentMonitor`; UI opens monitor through n8n subworkflow event. |
|
||||
| Agent Monitor panel directly consumes Ops APIs | source-confirmed | `src/n8n/N8nAgentMonitorPanel.tsx:5-31` | Panel imports many Ops API functions for tender details, AI review, layout, runs, trace, start/stop, take-in-work. |
|
||||
| Engine graph copy logic special-cases Ops bindings | source-confirmed | `src/store.ts:444-480` | Copied n8n nodes clear `opsAgentInstanceId`; copied `agentMonitor` nodes clear monitor profile/binding fields. |
|
||||
| Agent Monitor is an ENGINE canvas node | source-confirmed | `src/nodes/AgentMonitorNode.tsx:10-14`, `src/nodes/AgentMonitorNode.tsx:73-170` | Node type is `agentMonitor`; UI opens monitor through an L2 execution event. |
|
||||
| Agent Monitor panel directly consumes Ops APIs | source-confirmed | Engine internal monitor implementation | Panel imports many Ops API functions for tender details, AI review, layout, runs, trace, start/stop, take-in-work. |
|
||||
| Engine graph copy logic special-cases Ops bindings | source-confirmed | `src/store.ts:444-480` | Copied L2 nodes clear `opsAgentInstanceId`; copied `agentMonitor` nodes clear monitor profile/binding fields. |
|
||||
|
||||
## Guardrails Confirmed
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Status: source-backed ledger section
|
||||
Baseline: Canonical Entity Catalog v0.4-pre
|
||||
Date: 2026-06-18
|
||||
Scope: ENGINE L1/L2 workflow, ACL/share, n8n runtime bridge, runtime event tracking
|
||||
Scope: ENGINE L1/L2 workflow, ACL/share, execution bridge, runtime event tracking
|
||||
|
||||
Source root:
|
||||
|
||||
@@ -25,7 +25,7 @@ This pass confirms the minimum ENGINE ontology needed for the first implementati
|
||||
- `engine.workflow_l1`, `engine.node_l1`, and `engine.edge_l1` exist as NodeDC/React Flow graph concepts.
|
||||
- `engine.node_type_l1` is sourced from the auto node registry.
|
||||
- Workflow ACL/share/access-request concepts are implemented in source.
|
||||
- `engine.workflow_l2` is source-evidenced as an n8n subworkflow attached to an L1 workflow/node pair.
|
||||
- `engine.workflow_l2` is source-evidenced as a protected execution workflow attached to an L1 workflow/node pair.
|
||||
- Runtime events carry `workflowId`, `nodeId`, `runtimeWorkflowId`, `executionId`, `runId`, and `sessionId`.
|
||||
- Current runtime run/session tracking is tied to ENGINE-side OpsLayer tables and should remain tech-debt-adjacent.
|
||||
|
||||
@@ -41,14 +41,14 @@ This pass confirms the minimum ENGINE ontology needed for the first implementati
|
||||
| `engine.workflow_owner` | source-confirmed | `server/workflows/acl.js:109-165` | ACL owner field is normalized and role checked. |
|
||||
| `engine.workflow_share` | source-confirmed | `server/index.js:3185-3245`, `server/index.js:3250-3312`, `server/index.js:3371-3635` | Workflow share routes expose owner/users/groups/invites and email/internal invite operations. |
|
||||
| `engine.workflow_access_request` | source-confirmed | `server/index.js:2792`, `server/index.js:2860`, `server/index.js:2916`, `server/index.js:3314` | Workflow access request, approve, reject, and request routes are present. |
|
||||
| `engine.workflow_l2` | source-evidenced | `server/routes/n8n.js:11661-11737` | `/api/n8n/subworkflow/deploy` loads subworkflow graph for `workflowId` + `nodeId`, compiles it, injects runtime events, and writes compiled n8n workflow. |
|
||||
| `engine.node_l2` | source-evidenced | `server/routes/n8n.js:2288-2365`, `services/n8n/runtime-plugin/hooks.js:132-145` | Runtime event node and hooks capture n8n node name/id. |
|
||||
| `engine.l2_runtime_core` | source-evidenced | `server/routes/n8n.js:11685-11696`, `server/routes/n8n.js:11739-11748` | Deploy resolves n8n target/base URL/instance and docker compose service/container. |
|
||||
| `engine.workflow_l2_runtime_id` | source-confirmed | `services/n8n/runtime-plugin/hooks.js:52-61`, `server/routes/n8n.js:2323-2324`, `server/routes/n8n.js:12041-12055` | Runtime plugin extracts n8n workflow id; injected event uses `$workflow.id`; deploy sync stores resolved workflow id. |
|
||||
| `engine.l2_execution` | source-confirmed | `services/n8n/runtime-plugin/hooks.js:77-86`, `server/routes/n8n.js:2323-2325`, `server/ops/runtimeTap.js:773-818`, `server/ops/runtimeTap.js:820-855` | Runtime events carry execution id; runtime tap writes trace event and run execution binding. |
|
||||
| `engine.l2_run` | source-confirmed / tech-debt-adjacent | `services/n8n/runtime-plugin/hooks.js:88-106`, `server/ops/runtimeTap.js:247-270`, `server/ops/runtimeTap.js:467-542` | Hooks extract run id; runtime tap resolves/canonicalizes run id through ENGINE-side OpsLayer tables. |
|
||||
| `engine.l2_session` | source-confirmed / tech-debt-adjacent | `services/n8n/runtime-plugin/hooks.js:108-130`, `server/ops/runtimeTap.js:272-289`, `server/ops/runtimeTap.js:354-430` | Hooks extract session id or fall back to run id; runtime tap resolves session anchors and can create session-workflow split runs. |
|
||||
| `engine.l2_runtime_event` | source-confirmed | `services/n8n/runtime-plugin/hooks.js:220-231`, `services/n8n/runtime-plugin/hooks.js:233-279`, `services/n8n/runtime-plugin/hooks.js:285-387`, `server/routes/n8n.js:2298-2332`, `services/n8n/runtime-plugin/runtime-bridge.js:200-223` | Runtime events are emitted/enqueued for workflow/node start/finish/success/error and sent to NodeDC runtime endpoint with retry/failure handling. |
|
||||
| `engine.workflow_l2` | source-evidenced | Engine execution deployment route | Deploy loads the L2 graph for `workflowId` + `nodeId`, compiles it, injects runtime events, and writes the execution workflow. |
|
||||
| `engine.node_l2` | source-evidenced | Engine execution hooks | Runtime event hooks capture L2 node name/id. |
|
||||
| `engine.l2_runtime_core` | source-evidenced | Engine execution deployment route | Deploy resolves the execution target/base URL/instance and service/container. |
|
||||
| `engine.workflow_l2_runtime_id` | source-confirmed | Engine execution hooks and deploy sync | Runtime bridge extracts workflow id; injected event uses `$workflow.id`; deploy sync stores the resolved id. |
|
||||
| `engine.l2_execution` | source-confirmed | Engine execution hooks and runtime tap | Runtime events carry execution id; runtime tap writes trace event and run execution binding. |
|
||||
| `engine.l2_run` | source-confirmed / tech-debt-adjacent | Engine execution hooks and runtime tap | Hooks extract run id; runtime tap resolves/canonicalizes it through ENGINE-side OpsLayer tables. |
|
||||
| `engine.l2_session` | source-confirmed / tech-debt-adjacent | Engine execution hooks and runtime tap | Hooks extract session id or fall back to run id; runtime tap resolves session anchors and can create session-workflow split runs. |
|
||||
| `engine.l2_runtime_event` | source-confirmed | Engine execution hooks and runtime bridge | Runtime events are emitted/enqueued for workflow/node start/finish/success/error and sent to NodeDC runtime endpoint with retry/failure handling. |
|
||||
|
||||
## Guardrails Confirmed
|
||||
|
||||
@@ -62,6 +62,5 @@ Engine-side OpsLayer runtime tables do not define OPS Product ontology.
|
||||
## Non-Blocking Follow-Up
|
||||
|
||||
- Harden `engine.source_stamp` after implementation needs are clearer.
|
||||
- Separate stable L2 domain concepts from current n8n-specific deployment helpers.
|
||||
- Separate stable L2 domain concepts from current execution deployment helpers.
|
||||
- Decide whether runtime run/session belongs in future Ontology Core, Interface Layer, OPS Gateway, or a dedicated runtime observability package.
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ These rules exist to stop future agents and developers from merging similar-look
|
||||
|
||||
- ENGINE is workflow/dev environment, not the final home for heavy product UIs.
|
||||
- `engine.workflow_l1` is the NodeDC canvas/workflow graph.
|
||||
- `engine.workflow_l2` is a runtime/subworkflow layer, currently n8n-backed.
|
||||
- `engine.workflow_l2` is the protected Engine execution layer.
|
||||
- `engine.workflow_l2_runtime_id` is not the same as NodeDC `engine.workflow_l1` id.
|
||||
- `engine.l2_run`, `engine.l2_session`, and current trace tables are useful evidence but are tech-debt-adjacent because they live in ENGINE-side OpsLayer.
|
||||
- Do not turn ENGINE-side Agent Monitor/tender UI into core ontology roots.
|
||||
@@ -74,4 +74,3 @@ These rules exist to stop future agents and developers from merging similar-look
|
||||
- Do not delete files unless explicitly asked by the user.
|
||||
- Do not edit application source code from the ontology workspace.
|
||||
- Do not run Docker/build/install/test for docs-only ontology work.
|
||||
|
||||
|
||||
@@ -68,10 +68,10 @@ Relations are written as product-level semantics, not database foreign keys. Imp
|
||||
| `engine.workflow_acl.has_owner` | `engine.workflow_acl` | `engine.workflow_owner` | ACL owns owner field. | source-confirmed |
|
||||
| `engine.workflow_l1.has_share` | `engine.workflow_l1` | `engine.workflow_share` | Workflow can be shared/invited. | source-confirmed |
|
||||
| `engine.workflow_l1.has_access_request` | `engine.workflow_l1` | `engine.workflow_access_request` | Access requests target workflow. | source-confirmed |
|
||||
| `engine.node_l1.embeds_l2_workflow` | `engine.node_l1` | `engine.workflow_l2` | n8n subworkflow is attached to L1 node. | source-evidenced |
|
||||
| `engine.workflow_l2.deployed_as_runtime_workflow` | `engine.workflow_l2` | `engine.workflow_l2_runtime_id` | Compiled n8n workflow deploy returns runtime workflow id. | source-confirmed |
|
||||
| `engine.workflow_l2.runs_on_runtime_core` | `engine.workflow_l2` | `engine.l2_runtime_core` | L2 workflow targets n8n instance/core. | source-evidenced |
|
||||
| `engine.l2_execution.belongs_to_runtime_workflow` | `engine.l2_execution` | `engine.workflow_l2_runtime_id` | Execution is emitted for n8n workflow id. | source-confirmed |
|
||||
| `engine.node_l1.embeds_l2_workflow` | `engine.node_l1` | `engine.workflow_l2` | L2 execution workflow is attached to an L1 node. | source-evidenced |
|
||||
| `engine.workflow_l2.deployed_as_runtime_workflow` | `engine.workflow_l2` | `engine.workflow_l2_runtime_id` | L2 deploy returns a runtime workflow id. | source-confirmed |
|
||||
| `engine.workflow_l2.runs_on_runtime_core` | `engine.workflow_l2` | `engine.l2_runtime_core` | L2 workflow targets the Engine execution core. | source-evidenced |
|
||||
| `engine.l2_execution.belongs_to_runtime_workflow` | `engine.l2_execution` | `engine.workflow_l2_runtime_id` | Execution is emitted for the L2 runtime workflow id. | source-confirmed |
|
||||
| `engine.l2_runtime_event.describes_execution` | `engine.l2_runtime_event` | `engine.l2_execution` | Runtime events carry execution id. | source-confirmed |
|
||||
| `engine.l2_runtime_event.binds_run` | `engine.l2_runtime_event` | `engine.l2_run` | Event binds to run id. Current storage is ENGINE-side OpsLayer. | source-confirmed / tech-debt-adjacent |
|
||||
| `engine.l2_run.has_session` | `engine.l2_run` | `engine.l2_session` | Session anchors related runtime events/runs. | source-confirmed / tech-debt-adjacent |
|
||||
@@ -87,4 +87,3 @@ These are the first useful implementation targets for Ontology Core.
|
||||
| `ontology.resolves_hub_app_to_project_context` | `hub.application` | `ops.project` / `engine.workflow_l1` | Application selection can suggest project/workflow context. | product-required |
|
||||
| `ontology.selects_gateway_grant_context` | `assistant.bridge` | `agent.grant` | Ontology can help choose proper MCP/Gateway grant before tool calls. | product-required |
|
||||
| `ontology.routes_interface_view_to_domain_package` | `future.interface_view` | `future.domain_package` | Future UI layer renders domain-specific views from ontology-backed data. | future-concept |
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@ Date: 2026-06-18
|
||||
| Agent token | Opaque credential used by Gateway; not user identity. |
|
||||
| ENGINE | Workflow/dev environment. |
|
||||
| L1 workflow | NodeDC/React Flow canvas workflow. |
|
||||
| L2 workflow | Runtime/subworkflow layer, currently n8n-backed. |
|
||||
| Runtime workflow id | n8n runtime workflow id, separate from NodeDC L1 workflow id. |
|
||||
| L2 workflow | Protected Engine execution workflow layer. |
|
||||
| Runtime workflow id | Engine L2 runtime workflow id, separate from NodeDC L1 workflow id. |
|
||||
| Runtime event | Event emitted by L2 runtime: workflow/node start/finish/success/error. |
|
||||
| Assistant | Product assistant capability; exact source model pending AI Workspace pass. |
|
||||
| Ontology Core | Future platform service/module for canonical entities, relations, aliases, evidence, and resolver rules. |
|
||||
@@ -34,8 +34,8 @@ Date: 2026-06-18
|
||||
| Authentik user / OIDC subject / JWT subject | `ndcauth.identity` |
|
||||
| Launcher service / app | `hub.application` |
|
||||
| Application tile / app card | `hub.application_card` |
|
||||
| n8n workflow id | `engine.workflow_l2_runtime_id` |
|
||||
| n8n execution | `engine.l2_execution` |
|
||||
| runtime workflow id | `engine.workflow_l2_runtime_id` |
|
||||
| L2 execution | `engine.l2_execution` |
|
||||
| Engine-side OPS / Agent Monitor OPS | `engine.tech_debt.ops_layer`, not OPS Product |
|
||||
| Tender Agent Monitor UI | `engine.tech_debt.tender_domain_ui`, not future Interface Layer |
|
||||
|
||||
@@ -46,4 +46,3 @@ Date: 2026-06-18
|
||||
- Do not say "user" when the context needs HUB user, NDCAuth identity, OPS member, or Gateway agent identity.
|
||||
- Do not say "task" as the root concept when the object is an OPS card.
|
||||
- Do not treat current tender/Agent Monitor implementation as target architecture.
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
"id": "example-workflow-id",
|
||||
"label": "Example Engine Workflow",
|
||||
"nodeId": "example-node-id",
|
||||
"runtimeWorkflowId": "example-n8n-runtime-workflow-id",
|
||||
"n8nInstanceId": "example-n8n-instance"
|
||||
"runtimeWorkflowId": "example-runtime-workflow-id",
|
||||
"runtimeInstanceId": "example-engine-runtime-instance"
|
||||
},
|
||||
"context": {
|
||||
"status": "planned",
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"contractVersion": "0.1.0",
|
||||
"kind": "map.moving_object.current_position",
|
||||
"source": "gelios",
|
||||
"subjectId": "gelios.unit:example-unit-id",
|
||||
"observedAt": "2026-07-13T00:00:00.000Z",
|
||||
"receivedAt": "2026-07-13T00:00:03.000Z",
|
||||
"position": {
|
||||
"latitude": 55.000001,
|
||||
"longitude": 37.000001,
|
||||
"heightMeters": 0,
|
||||
"courseDegrees": 90,
|
||||
"speedKph": 0,
|
||||
"satellites": 0
|
||||
},
|
||||
"operationalStatus": "unknown",
|
||||
"display": {
|
||||
"label": "Robot2B unit",
|
||||
"visibilityProfile": "fleet-default"
|
||||
},
|
||||
"policy": {
|
||||
"scopeApproved": true,
|
||||
"fieldSet": "studio-current-position-v1"
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/mcp-server.mjs",
|
||||
"assistant:action": "node src/assistant-action-resolver.mjs",
|
||||
"assistant:caller": "node src/assistant-action-caller.mjs",
|
||||
"assistant:execute": "node src/assistant-action-executor.mjs",
|
||||
@@ -17,6 +18,7 @@
|
||||
"smoke:assistant-executor": "node src/assistant-action-executor.mjs --smoke",
|
||||
"smoke:assistant-actions": "node src/assistant-action-resolver.mjs --smoke",
|
||||
"smoke:assistant-policy": "node src/assistant-policy.mjs --smoke",
|
||||
"smoke:mcp": "node src/scripts/smoke-mcp.mjs",
|
||||
"smoke:resolver": "node src/resolver.mjs --smoke"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,14 @@ import { fileURLToPath } from 'node:url'
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = path.dirname(__filename)
|
||||
export const serviceRoot = path.resolve(__dirname, '..')
|
||||
export const catalogRoot = path.join(serviceRoot, 'catalog')
|
||||
export const catalogRoot = path.resolve(
|
||||
process.env.NODEDC_ONTOLOGY_CATALOG_ROOT || path.join(serviceRoot, 'catalog'),
|
||||
)
|
||||
export const domainPackagesRoot = path.join(catalogRoot, 'domain-packages')
|
||||
|
||||
export async function readJson(relativePath) {
|
||||
const fullPath = path.join(serviceRoot, relativePath)
|
||||
const relative = String(relativePath || '').replace(/^catalog\//, '')
|
||||
const fullPath = path.join(catalogRoot, relative)
|
||||
const raw = await fs.readFile(fullPath, 'utf8')
|
||||
return JSON.parse(raw)
|
||||
}
|
||||
@@ -180,7 +183,7 @@ function mergeCatalog(base, extension) {
|
||||
}
|
||||
|
||||
async function loadDomainPackage(packageDir) {
|
||||
const relativeDir = path.relative(serviceRoot, packageDir)
|
||||
const relativeDir = path.relative(catalogRoot, packageDir)
|
||||
const metadata = await readOptionalJsonAt(path.join(packageDir, 'package.json'), {})
|
||||
|
||||
return {
|
||||
@@ -225,6 +228,8 @@ async function loadDomainPackage(packageDir) {
|
||||
}),
|
||||
domainPackages: [{
|
||||
id: metadata.id || path.basename(packageDir),
|
||||
version: metadata.version || '',
|
||||
updatedAt: metadata.updatedAt || '',
|
||||
status: metadata.status || 'unknown',
|
||||
relativePath: relativeDir,
|
||||
summary: metadata.summary || '',
|
||||
|
||||
@@ -0,0 +1,581 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash, timingSafeEqual } from 'node:crypto'
|
||||
import { createServer } from 'node:http'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { canonicalizeEntityId, loadCatalog } from './catalog.mjs'
|
||||
import { resolveContext } from './resolver.mjs'
|
||||
|
||||
const MCP_PROTOCOL_VERSION = '2025-06-18'
|
||||
const MAX_BODY_BYTES = 1024 * 1024
|
||||
const MAX_SEARCH_RESULTS = 50
|
||||
|
||||
const TOOLS = [
|
||||
{
|
||||
name: 'ontology_status',
|
||||
title: 'Ontology catalog status',
|
||||
description: 'Returns a safe, read-only summary of the live NODE.DC ontology catalog and its loaded domain packages.',
|
||||
inputSchema: { type: 'object', additionalProperties: false, properties: {} },
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
||||
},
|
||||
{
|
||||
name: 'ontology_search',
|
||||
title: 'Search ontology catalog',
|
||||
description: 'Searches canonical entities, relations and aliases in the live ontology catalog. Use this before introducing names or data contracts into a workflow.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['query'],
|
||||
properties: {
|
||||
query: { type: 'string', minLength: 1, maxLength: 200 },
|
||||
limit: { type: 'integer', minimum: 1, maximum: MAX_SEARCH_RESULTS },
|
||||
},
|
||||
},
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
||||
},
|
||||
{
|
||||
name: 'ontology_get_entity',
|
||||
title: 'Get ontology entity',
|
||||
description: 'Resolves an entity id or alias and returns its safe definition, aliases, relations and applicable guardrails. It never returns runtime data, credentials or source filesystem paths.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
entityId: { type: 'string', minLength: 1, maxLength: 240 },
|
||||
term: { type: 'string', minLength: 1, maxLength: 240 },
|
||||
},
|
||||
anyOf: [{ required: ['entityId'] }, { required: ['term'] }],
|
||||
},
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
||||
},
|
||||
{
|
||||
name: 'ontology_get_guardrails',
|
||||
title: 'Get ontology guardrails',
|
||||
description: 'Returns semantic and safety guardrails for the full catalog or for one entity/alias. This is read-only advice; it does not grant capability access.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
entityId: { type: 'string', minLength: 1, maxLength: 240 },
|
||||
term: { type: 'string', minLength: 1, maxLength: 240 },
|
||||
},
|
||||
},
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
||||
},
|
||||
{
|
||||
name: 'ontology_resolve_context',
|
||||
title: 'Resolve ontology context',
|
||||
description: 'Resolves a semantic request to canonical entities and context routes. It deliberately returns no source-system ids, secrets, runtime data or execution capability.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
surface: { type: 'string', maxLength: 120 },
|
||||
entityId: { type: 'string', maxLength: 240 },
|
||||
alias: { type: 'string', maxLength: 240 },
|
||||
term: { type: 'string', maxLength: 240 },
|
||||
intent: { type: 'string', maxLength: 1000 },
|
||||
contextId: { type: 'string', maxLength: 240 },
|
||||
},
|
||||
},
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
||||
},
|
||||
]
|
||||
|
||||
export function createOntologyMcpServer(overrides = {}) {
|
||||
const config = { ...readConfig(), ...overrides }
|
||||
return createServer((req, res) => handleRequest(req, res, config).catch((error) => {
|
||||
if (!res.headersSent) {
|
||||
sendJson(res, 500, { ok: false, error: 'ontology_mcp_internal_error' })
|
||||
} else {
|
||||
res.destroy(error)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async function handleRequest(req, res, config) {
|
||||
const url = new URL(req.url || '/', 'http://localhost')
|
||||
if (url.pathname === '/healthz') {
|
||||
const catalog = await loadCatalog()
|
||||
sendJson(res, 200, {
|
||||
ok: true,
|
||||
service: 'nodedc-ontology-core-mcp',
|
||||
mcp: {
|
||||
protocolVersion: MCP_PROTOCOL_VERSION,
|
||||
tools: TOOLS.length,
|
||||
readOnly: true,
|
||||
},
|
||||
catalog: catalogStatus(catalog),
|
||||
internalApiConfigured: config.internalTokens.length > 0,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (url.pathname !== '/mcp') {
|
||||
sendJson(res, 404, { ok: false, error: 'not_found' })
|
||||
return
|
||||
}
|
||||
|
||||
if (!isAllowedOrigin(req.headers.origin, config.allowedOrigins)) {
|
||||
sendJson(res, 403, { ok: false, error: 'mcp_origin_forbidden' })
|
||||
return
|
||||
}
|
||||
|
||||
if (!isAuthorized(req.headers.authorization, config.internalTokens)) {
|
||||
sendJson(res, config.internalTokens.length ? 401 : 503, {
|
||||
ok: false,
|
||||
error: config.internalTokens.length ? 'ontology_mcp_unauthorized' : 'ontology_mcp_token_not_configured',
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const method = String(req.method || 'GET').toUpperCase()
|
||||
if (method === 'GET') {
|
||||
res.setHeader('allow', 'POST')
|
||||
sendJson(res, 405, { ok: false, error: 'mcp_sse_not_supported' })
|
||||
return
|
||||
}
|
||||
if (method !== 'POST') {
|
||||
res.setHeader('allow', 'POST')
|
||||
sendJson(res, 405, { ok: false, error: 'method_not_allowed' })
|
||||
return
|
||||
}
|
||||
|
||||
let payload
|
||||
try {
|
||||
payload = JSON.parse((await readRequestBody(req, config.maxBodyBytes)).toString('utf8'))
|
||||
} catch (error) {
|
||||
sendMcpError(res, null, -32700, error?.code === 'body_too_large' ? 'request_too_large' : 'parse_error')
|
||||
return
|
||||
}
|
||||
|
||||
if (!isPlainObject(payload) || payload.jsonrpc !== '2.0' || !cleanString(payload.method, 160)) {
|
||||
sendMcpError(res, isPlainObject(payload) ? payload.id ?? null : null, -32600, 'invalid_request')
|
||||
return
|
||||
}
|
||||
|
||||
const isNotification = !Object.hasOwn(payload, 'id')
|
||||
const response = await dispatchMcpRequest(payload)
|
||||
if (isNotification) {
|
||||
res.statusCode = 202
|
||||
res.end()
|
||||
return
|
||||
}
|
||||
sendJson(res, 200, response, { 'mcp-protocol-version': MCP_PROTOCOL_VERSION })
|
||||
}
|
||||
|
||||
async function dispatchMcpRequest(payload) {
|
||||
const id = payload.id ?? null
|
||||
const method = cleanString(payload.method, 160)
|
||||
const params = isPlainObject(payload.params) ? payload.params : {}
|
||||
|
||||
if (method === 'initialize') {
|
||||
return mcpResult(id, {
|
||||
protocolVersion: MCP_PROTOCOL_VERSION,
|
||||
capabilities: { tools: { listChanged: false } },
|
||||
serverInfo: {
|
||||
name: 'nodedc-ontology-core',
|
||||
version: '0.1.0',
|
||||
},
|
||||
instructions: 'Read-only semantic catalog. It exposes no runtime telemetry, database access, credentials, source paths, workflow mutations, command dispatch or Studio rendering controls.',
|
||||
})
|
||||
}
|
||||
|
||||
if (method === 'notifications/initialized') return mcpResult(id, {})
|
||||
if (method === 'ping') return mcpResult(id, {})
|
||||
if (method === 'tools/list') return mcpResult(id, { tools: TOOLS })
|
||||
if (method !== 'tools/call') return mcpError(id, -32601, 'method_not_found')
|
||||
|
||||
const toolName = cleanString(params.name, 120)
|
||||
const argumentsValue = isPlainObject(params.arguments) ? params.arguments : {}
|
||||
return mcpResult(id, await callTool(toolName, argumentsValue))
|
||||
}
|
||||
|
||||
async function callTool(name, input) {
|
||||
if (name === 'ontology_status') {
|
||||
const catalog = await loadCatalog()
|
||||
return toolResult({
|
||||
schemaVersion: 'nodedc.ontology-mcp.status.v1',
|
||||
readOnly: true,
|
||||
catalog: catalogStatus(catalog),
|
||||
})
|
||||
}
|
||||
|
||||
if (name === 'ontology_search') {
|
||||
const query = cleanString(input.query, 200)
|
||||
if (!query) return toolError('query_required')
|
||||
const catalog = await loadCatalog()
|
||||
return toolResult(searchCatalog(catalog, query, boundedLimit(input.limit)))
|
||||
}
|
||||
|
||||
if (name === 'ontology_get_entity') {
|
||||
const catalog = await loadCatalog()
|
||||
const requested = cleanString(input.entityId || input.term, 240)
|
||||
const entityId = canonicalizeEntityId(requested, catalog)
|
||||
const entity = catalog.entityById.get(entityId)
|
||||
if (!entity) return toolError('entity_not_found', { requested })
|
||||
return toolResult(entityDetails(catalog, entity))
|
||||
}
|
||||
|
||||
if (name === 'ontology_get_guardrails') {
|
||||
const catalog = await loadCatalog()
|
||||
const requested = cleanString(input.entityId || input.term, 240)
|
||||
const entityId = requested ? canonicalizeEntityId(requested, catalog) : ''
|
||||
if (requested && !catalog.entityById.has(entityId)) return toolError('entity_not_found', { requested })
|
||||
return toolResult(guardrailDetails(catalog, entityId))
|
||||
}
|
||||
|
||||
if (name === 'ontology_resolve_context') {
|
||||
const resolved = await resolveContext({
|
||||
surface: cleanString(input.surface, 120),
|
||||
entityId: cleanString(input.entityId, 240),
|
||||
alias: cleanString(input.alias, 240),
|
||||
term: cleanString(input.term, 240),
|
||||
intent: cleanString(input.intent, 1000),
|
||||
contextId: cleanString(input.contextId, 240),
|
||||
})
|
||||
return toolResult(sanitizeResolvedContext(resolved))
|
||||
}
|
||||
|
||||
return toolError('tool_not_found', { name })
|
||||
}
|
||||
|
||||
function catalogStatus(catalog) {
|
||||
const packages = (catalog.domainPackages || [])
|
||||
.map((item) => ({
|
||||
id: cleanString(item.id, 120),
|
||||
version: cleanString(item.version, 80),
|
||||
updatedAt: cleanString(item.updatedAt, 80),
|
||||
status: cleanString(item.status, 120),
|
||||
summary: cleanString(item.summary, 1000),
|
||||
}))
|
||||
.sort((left, right) => left.id.localeCompare(right.id))
|
||||
const snapshot = {
|
||||
entityIds: (catalog.entities?.entities || []).map((item) => item.id).sort(),
|
||||
relationIds: (catalog.relations?.relations || []).map((item) => item.id).sort(),
|
||||
aliases: (catalog.aliases?.aliases || []).map((item) => [item.alias, item.canonicalId]).sort((left, right) => String(left[0]).localeCompare(String(right[0]))),
|
||||
packages,
|
||||
}
|
||||
return {
|
||||
schemaVersion: 'nodedc.ontology-catalog.status.v1',
|
||||
catalogHash: createHash('sha256').update(JSON.stringify(snapshot)).digest('hex').slice(0, 16),
|
||||
entities: snapshot.entityIds.length,
|
||||
relations: snapshot.relationIds.length,
|
||||
aliases: snapshot.aliases.length,
|
||||
guardrails: (catalog.guardrails?.rules || []).length,
|
||||
blockedConflations: (catalog.guardrails?.blockedConflations || []).length,
|
||||
domainPackages: packages,
|
||||
}
|
||||
}
|
||||
|
||||
function searchCatalog(catalog, query, limit) {
|
||||
const needle = normalizeSearch(query)
|
||||
const items = []
|
||||
for (const entity of catalog.entities?.entities || []) {
|
||||
const aliasScore = (catalog.aliases?.aliases || [])
|
||||
.filter((alias) => alias.canonicalId === entity.id)
|
||||
.reduce((best, alias) => Math.max(best, searchScore(needle, alias.alias)), 0)
|
||||
const score = Math.max(
|
||||
aliasScore,
|
||||
searchScore(needle, entity.id, entity.name, entity.summary, entity.surface, entity.authority),
|
||||
)
|
||||
if (score > 0) items.push({ kind: 'entity', score, entity: safeEntity(entity) })
|
||||
}
|
||||
for (const relation of catalog.relations?.relations || []) {
|
||||
const score = searchScore(needle, relation.id, relation.summary, ...(relation.from || []), ...(relation.to || []))
|
||||
if (score > 0) items.push({ kind: 'relation', score, relation: safeRelation(relation) })
|
||||
}
|
||||
for (const alias of catalog.aliases?.aliases || []) {
|
||||
const score = searchScore(needle, alias.alias, alias.canonicalId)
|
||||
if (score > 0) items.push({ kind: 'alias', score, alias: safeAlias(alias) })
|
||||
}
|
||||
items.sort((left, right) => right.score - left.score || stableItemId(left).localeCompare(stableItemId(right)))
|
||||
return {
|
||||
schemaVersion: 'nodedc.ontology-mcp.search.v1',
|
||||
query,
|
||||
totalMatches: items.length,
|
||||
results: items.slice(0, limit).map(({ score, ...item }) => item),
|
||||
}
|
||||
}
|
||||
|
||||
function entityDetails(catalog, entity) {
|
||||
const entityId = entity.id
|
||||
return {
|
||||
schemaVersion: 'nodedc.ontology-mcp.entity.v1',
|
||||
entity: safeEntity(entity),
|
||||
aliases: (catalog.aliases?.aliases || [])
|
||||
.filter((item) => item.canonicalId === entityId)
|
||||
.map(safeAlias)
|
||||
.sort((left, right) => left.alias.localeCompare(right.alias)),
|
||||
relations: (catalog.relations?.relations || [])
|
||||
.filter((item) => (item.from || []).includes(entityId) || (item.to || []).includes(entityId))
|
||||
.map(safeRelation)
|
||||
.sort((left, right) => left.id.localeCompare(right.id)),
|
||||
guardrails: (catalog.guardrails?.rules || [])
|
||||
.filter((item) => (item.entityIds || []).includes(entityId))
|
||||
.map(safeGuardrail)
|
||||
.sort((left, right) => left.id.localeCompare(right.id)),
|
||||
blockedConflations: (catalog.guardrails?.blockedConflations || [])
|
||||
.filter((pair) => Array.isArray(pair) && pair.includes(entityId))
|
||||
.map((pair) => pair.map((item) => cleanString(item, 240))),
|
||||
}
|
||||
}
|
||||
|
||||
function guardrailDetails(catalog, entityId) {
|
||||
const rules = (catalog.guardrails?.rules || [])
|
||||
.filter((item) => !entityId || (item.entityIds || []).includes(entityId))
|
||||
.map(safeGuardrail)
|
||||
.sort((left, right) => left.id.localeCompare(right.id))
|
||||
const blockedConflations = (catalog.guardrails?.blockedConflations || [])
|
||||
.filter((pair) => !entityId || (Array.isArray(pair) && pair.includes(entityId)))
|
||||
.map((pair) => Array.isArray(pair) ? pair.map((item) => cleanString(item, 240)) : [])
|
||||
return {
|
||||
schemaVersion: 'nodedc.ontology-mcp.guardrails.v1',
|
||||
entityId: entityId || null,
|
||||
rules,
|
||||
blockedConflations,
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeResolvedContext(resolved) {
|
||||
return {
|
||||
schemaVersion: 'nodedc.ontology-mcp.context-resolution.v1',
|
||||
input: {
|
||||
surface: cleanString(resolved?.input?.surface, 120),
|
||||
entityId: cleanString(resolved?.input?.entityId, 240),
|
||||
intent: cleanString(resolved?.input?.intent, 1000),
|
||||
},
|
||||
canonicalEntity: resolved?.canonicalEntity ? safeEntity(resolved.canonicalEntity) : null,
|
||||
matchedContexts: (resolved?.matchedContexts || []).map((item) => ({
|
||||
surface: cleanString(item.surface, 120),
|
||||
entityId: cleanString(item.entityId, 240),
|
||||
label: cleanString(item.label, 1000),
|
||||
sourceSystem: cleanString(item.sourceSystem, 160),
|
||||
status: cleanString(item.status, 120),
|
||||
})),
|
||||
bindings: (resolved?.bindings || []).map((item) => ({
|
||||
typeId: cleanString(item.typeId, 240),
|
||||
status: cleanString(item.status, 120),
|
||||
relationId: cleanString(item.relationId, 240),
|
||||
summary: cleanString(item.summary, 1000),
|
||||
})),
|
||||
missingBindingTypes: (resolved?.missingBindingTypes || []).map((item) => ({
|
||||
id: cleanString(item.id, 240),
|
||||
relationId: cleanString(item.relationId, 240),
|
||||
summary: cleanString(item.summary, 1000),
|
||||
})),
|
||||
selectedRule: resolved?.selectedRule ? {
|
||||
id: cleanString(resolved.selectedRule.id, 240),
|
||||
fromSurface: cleanString(resolved.selectedRule.fromSurface, 120),
|
||||
outputSurface: cleanString(resolved.selectedRule.outputSurface, 120),
|
||||
inputEntityIds: stringList(resolved.selectedRule.inputEntityIds),
|
||||
outputEntityIds: stringList(resolved.selectedRule.outputEntityIds),
|
||||
requiredBindings: stringList(resolved.selectedRule.requiredBindings),
|
||||
summary: cleanString(resolved.selectedRule.summary, 1000),
|
||||
} : null,
|
||||
candidates: (resolved?.candidates || []).map((item) => ({
|
||||
id: cleanString(item.id, 240),
|
||||
score: Number.isFinite(Number(item.score)) ? Number(item.score) : 0,
|
||||
outputSurface: cleanString(item.outputSurface, 120),
|
||||
outputEntityIds: stringList(item.outputEntityIds),
|
||||
requiredBindings: stringList(item.requiredBindings),
|
||||
summary: cleanString(item.summary, 1000),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
function safeEntity(value) {
|
||||
return {
|
||||
id: cleanString(value?.id, 240),
|
||||
name: cleanString(value?.name, 240),
|
||||
surface: cleanString(value?.surface, 120),
|
||||
status: stringList(value?.status),
|
||||
authority: cleanString(value?.authority, 240),
|
||||
summary: cleanString(value?.summary, 2000),
|
||||
}
|
||||
}
|
||||
|
||||
function safeRelation(value) {
|
||||
return {
|
||||
id: cleanString(value?.id, 240),
|
||||
from: stringList(value?.from),
|
||||
to: stringList(value?.to),
|
||||
status: cleanString(value?.status, 120),
|
||||
summary: cleanString(value?.summary, 2000),
|
||||
}
|
||||
}
|
||||
|
||||
function safeAlias(value) {
|
||||
return {
|
||||
alias: cleanString(value?.alias, 240),
|
||||
canonicalId: cleanString(value?.canonicalId, 240),
|
||||
}
|
||||
}
|
||||
|
||||
function safeGuardrail(value) {
|
||||
return {
|
||||
id: cleanString(value?.id, 240),
|
||||
severity: cleanString(value?.severity, 40),
|
||||
summary: cleanString(value?.summary, 2000),
|
||||
entityIds: stringList(value?.entityIds),
|
||||
}
|
||||
}
|
||||
|
||||
function toolResult(value) {
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(value, null, 2) }],
|
||||
structuredContent: value,
|
||||
}
|
||||
}
|
||||
|
||||
function toolError(error, details = {}) {
|
||||
const value = { ok: false, error, ...details }
|
||||
return {
|
||||
content: [{ type: 'text', text: JSON.stringify(value, null, 2) }],
|
||||
structuredContent: value,
|
||||
isError: true,
|
||||
}
|
||||
}
|
||||
|
||||
function mcpResult(id, result) {
|
||||
return { jsonrpc: '2.0', id, result }
|
||||
}
|
||||
|
||||
function mcpError(id, code, message) {
|
||||
return { jsonrpc: '2.0', id, error: { code, message } }
|
||||
}
|
||||
|
||||
function sendMcpError(res, id, code, message) {
|
||||
sendJson(res, 200, mcpError(id, code, message), { 'mcp-protocol-version': MCP_PROTOCOL_VERSION })
|
||||
}
|
||||
|
||||
function sendJson(res, status, body, headers = {}) {
|
||||
res.statusCode = status
|
||||
res.setHeader('content-type', 'application/json; charset=utf-8')
|
||||
res.setHeader('cache-control', 'no-store')
|
||||
for (const [key, value] of Object.entries(headers)) res.setHeader(key, value)
|
||||
res.end(JSON.stringify(body))
|
||||
}
|
||||
|
||||
async function readRequestBody(req, maxBytes) {
|
||||
const chunks = []
|
||||
let size = 0
|
||||
for await (const chunk of req) {
|
||||
size += chunk.length
|
||||
if (size > maxBytes) {
|
||||
const error = new Error('body_too_large')
|
||||
error.code = 'body_too_large'
|
||||
throw error
|
||||
}
|
||||
chunks.push(chunk)
|
||||
}
|
||||
return Buffer.concat(chunks)
|
||||
}
|
||||
|
||||
function isAuthorized(header, candidates) {
|
||||
const match = String(header || '').match(/^Bearer\s+(.+)$/i)
|
||||
const token = match?.[1]?.trim() || ''
|
||||
if (!token) return false
|
||||
return (candidates || []).some((candidate) => safeEqual(token, candidate))
|
||||
}
|
||||
|
||||
function safeEqual(left, right) {
|
||||
const leftBuffer = Buffer.from(String(left || ''))
|
||||
const rightBuffer = Buffer.from(String(right || ''))
|
||||
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer)
|
||||
}
|
||||
|
||||
function isAllowedOrigin(origin, allowedOrigins) {
|
||||
if (!origin) return true
|
||||
const normalized = normalizeOrigin(origin)
|
||||
return Boolean(normalized && allowedOrigins.includes(normalized))
|
||||
}
|
||||
|
||||
function normalizeOrigin(value) {
|
||||
try {
|
||||
const url = new URL(String(value || ''))
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') return ''
|
||||
return url.origin
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
function readConfig() {
|
||||
const internalTokens = uniqueStrings([
|
||||
process.env.ONTOLOGY_CORE_TOKEN,
|
||||
process.env.NODEDC_INTERNAL_ACCESS_TOKEN,
|
||||
process.env.NODEDC_PLATFORM_SERVICE_TOKEN,
|
||||
])
|
||||
return {
|
||||
port: boundedNumber(process.env.PORT || process.env.ONTOLOGY_CORE_PORT, 18104, 1, 65535),
|
||||
internalTokens,
|
||||
allowedOrigins: uniqueStrings(String(process.env.ONTOLOGY_MCP_ALLOWED_ORIGINS || '')
|
||||
.split(/[\n,;]+/)
|
||||
.map(normalizeOrigin)
|
||||
.filter(Boolean)),
|
||||
maxBodyBytes: boundedNumber(process.env.ONTOLOGY_MCP_MAX_BODY_BYTES, MAX_BODY_BYTES, 1024, 10 * 1024 * 1024),
|
||||
}
|
||||
}
|
||||
|
||||
function boundedLimit(value) {
|
||||
return boundedNumber(value, 20, 1, MAX_SEARCH_RESULTS)
|
||||
}
|
||||
|
||||
function boundedNumber(value, fallback, min, max) {
|
||||
const numeric = Number(value)
|
||||
if (!Number.isFinite(numeric)) return fallback
|
||||
return Math.min(Math.max(Math.trunc(numeric), min), max)
|
||||
}
|
||||
|
||||
function normalizeSearch(value) {
|
||||
return String(value || '').trim().toLocaleLowerCase('ru-RU')
|
||||
}
|
||||
|
||||
function searchScore(needle, ...values) {
|
||||
let score = 0
|
||||
for (const value of values) {
|
||||
const haystack = normalizeSearch(value)
|
||||
if (!haystack) continue
|
||||
if (haystack === needle) score = Math.max(score, 100)
|
||||
else if (haystack.startsWith(needle)) score = Math.max(score, 60)
|
||||
else if (haystack.includes(needle)) score = Math.max(score, 30)
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
function stableItemId(item) {
|
||||
return item.entity?.id || item.relation?.id || item.alias?.alias || ''
|
||||
}
|
||||
|
||||
function stringList(value) {
|
||||
if (!Array.isArray(value)) return []
|
||||
return value.map((item) => cleanString(item, 240)).filter(Boolean)
|
||||
}
|
||||
|
||||
function cleanString(value, maxLength) {
|
||||
return String(value || '').trim().slice(0, maxLength)
|
||||
}
|
||||
|
||||
function uniqueStrings(values) {
|
||||
return [...new Set((values || []).map((item) => cleanString(item, 4000)).filter(Boolean))]
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
|
||||
const currentFile = fileURLToPath(import.meta.url)
|
||||
const isEntrypoint = process.argv[1] && path.resolve(process.argv[1]) === currentFile
|
||||
|
||||
if (isEntrypoint) {
|
||||
const config = readConfig()
|
||||
const server = createOntologyMcpServer(config)
|
||||
server.listen(config.port, '0.0.0.0', () => {
|
||||
console.log(`NODE.DC Ontology Core MCP listening on http://0.0.0.0:${config.port}`)
|
||||
})
|
||||
const shutdown = () => server.close(() => process.exit(0))
|
||||
process.on('SIGTERM', shutdown)
|
||||
process.on('SIGINT', shutdown)
|
||||
}
|
||||
@@ -191,7 +191,7 @@ function buildEngineContextFromManifest(manifest) {
|
||||
}
|
||||
if (workflow.nodeId) refs.node_id = String(workflow.nodeId)
|
||||
if (workflow.runtimeWorkflowId) refs.runtime_workflow_id = String(workflow.runtimeWorkflowId)
|
||||
if (workflow.n8nInstanceId) refs.n8n_instance_id = String(workflow.n8nInstanceId)
|
||||
if (workflow.runtimeInstanceId) refs.runtime_instance_id = String(workflow.runtimeInstanceId)
|
||||
|
||||
return {
|
||||
id: `context.engine.${slugPart(workflow.id)}`,
|
||||
|
||||
@@ -185,7 +185,7 @@ async function runSmoke() {
|
||||
},
|
||||
{ surface: 'engine', entityId: 'engine.workflow_l1', intent: 'создай карточку в ops по текущему workflow' },
|
||||
{ surface: 'assistant', entityId: 'assistant.bridge', intent: 'select mcp gateway grant and tool' },
|
||||
{ term: 'n8n workflow id', intent: 'runtime workflow id lookup' },
|
||||
{ term: 'runtime workflow id', intent: 'runtime workflow id lookup' },
|
||||
]
|
||||
|
||||
const results = []
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import assert from 'node:assert/strict'
|
||||
import { createOntologyMcpServer } from '../mcp-server.mjs'
|
||||
|
||||
const TOKEN = 'ontology-mcp-smoke-token'
|
||||
const server = createOntologyMcpServer({
|
||||
internalTokens: [TOKEN],
|
||||
allowedOrigins: [],
|
||||
maxBodyBytes: 1024 * 1024,
|
||||
})
|
||||
|
||||
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
|
||||
const address = server.address()
|
||||
const baseUrl = `http://127.0.0.1:${address.port}`
|
||||
|
||||
try {
|
||||
const health = await fetch(`${baseUrl}/healthz`)
|
||||
const healthPayload = await health.json()
|
||||
assert.equal(health.ok, true)
|
||||
assert.equal(healthPayload.ok, true)
|
||||
assert.equal(healthPayload.mcp.readOnly, true)
|
||||
|
||||
const initialized = await rpc(baseUrl, TOKEN, 1, 'initialize', { protocolVersion: '2025-06-18' })
|
||||
assert.equal(initialized.result.protocolVersion, '2025-06-18')
|
||||
assert.equal(initialized.result.capabilities.tools.listChanged, false)
|
||||
|
||||
const tools = await rpc(baseUrl, TOKEN, 2, 'tools/list', {})
|
||||
const toolNames = tools.result.tools.map((tool) => tool.name).sort()
|
||||
assert.deepEqual(toolNames, [
|
||||
'ontology_get_entity',
|
||||
'ontology_get_guardrails',
|
||||
'ontology_resolve_context',
|
||||
'ontology_search',
|
||||
'ontology_status',
|
||||
])
|
||||
|
||||
const search = await rpc(baseUrl, TOKEN, 3, 'tools/call', {
|
||||
name: 'ontology_search',
|
||||
arguments: { query: 'трайк', limit: 10 },
|
||||
})
|
||||
assert.equal(search.result.isError, undefined)
|
||||
assert.equal(search.result.structuredContent.query, 'трайк')
|
||||
assert.equal(search.result.structuredContent.results.some((item) => item.entity?.id === 'gelios.unit'), true)
|
||||
|
||||
const entity = await rpc(baseUrl, TOKEN, 4, 'tools/call', {
|
||||
name: 'ontology_get_entity',
|
||||
arguments: { term: 'helius' },
|
||||
})
|
||||
assert.equal(entity.result.structuredContent.entity.id, 'gelios.integration')
|
||||
assert.equal(JSON.stringify(entity.result.structuredContent).includes('/Users/'), false)
|
||||
|
||||
const guardrails = await rpc(baseUrl, TOKEN, 5, 'tools/call', {
|
||||
name: 'ontology_get_guardrails',
|
||||
arguments: { entityId: 'gelios.command_dispatch' },
|
||||
})
|
||||
assert.equal(guardrails.result.structuredContent.rules.some((rule) => rule.id === 'guardrail.gelios.commands_are_red_domain'), true)
|
||||
|
||||
const unauthorized = await fetch(`${baseUrl}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ jsonrpc: '2.0', id: 6, method: 'tools/list', params: {} }),
|
||||
})
|
||||
assert.equal(unauthorized.status, 401)
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
checks: [
|
||||
'health',
|
||||
'mcp_initialize',
|
||||
'read_only_tool_catalog',
|
||||
'gelios_alias_resolution',
|
||||
'gelios_command_guardrail_visible',
|
||||
'evidence_paths_not_exposed',
|
||||
'internal_bearer_required',
|
||||
],
|
||||
}, null, 2))
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()))
|
||||
}
|
||||
|
||||
async function rpc(baseUrl, token, id, method, params) {
|
||||
const response = await fetch(`${baseUrl}/mcp`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
accept: 'application/json, text/event-stream',
|
||||
'mcp-protocol-version': '2025-06-18',
|
||||
},
|
||||
body: JSON.stringify({ jsonrpc: '2.0', id, method, params }),
|
||||
})
|
||||
assert.equal(response.ok, true)
|
||||
return response.json()
|
||||
}
|
||||
Reference in New Issue
Block a user