SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(platform): complete the Gelios external data loop

Browse Source
This commit is contained in:
Codex
2026-07-20 20:45:05 +03:00
parent def9a24e0d
commit 8a7465cf0e
66 changed files with 5730 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/external-data-plane/src/data-product-delivery.mjs
+54 -4
View File
@@ -13,7 +13,7 @@ export async function loadDataProductDefinition(db, dataProductId, { activeOnly
const result = await db.query(
`select id, version, ontology_revision as "ontologyRevision",
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
fields, history_policy as "historyPolicy", active,
fields, field_contracts as "fieldContracts", history_policy as "historyPolicy", active,
created_at as "createdAt", updated_at as "updatedAt"
from external_data_plane_products
where id = $1 ${activeOnly ? "and active = true" : ""}`,
@@ -25,8 +25,8 @@ export async function loadDataProductDefinition(db, dataProductId, { activeOnly
export async function persistDataProductDefinition(db, definition) {
const result = await db.query(
`insert into external_data_plane_products (
id, version, ontology_revision, delivery_mode, semantic_types, fields, history_policy
) values ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7::jsonb)
id, version, ontology_revision, delivery_mode, semantic_types, fields, field_contracts, history_policy
) values ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7::jsonb, $8::jsonb)
on conflict (id) do update set
active = true,
updated_at = now()
@@ -35,10 +35,11 @@ export async function persistDataProductDefinition(db, definition) {
and external_data_plane_products.delivery_mode = excluded.delivery_mode
and external_data_plane_products.semantic_types = excluded.semantic_types
and external_data_plane_products.fields = excluded.fields
and external_data_plane_products.field_contracts = excluded.field_contracts
and external_data_plane_products.history_policy = excluded.history_policy
returning id, version, ontology_revision as "ontologyRevision",
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
fields, history_policy as "historyPolicy", active,
fields, field_contracts as "fieldContracts", history_policy as "historyPolicy", active,
created_at as "createdAt", updated_at as "updatedAt"`,
[
definition.id,
@@ -47,6 +48,7 @@ export async function persistDataProductDefinition(db, definition) {
definition.deliveryMode,
JSON.stringify(definition.semanticTypes),
JSON.stringify(definition.fields),
JSON.stringify(definition.fieldContracts || {}),
JSON.stringify(definition.history),
],
);
@@ -658,6 +660,8 @@ function publishFingerprint(batch) {
export function assertPublishMatchesDefinition(batch, definition) {
const semanticTypes = new Set(Array.isArray(definition?.semanticTypes) ? definition.semanticTypes : []);
const fields = new Set(Array.isArray(definition?.fields) ? definition.fields : []);
const fieldContracts = definition?.fieldContracts && typeof definition.fieldContracts === "object"
&& !Array.isArray(definition.fieldContracts) ? definition.fieldContracts : {};
const entityKeys = new Set();
for (const fact of batch?.facts || []) {
if (!semanticTypes.has(fact.semanticType)) throw deliveryError("data_product_semantic_type_forbidden", 422);
@@ -672,9 +676,55 @@ export function assertPublishMatchesDefinition(batch, definition) {
if (fact.geometry !== undefined && !geometryDeclared(fields)) {
throw deliveryError("data_product_geometry_forbidden", 422);
}
for (const [field, contract] of Object.entries(fieldContracts)) {
const resolved = resolveContractField(fact, field);
if (!resolved.present) {
if (contract.required === true) throw deliveryError("data_product_field_required", 422);
continue;
}
assertFieldContractValue(resolved.value, contract);
}
}
}
function resolveContractField(fact, field) {
if (field === "geometry") return { present: fact.geometry !== undefined, value: fact.geometry };
if (field === "source_id") return { present: fact.sourceId !== undefined, value: fact.sourceId };
if (field === "semantic_type") return { present: fact.semanticType !== undefined, value: fact.semanticType };
if (field === "observed_at") return { present: fact.observedAt !== undefined, value: fact.observedAt };
const attribute = field.startsWith("attributes.") ? field.slice("attributes.".length) : field;
const attributes = fact.attributes || {};
return { present: Object.hasOwn(attributes, attribute), value: attributes[attribute] };
}
function assertFieldContractValue(value, contract) {
if (!fieldContractValueMatchesType(value, contract.type)) {
throw deliveryError("data_product_field_type_invalid", 422);
}
if (Array.isArray(contract.enum) && !contract.enum.some((allowed) => Object.is(allowed, value))) {
throw deliveryError("data_product_field_value_forbidden", 422);
}
if (typeof value === "number" && (
(Number.isFinite(contract.minimum) && value < contract.minimum)
|| (Number.isFinite(contract.maximum) && value > contract.maximum)
)) throw deliveryError("data_product_field_value_out_of_range", 422);
}
function fieldContractValueMatchesType(value, type) {
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
if (type === "point") {
return value?.type === "Point"
&& Array.isArray(value.coordinates)
&& value.coordinates.length === 2
&& value.coordinates.every(Number.isFinite)
&& value.coordinates[0] >= -180
&& value.coordinates[0] <= 180
&& value.coordinates[1] >= -90
&& value.coordinates[1] <= 90;
}
return typeof value === type && (type !== "number" || Number.isFinite(value));
}
function geometryDeclared(fields) {
return fields.has("geometry")
|| fields.has("coordinates")
services/external-data-plane/src/data-product-policy.mjs
+63 -2
View File
@@ -2,10 +2,12 @@ const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]);
const DEFINITION_KEYS = new Set([
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "history",
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history",
]);
const HISTORY_KEYS = new Set(["mode", "intervalMs", "retentionDays", "strategy"]);
const FIELD_CONTRACT_KEYS = new Set(["type", "required", "enum", "minimum", "maximum"]);
export function normalizeDataProductDefinition(value) {
if (!isPlainObject(value) || !hasOnlyKeys(value, DEFINITION_KEYS)) {
@@ -30,8 +32,58 @@ export function normalizeDataProductDefinition(value) {
}
if (!semanticTypes.length || !fields.length) throw policyError("data_product_definition_shape_invalid");
const fieldContracts = normalizeFieldContracts(value.fieldContracts, fields);
const history = normalizeHistoryPolicy(value.history);
return Object.freeze({ id, version, ontologyRevision, deliveryMode, semanticTypes, fields, history });
return Object.freeze({ id, version, ontologyRevision, deliveryMode, semanticTypes, fields, fieldContracts, history });
}
export function normalizeFieldContracts(value, fields) {
if (value === undefined) return Object.freeze({});
if (!isPlainObject(value)) throw policyError("data_product_field_contracts_invalid");
const names = Object.keys(value);
if (names.length === 0) return Object.freeze({});
if (names.length !== fields.length || fields.some((field) => !Object.hasOwn(value, field))) {
throw policyError("data_product_field_contracts_must_match_fields");
}
const normalized = {};
for (const field of fields) normalized[field] = normalizeFieldContract(value[field]);
return Object.freeze(normalized);
}
function normalizeFieldContract(value) {
if (!isPlainObject(value) || !hasOnlyKeys(value, FIELD_CONTRACT_KEYS)) {
throw policyError("data_product_field_contract_invalid");
}
const type = string(value.type);
if (!FIELD_CONTRACT_TYPES.has(type) || typeof value.required !== "boolean") {
throw policyError("data_product_field_contract_shape_invalid");
}
const contract = { type, required: value.required };
if (value.enum !== undefined) {
if (!Array.isArray(value.enum) || value.enum.length === 0
|| new Set(value.enum.map(stableLiteral)).size !== value.enum.length
|| new Set(["point", "string_array"]).has(type)
|| value.enum.some((item) => !fieldContractValueMatchesType(item, type))) {
throw policyError("data_product_field_contract_enum_invalid");
}
contract.enum = Object.freeze([...value.enum].sort((left, right) => stableLiteral(left).localeCompare(stableLiteral(right))));
}
if (value.minimum !== undefined || value.maximum !== undefined) {
if (type !== "number"
|| (value.minimum !== undefined && !Number.isFinite(value.minimum))
|| (value.maximum !== undefined && !Number.isFinite(value.maximum))
|| (Number.isFinite(value.minimum) && Number.isFinite(value.maximum) && value.minimum > value.maximum)) {
throw policyError("data_product_field_contract_range_invalid");
}
if (value.minimum !== undefined) contract.minimum = value.minimum;
if (value.maximum !== undefined) contract.maximum = value.maximum;
if (contract.enum?.some((item) => (
(contract.minimum !== undefined && item < contract.minimum)
|| (contract.maximum !== undefined && item > contract.maximum)
))) throw policyError("data_product_field_contract_enum_out_of_range");
}
return Object.freeze(contract);
}
export function normalizeHistoryPolicy(value = { mode: "none" }) {
@@ -62,6 +114,7 @@ export function safeDataProductDefinition(row) {
deliveryMode: row.deliveryMode,
semanticTypes: array(row.semanticTypes),
fields: array(row.fields),
fieldContracts: isPlainObject(row.fieldContracts) ? row.fieldContracts : {},
history: isPlainObject(row.historyPolicy) ? row.historyPolicy : {},
active: row.active === true,
createdAt: iso(row.createdAt),
@@ -116,3 +169,11 @@ function isPlainObject(value) {
function hasOnlyKeys(value, allowed) {
return Object.keys(value).every((key) => allowed.has(key));
}
function fieldContractValueMatchesType(value, type) {
return typeof value === type && (type !== "number" || Number.isFinite(value));
}
function stableLiteral(value) {
return `${typeof value}:${JSON.stringify(value)}`;
}
services/external-data-plane/src/schema.mjs
+20 -1
View File
@@ -12,14 +12,33 @@ export async function migrate(pool) {
delivery_mode text not null,
semantic_types jsonb not null,
fields jsonb not null,
field_contracts jsonb not null default '{}'::jsonb,
history_policy jsonb not null,
active boolean not null default true,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
check (jsonb_typeof(semantic_types) = 'array' and jsonb_array_length(semantic_types) > 0),
check (jsonb_typeof(fields) = 'array' and jsonb_array_length(fields) > 0)
check (jsonb_typeof(fields) = 'array' and jsonb_array_length(fields) > 0),
constraint external_data_plane_products_field_contracts_object_ck
check (jsonb_typeof(field_contracts) = 'object')
)
`);
await pool.query("alter table external_data_plane_products add column if not exists field_contracts jsonb not null default '{}'::jsonb");
await pool.query(`
do $$
begin
if not exists (
select 1 from pg_constraint
where conrelid = 'external_data_plane_products'::regclass
and conname = 'external_data_plane_products_field_contracts_object_ck'
) then
alter table external_data_plane_products
add constraint external_data_plane_products_field_contracts_object_ck
check (jsonb_typeof(field_contracts) = 'object');
end if;
end
$$
`);
await pool.query(`
create table if not exists external_data_plane_batches (
services/external-data-plane/src/server.mjs
+1 -1
View File
@@ -1179,7 +1179,7 @@ async function listGrantedProducts(binding) {
const result = await pool.query(
`select id, version, ontology_revision as "ontologyRevision",
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
fields, history_policy as "historyPolicy", active,
fields, field_contracts as "fieldContracts", history_policy as "historyPolicy", active,
created_at as "createdAt", updated_at as "updatedAt"
from external_data_plane_products
where active = true and id = any($1::text[])