feat(provider-contract): define materializable L2 graph blueprints
This commit is contained in:
parent
a5a5644dbf
commit
4cf3a83ef1
|
|
@ -52,11 +52,17 @@ Foundry.
|
||||||
детерминированный `nodedc.l2-execution-plan/v1`: разрешает generic шаги
|
детерминированный `nodedc.l2-execution-plan/v1`: разрешает generic шаги
|
||||||
collection/request/extract/mapping/publish, прикладывает полные декларативные
|
collection/request/extract/mapping/publish, прикладывает полные декларативные
|
||||||
request/mapping contracts и фиксирует SHA-256 digests package, profile,
|
request/mapping contracts и фиксирует SHA-256 digests package, profile,
|
||||||
template, mapping, field policy, Data Product и telemetry registry. В
|
template, mapping, field policy, Data Product и telemetry registry. Plan
|
||||||
компиляторе нет веток по provider ID. Provider-specific URL, source paths и
|
содержит exact `nodedc.l2-graph-blueprint/v1`: линейный набор generic runtime
|
||||||
параметры являются данными immutable package. После материализации Engine
|
node kinds, exact edges, config digests, credential slots, trigger/retry/batch/
|
||||||
`attestL2ExecutionPlanMaterialization` связывает exact plan digest с exact
|
cardinality policy и закрытую матрицу требуемых mapping operators. Blueprint
|
||||||
graph revision/digest и точным списком materialized steps.
|
не содержит credential reference и фиксируется отдельным SHA-256 digest.
|
||||||
|
Runtime обязан fail-closed отклонить неизвестный operator, rule или geometry
|
||||||
|
strategy; молча подставлять provider-specific code запрещено. В компиляторе
|
||||||
|
нет веток по provider ID. Provider-specific URL, source paths и параметры
|
||||||
|
являются данными immutable package. После материализации Engine
|
||||||
|
`attestL2ExecutionPlanMaterialization` связывает exact plan/blueprint digest с
|
||||||
|
exact graph revision/digest и точным списком materialized steps.
|
||||||
|
|
||||||
Динамическая телеметрия проходит отдельный
|
Динамическая телеметрия проходит отдельный
|
||||||
`nodedc.telemetry-field-registry/v1`. Реестр хранит только имена source
|
`nodedc.telemetry-field-registry/v1`. Реестр хранит только имена source
|
||||||
|
|
|
||||||
|
|
@ -146,18 +146,23 @@ value.mappingContracts.push({
|
||||||
observedAt: { strategy: "first_non_empty", paths: ["updatedAt"], coerce: "unix_or_iso_timestamp", fallback: "collection_received_at" },
|
observedAt: { strategy: "first_non_empty", paths: ["updatedAt"], coerce: "unix_or_iso_timestamp", fallback: "collection_received_at" },
|
||||||
geometry: {
|
geometry: {
|
||||||
type: "GeoJSON",
|
type: "GeoJSON",
|
||||||
strategy: "gelios_geozone_v1",
|
strategy: "bounded_zone_geometry_v1",
|
||||||
paths: ["type", "points", "line", "radius"],
|
kindPath: "type",
|
||||||
|
polygonPointsPath: "points",
|
||||||
|
corridorPointsPath: "line",
|
||||||
|
radiusPath: "radius",
|
||||||
|
coordinateOrder: "latitude_longitude",
|
||||||
|
circleSegments: 64,
|
||||||
allowedTypes: ["Polygon", "MultiPolygon"],
|
allowedTypes: ["Polygon", "MultiPolygon"],
|
||||||
omitIfInvalid: false,
|
omitIfInvalid: false,
|
||||||
},
|
},
|
||||||
attributes: {
|
attributes: {
|
||||||
area_square_meters: { strategy: "first_non_empty", paths: ["surfaceArea", "surface_area"], coerce: "number", minimum: 0, omitIfInvalid: true },
|
area_square_meters: { strategy: "first_non_empty", paths: ["surfaceArea", "surface_area"], coerce: "number", minimum: 0, omitIfMissing: true, omitIfInvalid: true },
|
||||||
description: { strategy: "first_non_empty", paths: ["description", "descr"], coerce: "string", omitIfMissing: true },
|
description: { strategy: "first_non_empty", paths: ["description", "descr"], coerce: "string", omitIfMissing: true },
|
||||||
display_name: { strategy: "first_non_empty", paths: ["name"], coerce: "string" },
|
display_name: { strategy: "first_non_empty", paths: ["name"], coerce: "string" },
|
||||||
geometry_kind: { derive: "geometry_kind" },
|
geometry_kind: { derive: "geometry_kind" },
|
||||||
max_speed_kph: { strategy: "first_non_empty", paths: ["maxPermissibleSpeed", "max_permissible_speed"], coerce: "number", minimum: 0, omitIfInvalid: true },
|
max_speed_kph: { strategy: "first_non_empty", paths: ["maxPermissibleSpeed", "max_permissible_speed"], coerce: "number", minimum: 0, omitIfMissing: true, omitIfInvalid: true },
|
||||||
perimeter_meters: { strategy: "first_non_empty", paths: ["perimeter"], coerce: "number", minimum: 0, omitIfInvalid: true },
|
perimeter_meters: { strategy: "first_non_empty", paths: ["perimeter"], coerce: "number", minimum: 0, omitIfMissing: true, omitIfInvalid: true },
|
||||||
source_kind: { constant: "live_api" },
|
source_kind: { constant: "live_api" },
|
||||||
source_revision: { constant: "gelios-rest-v1" },
|
source_revision: { constant: "gelios-rest-v1" },
|
||||||
style_color: { strategy: "first_non_empty", paths: ["color"], coerce: "string", omitIfMissing: true },
|
style_color: { strategy: "first_non_empty", paths: ["color"], coerce: "string", omitIfMissing: true },
|
||||||
|
|
|
||||||
|
|
@ -138,7 +138,7 @@ value.collectionProfiles.push({
|
||||||
maxCurrentEntities: 5000,
|
maxCurrentEntities: 5000,
|
||||||
onExceed: "require_partitioned_data_product",
|
onExceed: "require_partitioned_data_product",
|
||||||
},
|
},
|
||||||
retry: { maxAttempts: 4, backoff: "exponential_with_jitter" },
|
retry: { maxAttempts: 4, backoff: "fixed_delay", delayMs: 2000 },
|
||||||
});
|
});
|
||||||
|
|
||||||
value.dataProducts.push({
|
value.dataProducts.push({
|
||||||
|
|
@ -232,6 +232,13 @@ value.l2Templates.push({
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The target Engine runtime materializes bounded fixed-delay retries exactly.
|
||||||
|
// v8 therefore avoids promising a backoff algorithm that n8n cannot execute.
|
||||||
|
value.collectionProfiles = value.collectionProfiles.map((profile) => ({
|
||||||
|
...profile,
|
||||||
|
retry: { maxAttempts: 4, backoff: "fixed_delay", delayMs: 2000 },
|
||||||
|
}));
|
||||||
|
|
||||||
export const geliosProviderPackageV8 = deepFreeze(value);
|
export const geliosProviderPackageV8 = deepFreeze(value);
|
||||||
|
|
||||||
function optional(type) {
|
function optional(type) {
|
||||||
|
|
|
||||||
|
|
@ -24,9 +24,12 @@ export {
|
||||||
export {
|
export {
|
||||||
L2_EXECUTION_PLAN_COMPILER_VERSION,
|
L2_EXECUTION_PLAN_COMPILER_VERSION,
|
||||||
L2_EXECUTION_PLAN_SCHEMA_VERSION,
|
L2_EXECUTION_PLAN_SCHEMA_VERSION,
|
||||||
|
L2_GRAPH_BLUEPRINT_SCHEMA_VERSION,
|
||||||
|
L2_MAPPING_RUNTIME_SCHEMA_VERSION,
|
||||||
L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION,
|
L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION,
|
||||||
attestL2ExecutionPlanMaterialization,
|
attestL2ExecutionPlanMaterialization,
|
||||||
compileL2ExecutionPlan,
|
compileL2ExecutionPlan,
|
||||||
|
compileL2GraphBlueprint,
|
||||||
validateL2ExecutionPlan,
|
validateL2ExecutionPlan,
|
||||||
} from "./l2-execution-plan.mjs";
|
} from "./l2-execution-plan.mjs";
|
||||||
export {
|
export {
|
||||||
|
|
|
||||||
|
|
@ -6,14 +6,23 @@ import {
|
||||||
} from "./telemetry-field-registry.mjs";
|
} from "./telemetry-field-registry.mjs";
|
||||||
|
|
||||||
export const L2_EXECUTION_PLAN_SCHEMA_VERSION = "nodedc.l2-execution-plan/v1";
|
export const L2_EXECUTION_PLAN_SCHEMA_VERSION = "nodedc.l2-execution-plan/v1";
|
||||||
|
export const L2_GRAPH_BLUEPRINT_SCHEMA_VERSION = "nodedc.l2-graph-blueprint/v1";
|
||||||
export const L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION = "nodedc.l2-materialization-receipt/v1";
|
export const L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION = "nodedc.l2-materialization-receipt/v1";
|
||||||
export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.0.0";
|
export const L2_MAPPING_RUNTIME_SCHEMA_VERSION = "nodedc.semantic-mapping-runtime/v1";
|
||||||
|
export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.1.0";
|
||||||
|
|
||||||
const HASH = /^(?:sha256:)?[a-f0-9]{64}$/;
|
const HASH = /^(?:sha256:)?[a-f0-9]{64}$/;
|
||||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||||
const CREDENTIAL_REF = /^ndc-credref:[A-Za-z0-9._:-]{8,255}$/;
|
const CREDENTIAL_REF = /^ndc-credref:[A-Za-z0-9._:-]{8,255}$/;
|
||||||
const COMPILE_OPTION_KEYS = new Set(["telemetryFieldRegistry"]);
|
const COMPILE_OPTION_KEYS = new Set(["telemetryFieldRegistry"]);
|
||||||
const RECEIPT_OPTION_KEYS = new Set(["graphRevision", "graphDigest", "materializedStepIds"]);
|
const RECEIPT_OPTION_KEYS = new Set(["graphRevision", "graphDigest", "materializedStepIds"]);
|
||||||
|
const STEP_RUNTIME_KINDS = Object.freeze({
|
||||||
|
collection_trigger: "ndc.collection-trigger",
|
||||||
|
provider_request: "ndc.provider-request",
|
||||||
|
extract_items: "ndc.extract-items",
|
||||||
|
semantic_mapping: "ndc.semantic-mapping",
|
||||||
|
data_product_publish: "ndc.data-product-publish",
|
||||||
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve one immutable provider package plus one connection instance into a
|
* Resolve one immutable provider package plus one connection instance into a
|
||||||
|
|
@ -86,6 +95,12 @@ export function compileL2ExecutionPlan(providerPackage, connectionInstance, opti
|
||||||
telemetryProjection,
|
telemetryProjection,
|
||||||
artifacts,
|
artifacts,
|
||||||
}));
|
}));
|
||||||
|
const graphBlueprint = compileGraphBlueprint({
|
||||||
|
profile,
|
||||||
|
template,
|
||||||
|
mapping,
|
||||||
|
steps,
|
||||||
|
});
|
||||||
|
|
||||||
const planWithoutDigest = {
|
const planWithoutDigest = {
|
||||||
schemaVersion: L2_EXECUTION_PLAN_SCHEMA_VERSION,
|
schemaVersion: L2_EXECUTION_PLAN_SCHEMA_VERSION,
|
||||||
|
|
@ -108,6 +123,7 @@ export function compileL2ExecutionPlan(providerPackage, connectionInstance, opti
|
||||||
collectionProfile: structuredClone(profile),
|
collectionProfile: structuredClone(profile),
|
||||||
artifacts,
|
artifacts,
|
||||||
steps,
|
steps,
|
||||||
|
graphBlueprint,
|
||||||
};
|
};
|
||||||
|
|
||||||
return deepFreeze({
|
return deepFreeze({
|
||||||
|
|
@ -128,6 +144,7 @@ export function validateL2ExecutionPlan(value) {
|
||||||
if (ids.some((id) => typeof id !== "string" || !IDENTIFIER.test(id))) errors.push("executionPlan.steps_id_invalid");
|
if (ids.some((id) => typeof id !== "string" || !IDENTIFIER.test(id))) errors.push("executionPlan.steps_id_invalid");
|
||||||
if (new Set(ids).size !== ids.length) errors.push("executionPlan.steps_id_must_be_unique");
|
if (new Set(ids).size !== ids.length) errors.push("executionPlan.steps_id_must_be_unique");
|
||||||
}
|
}
|
||||||
|
validateGraphBlueprint(value, errors);
|
||||||
if (!HASH.test(String(value.executionPlanDigest || ""))) {
|
if (!HASH.test(String(value.executionPlanDigest || ""))) {
|
||||||
errors.push("executionPlan.executionPlanDigest_invalid");
|
errors.push("executionPlan.executionPlanDigest_invalid");
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -140,6 +157,20 @@ export function validateL2ExecutionPlan(value) {
|
||||||
return result(errors);
|
return result(errors);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Produce the exact provider-neutral graph shape required by one resolved
|
||||||
|
* execution plan. Runtime adapters may translate these generic node kinds to
|
||||||
|
* their native graph representation, but may not add provider branches or
|
||||||
|
* silently weaken the declared execution policies.
|
||||||
|
*/
|
||||||
|
export function compileL2GraphBlueprint(executionPlan) {
|
||||||
|
const validation = validateL2ExecutionPlan(executionPlan);
|
||||||
|
if (!validation.ok) {
|
||||||
|
throw new Error(`l2_execution_plan_invalid:${validation.errors.join(",")}`);
|
||||||
|
}
|
||||||
|
return executionPlan.graphBlueprint;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Bind a successful Engine graph materialization back to the exact compiled
|
* Bind a successful Engine graph materialization back to the exact compiled
|
||||||
* plan without teaching this package how Engine stores or renders its graph.
|
* plan without teaching this package how Engine stores or renders its graph.
|
||||||
|
|
@ -245,6 +276,180 @@ function compileStep(context) {
|
||||||
throw new Error(`l2_execution_plan_step_kind_unsupported:${String(step.kind)}`);
|
throw new Error(`l2_execution_plan_step_kind_unsupported:${String(step.kind)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function compileGraphBlueprint({ profile, template, mapping, steps }) {
|
||||||
|
const nodes = steps.flatMap((step) => {
|
||||||
|
if (step.kind !== "collection_trigger") return [compileBlueprintNode(step, step.id)];
|
||||||
|
if (profile.mode !== "realtime") {
|
||||||
|
return [compileBlueprintNode(step, step.id, "ndc.manual-trigger", { entrypoint: "manual" })];
|
||||||
|
}
|
||||||
|
return [
|
||||||
|
compileBlueprintNode(
|
||||||
|
step,
|
||||||
|
`${step.id}.manual`,
|
||||||
|
"ndc.manual-trigger",
|
||||||
|
{ entrypoint: "manual" },
|
||||||
|
),
|
||||||
|
compileBlueprintNode(
|
||||||
|
step,
|
||||||
|
`${step.id}.interval`,
|
||||||
|
"ndc.interval-trigger",
|
||||||
|
{ entrypoint: "interval", intervalMs: profile.schedule.intervalMs },
|
||||||
|
),
|
||||||
|
];
|
||||||
|
}).map((node, order) => ({ ...node, order }));
|
||||||
|
const triggerNodes = nodes.filter((node) => node.stepId === steps[0].id);
|
||||||
|
const executableNodes = nodes.filter((node) => node.stepId !== steps[0].id);
|
||||||
|
const edgePairs = [
|
||||||
|
...triggerNodes.map((node) => ({ from: node.id, to: executableNodes[0].id })),
|
||||||
|
...executableNodes.slice(1).map((node, index) => ({
|
||||||
|
from: executableNodes[index].id,
|
||||||
|
to: node.id,
|
||||||
|
})),
|
||||||
|
];
|
||||||
|
const edges = edgePairs.map((edge, index) => ({
|
||||||
|
id: `edge.${String(index + 1).padStart(3, "0")}`,
|
||||||
|
...edge,
|
||||||
|
channel: "main",
|
||||||
|
order: 0,
|
||||||
|
}));
|
||||||
|
const blueprintWithoutDigest = {
|
||||||
|
schemaVersion: L2_GRAPH_BLUEPRINT_SCHEMA_VERSION,
|
||||||
|
runtime: {
|
||||||
|
target: template.runtime,
|
||||||
|
instanceMode: template.instanceMode,
|
||||||
|
executionMode: profile.mode,
|
||||||
|
entrypoints: profile.mode === "realtime"
|
||||||
|
? [
|
||||||
|
{ kind: "manual", purpose: "managed_deploy_and_run" },
|
||||||
|
{ kind: "interval", intervalMs: profile.schedule.intervalMs, purpose: "scheduled_collection" },
|
||||||
|
]
|
||||||
|
: [{ kind: "manual", purpose: "managed_deploy_and_run" }],
|
||||||
|
envelope: {
|
||||||
|
schemaVersion: "nodedc.l2-runtime-envelope/v1",
|
||||||
|
providerResponses: "keyed_by_capability_id",
|
||||||
|
collectionReceivedAt: "required_iso_timestamp",
|
||||||
|
canonicalFactsOnlyAtPublish: true,
|
||||||
|
rawProviderPayloadAtPublish: "forbidden",
|
||||||
|
maxProviderResponseBytes: 16 * 1024 * 1024,
|
||||||
|
maxExtractedItems: profile.cardinality.maxCurrentEntities,
|
||||||
|
},
|
||||||
|
retry: structuredClone(profile.retry),
|
||||||
|
batching: structuredClone(profile.batching),
|
||||||
|
cardinality: structuredClone(profile.cardinality),
|
||||||
|
failureMode: "fail_closed",
|
||||||
|
mappingRuntime: compileMappingRuntimeRequirements(mapping),
|
||||||
|
},
|
||||||
|
nodes,
|
||||||
|
edges,
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
...blueprintWithoutDigest,
|
||||||
|
blueprintDigest: canonicalDigest(blueprintWithoutDigest),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function compileBlueprintNode(step, id, runtimeKind = STEP_RUNTIME_KINDS[step.kind], adapterConfig) {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
stepId: step.id,
|
||||||
|
runtimeKind,
|
||||||
|
configDigest: canonicalDigest({
|
||||||
|
stepConfig: step.config,
|
||||||
|
...(adapterConfig ? { adapterConfig } : {}),
|
||||||
|
}),
|
||||||
|
...(adapterConfig ? { adapterConfig } : {}),
|
||||||
|
credentialSlots: step.config?.credentialBinding
|
||||||
|
? [{
|
||||||
|
role: step.config.credentialBinding,
|
||||||
|
slot: step.config.credentialBinding,
|
||||||
|
authority: "engine_managed_opaque_ref",
|
||||||
|
}]
|
||||||
|
: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function compileMappingRuntimeRequirements(mapping) {
|
||||||
|
const expressions = [];
|
||||||
|
collectMappingExpressions(mapping.fact, expressions);
|
||||||
|
const geometry = mapping.fact?.geometry;
|
||||||
|
return {
|
||||||
|
schemaVersion: L2_MAPPING_RUNTIME_SCHEMA_VERSION,
|
||||||
|
expressionSources: uniqueSorted(expressions.map(expressionSourceKind)),
|
||||||
|
coercions: uniqueSorted(expressions.map((expression) => expression.coerce).filter(Boolean)),
|
||||||
|
derivationKinds: uniqueSorted(Object.values(mapping.derivations || {}).map((item) => item.kind)),
|
||||||
|
ruleIds: uniqueSorted(
|
||||||
|
Object.values(mapping.derivations || {}).flatMap((item) => item.rules || []),
|
||||||
|
),
|
||||||
|
geometryStrategies: geometry
|
||||||
|
? [geometry.type === "Point" ? "point_from_longitude_latitude" : geometry.strategy]
|
||||||
|
: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectMappingExpressions(value, found) {
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
value.forEach((item) => collectMappingExpressions(item, found));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!isPlainObject(value)) return;
|
||||||
|
if (Array.isArray(value.paths) || value.constant !== undefined || typeof value.derive === "string") {
|
||||||
|
found.push(value);
|
||||||
|
}
|
||||||
|
Object.values(value).forEach((item) => collectMappingExpressions(item, found));
|
||||||
|
}
|
||||||
|
|
||||||
|
function expressionSourceKind(value) {
|
||||||
|
if (Array.isArray(value.paths)) return value.strategy || "first_non_empty";
|
||||||
|
if (value.constant !== undefined) return "constant";
|
||||||
|
return "derive";
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateGraphBlueprint(executionPlan, errors) {
|
||||||
|
if (!isPlainObject(executionPlan.graphBlueprint)) {
|
||||||
|
errors.push("executionPlan.graphBlueprint_must_be_object");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const blueprint = executionPlan.graphBlueprint;
|
||||||
|
if (blueprint.schemaVersion !== L2_GRAPH_BLUEPRINT_SCHEMA_VERSION) {
|
||||||
|
errors.push("executionPlan.graphBlueprint.schemaVersion_mismatch");
|
||||||
|
}
|
||||||
|
if (!HASH.test(String(blueprint.blueprintDigest || ""))) {
|
||||||
|
errors.push("executionPlan.graphBlueprint.blueprintDigest_invalid");
|
||||||
|
} else {
|
||||||
|
const descriptor = structuredClone(blueprint);
|
||||||
|
delete descriptor.blueprintDigest;
|
||||||
|
if (normalizeDigest(blueprint.blueprintDigest) !== canonicalDigest(descriptor)) {
|
||||||
|
errors.push("executionPlan.graphBlueprint.blueprintDigest_mismatch");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!Array.isArray(executionPlan.steps) || !isPlainObject(executionPlan.collectionProfile)) return;
|
||||||
|
try {
|
||||||
|
const expected = compileGraphBlueprintDescriptor({
|
||||||
|
profile: executionPlan.collectionProfile,
|
||||||
|
steps: executionPlan.steps,
|
||||||
|
mapping: executionPlan.steps.find((step) => step?.kind === "semantic_mapping")?.config?.mappingContract,
|
||||||
|
});
|
||||||
|
if (stableJson(blueprint) !== stableJson(expected)) {
|
||||||
|
errors.push("executionPlan.graphBlueprint_mismatch");
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
errors.push("executionPlan.graphBlueprint_unverifiable");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function compileGraphBlueprintDescriptor({ profile, steps, mapping }) {
|
||||||
|
if (!isPlainObject(mapping)) throw new Error("graph_blueprint_context_invalid");
|
||||||
|
const template = {
|
||||||
|
runtime: "ndc_l2",
|
||||||
|
instanceMode: "one_connection_per_workflow",
|
||||||
|
};
|
||||||
|
return compileGraphBlueprint({ profile, template, mapping, steps });
|
||||||
|
}
|
||||||
|
|
||||||
|
function uniqueSorted(values) {
|
||||||
|
return [...new Set(values)].sort();
|
||||||
|
}
|
||||||
|
|
||||||
function validateConnectionInstance(providerPackage, value) {
|
function validateConnectionInstance(providerPackage, value) {
|
||||||
if (!isPlainObject(value)) throw new Error("l2_execution_plan_connection_must_be_object");
|
if (!isPlainObject(value)) throw new Error("l2_execution_plan_connection_must_be_object");
|
||||||
if (value.schemaVersion !== "nodedc.l2-connection-instance/v1") {
|
if (value.schemaVersion !== "nodedc.l2-connection-instance/v1") {
|
||||||
|
|
|
||||||
|
|
@ -144,6 +144,8 @@ const EXPRESSION_KEYS = new Set([
|
||||||
]);
|
]);
|
||||||
const GEOMETRY_KEYS = new Set([
|
const GEOMETRY_KEYS = new Set([
|
||||||
"type", "longitude", "latitude", "strategy", "paths", "allowedTypes", "omitIfInvalid",
|
"type", "longitude", "latitude", "strategy", "paths", "allowedTypes", "omitIfInvalid",
|
||||||
|
"kindPath", "polygonPointsPath", "corridorPointsPath", "radiusPath",
|
||||||
|
"coordinateOrder", "circleSegments",
|
||||||
]);
|
]);
|
||||||
const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]);
|
const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]);
|
||||||
const TEMPLATE_KEYS = new Set([
|
const TEMPLATE_KEYS = new Set([
|
||||||
|
|
@ -593,9 +595,18 @@ function validateCollectionProfileTemplate(value, path, errors) {
|
||||||
if (!isPlainObject(value.retry)) {
|
if (!isPlainObject(value.retry)) {
|
||||||
errors.push(`${path}.retry_must_be_object`);
|
errors.push(`${path}.retry_must_be_object`);
|
||||||
} else {
|
} else {
|
||||||
rejectUnknownKeys(value.retry, new Set(["maxAttempts", "backoff"]), `${path}.retry`, errors);
|
rejectUnknownKeys(value.retry, new Set(["maxAttempts", "backoff", "delayMs"]), `${path}.retry`, errors);
|
||||||
if (!Number.isInteger(value.retry.maxAttempts) || value.retry.maxAttempts < 1 || value.retry.maxAttempts > 10) errors.push(`${path}.retry.maxAttempts_invalid`);
|
if (!Number.isInteger(value.retry.maxAttempts) || value.retry.maxAttempts < 1 || value.retry.maxAttempts > 10) errors.push(`${path}.retry.maxAttempts_invalid`);
|
||||||
if (value.retry.backoff !== "exponential_with_jitter") errors.push(`${path}.retry.backoff_invalid`);
|
if (!["exponential_with_jitter", "fixed_delay"].includes(value.retry.backoff)) {
|
||||||
|
errors.push(`${path}.retry.backoff_invalid`);
|
||||||
|
}
|
||||||
|
if (value.retry.backoff === "fixed_delay") {
|
||||||
|
if (!Number.isInteger(value.retry.delayMs) || value.retry.delayMs < 100 || value.retry.delayMs > 60_000) {
|
||||||
|
errors.push(`${path}.retry.delayMs_invalid`);
|
||||||
|
}
|
||||||
|
} else if (value.retry.delayMs !== undefined) {
|
||||||
|
errors.push(`${path}.retry.delayMs_requires_fixed_delay`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -672,13 +683,29 @@ function validateMappingContract(value, path, errors) {
|
||||||
errors.push(`${path}.fact.geometry.point_shape_invalid`);
|
errors.push(`${path}.fact.geometry.point_shape_invalid`);
|
||||||
}
|
}
|
||||||
} else if (value.fact.geometry.type === "GeoJSON") {
|
} else if (value.fact.geometry.type === "GeoJSON") {
|
||||||
if (value.fact.geometry.strategy !== "gelios_geozone_v1") errors.push(`${path}.fact.geometry.strategy_invalid`);
|
if (value.fact.geometry.strategy !== "bounded_zone_geometry_v1") errors.push(`${path}.fact.geometry.strategy_invalid`);
|
||||||
requiredSourcePathArray(value.fact.geometry.paths, `${path}.fact.geometry.paths`, errors);
|
for (const key of ["kindPath", "polygonPointsPath", "corridorPointsPath", "radiusPath"]) {
|
||||||
|
if (!isCanonicalSourcePath(value.fact.geometry[key])) {
|
||||||
|
errors.push(`${path}.fact.geometry.${key}_invalid`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!["latitude_longitude", "longitude_latitude"].includes(value.fact.geometry.coordinateOrder)) {
|
||||||
|
errors.push(`${path}.fact.geometry.coordinateOrder_invalid`);
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(value.fact.geometry.circleSegments)
|
||||||
|
|| value.fact.geometry.circleSegments < 16
|
||||||
|
|| value.fact.geometry.circleSegments > 256) {
|
||||||
|
errors.push(`${path}.fact.geometry.circleSegments_invalid`);
|
||||||
|
}
|
||||||
if (!sameSet(value.fact.geometry.allowedTypes, ["Polygon", "MultiPolygon"])) {
|
if (!sameSet(value.fact.geometry.allowedTypes, ["Polygon", "MultiPolygon"])) {
|
||||||
errors.push(`${path}.fact.geometry.allowedTypes_must_be_polygon_or_multipolygon`);
|
errors.push(`${path}.fact.geometry.allowedTypes_must_be_polygon_or_multipolygon`);
|
||||||
}
|
}
|
||||||
if (value.fact.geometry.omitIfInvalid !== false) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_false`);
|
if (value.fact.geometry.omitIfInvalid !== false) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_false`);
|
||||||
if (value.fact.geometry.longitude !== undefined || value.fact.geometry.latitude !== undefined) {
|
if (
|
||||||
|
value.fact.geometry.longitude !== undefined
|
||||||
|
|| value.fact.geometry.latitude !== undefined
|
||||||
|
|| value.fact.geometry.paths !== undefined
|
||||||
|
) {
|
||||||
errors.push(`${path}.fact.geometry.geojson_shape_invalid`);
|
errors.push(`${path}.fact.geometry.geojson_shape_invalid`);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -900,6 +927,9 @@ function validateExpressionAgainstFieldContract(expression, contract, derivation
|
||||||
errors.push(`${path}_required_but_mapping_can_omit_invalid`);
|
errors.push(`${path}_required_but_mapping_can_omit_invalid`);
|
||||||
}
|
}
|
||||||
if (expression.paths !== undefined) {
|
if (expression.paths !== undefined) {
|
||||||
|
if (contract.required === false && expression.omitIfMissing !== true) {
|
||||||
|
errors.push(`${path}_optional_source_field_must_omit_missing`);
|
||||||
|
}
|
||||||
const expectedCoerce = new Map([["string", "string"], ["number", "number"], ["boolean", "boolean"]]).get(contract.type);
|
const expectedCoerce = new Map([["string", "string"], ["number", "number"], ["boolean", "boolean"]]).get(contract.type);
|
||||||
if (expectedCoerce && expression.coerce !== expectedCoerce) errors.push(`${path}.coerce_must_match_field_contract`);
|
if (expectedCoerce && expression.coerce !== expectedCoerce) errors.push(`${path}.coerce_must_match_field_contract`);
|
||||||
if (Array.isArray(contract.enum)) errors.push(`${path}.enum_field_must_use_constant_or_derivation`);
|
if (Array.isArray(contract.enum)) errors.push(`${path}.enum_field_must_use_constant_or_derivation`);
|
||||||
|
|
@ -1041,8 +1071,11 @@ function mappingSourcePaths(value, found = []) {
|
||||||
if (isCanonicalSourcePath(path)) found.push(path);
|
if (isCanonicalSourcePath(path)) found.push(path);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
for (const key of ["kindPath", "polygonPointsPath", "corridorPointsPath", "radiusPath"]) {
|
||||||
|
if (isCanonicalSourcePath(value[key])) found.push(value[key]);
|
||||||
|
}
|
||||||
Object.entries(value).forEach(([key, item]) => {
|
Object.entries(value).forEach(([key, item]) => {
|
||||||
if (key !== "paths") mappingSourcePaths(item, found);
|
if (key !== "paths" && !key.endsWith("Path")) mappingSourcePaths(item, found);
|
||||||
});
|
});
|
||||||
return found;
|
return found;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,13 @@
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
import {
|
import {
|
||||||
L2_EXECUTION_PLAN_SCHEMA_VERSION,
|
L2_EXECUTION_PLAN_SCHEMA_VERSION,
|
||||||
|
L2_GRAPH_BLUEPRINT_SCHEMA_VERSION,
|
||||||
|
L2_MAPPING_RUNTIME_SCHEMA_VERSION,
|
||||||
L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION,
|
L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION,
|
||||||
attestL2ExecutionPlanMaterialization,
|
attestL2ExecutionPlanMaterialization,
|
||||||
compileL2ExecutionPlan,
|
compileL2ExecutionPlan,
|
||||||
|
compileL2GraphBlueprint,
|
||||||
instantiateL2Connection,
|
instantiateL2Connection,
|
||||||
validateL2ExecutionPlan,
|
validateL2ExecutionPlan,
|
||||||
} from "../src/index.mjs";
|
} from "../src/index.mjs";
|
||||||
|
|
@ -30,6 +34,58 @@ assert.match(positionsPlan.executionPlanDigest, /^sha256:[a-f0-9]{64}$/);
|
||||||
assert.match(positionsPlan.artifacts.packageDigest, /^sha256:[a-f0-9]{64}$/);
|
assert.match(positionsPlan.artifacts.packageDigest, /^sha256:[a-f0-9]{64}$/);
|
||||||
assert.match(positionsPlan.artifacts.mappingContractDigest, /^sha256:[a-f0-9]{64}$/);
|
assert.match(positionsPlan.artifacts.mappingContractDigest, /^sha256:[a-f0-9]{64}$/);
|
||||||
assert.match(positionsPlan.artifacts.telemetryRegistryDigest, /^sha256:[a-f0-9]{64}$/);
|
assert.match(positionsPlan.artifacts.telemetryRegistryDigest, /^sha256:[a-f0-9]{64}$/);
|
||||||
|
assert.equal(positionsPlan.graphBlueprint.schemaVersion, L2_GRAPH_BLUEPRINT_SCHEMA_VERSION);
|
||||||
|
assert.equal(
|
||||||
|
positionsPlan.graphBlueprint.runtime.mappingRuntime.schemaVersion,
|
||||||
|
L2_MAPPING_RUNTIME_SCHEMA_VERSION,
|
||||||
|
);
|
||||||
|
assert.match(positionsPlan.graphBlueprint.blueprintDigest, /^sha256:[a-f0-9]{64}$/);
|
||||||
|
assert.equal(compileL2GraphBlueprint(positionsPlan), positionsPlan.graphBlueprint);
|
||||||
|
assert.deepEqual(
|
||||||
|
positionsPlan.graphBlueprint.nodes.map(({ id, stepId, runtimeKind, order }) => ({
|
||||||
|
id,
|
||||||
|
stepId,
|
||||||
|
runtimeKind,
|
||||||
|
order,
|
||||||
|
})),
|
||||||
|
[
|
||||||
|
{ id: "collection.trigger.manual", stepId: "collection.trigger", runtimeKind: "ndc.manual-trigger", order: 0 },
|
||||||
|
{ id: "collection.trigger.interval", stepId: "collection.trigger", runtimeKind: "ndc.interval-trigger", order: 1 },
|
||||||
|
{ id: "provider.fetch-monitoring-config", stepId: "provider.fetch-monitoring-config", runtimeKind: "ndc.provider-request", order: 2 },
|
||||||
|
{ id: "provider.fetch-units", stepId: "provider.fetch-units", runtimeKind: "ndc.provider-request", order: 3 },
|
||||||
|
{ id: "provider.extract-units", stepId: "provider.extract-units", runtimeKind: "ndc.extract-items", order: 4 },
|
||||||
|
{ id: "ontology.map", stepId: "ontology.map", runtimeKind: "ndc.semantic-mapping", order: 5 },
|
||||||
|
{ id: "data-product.publish", stepId: "data-product.publish", runtimeKind: "ndc.data-product-publish", order: 6 },
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert.deepEqual(
|
||||||
|
positionsPlan.graphBlueprint.edges.map(({ from, to }) => ({ from, to })),
|
||||||
|
[
|
||||||
|
{ from: "collection.trigger.manual", to: "provider.fetch-monitoring-config" },
|
||||||
|
{ from: "collection.trigger.interval", to: "provider.fetch-monitoring-config" },
|
||||||
|
{ from: "provider.fetch-monitoring-config", to: "provider.fetch-units" },
|
||||||
|
{ from: "provider.fetch-units", to: "provider.extract-units" },
|
||||||
|
{ from: "provider.extract-units", to: "ontology.map" },
|
||||||
|
{ from: "ontology.map", to: "data-product.publish" },
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert.deepEqual(positionsPlan.graphBlueprint.runtime.mappingRuntime.derivationKinds, [
|
||||||
|
"bounded_readings",
|
||||||
|
"ordered_rules",
|
||||||
|
]);
|
||||||
|
assert.deepEqual(positionsPlan.graphBlueprint.runtime.retry, {
|
||||||
|
maxAttempts: 4,
|
||||||
|
backoff: "fixed_delay",
|
||||||
|
delayMs: 2000,
|
||||||
|
});
|
||||||
|
assert.equal(
|
||||||
|
JSON.stringify(positionsPlan.graphBlueprint).includes("ndc-credref:"),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
positionsPlan.graphBlueprint.nodes.every((node) => !node.runtimeKind.includes("gelios")),
|
||||||
|
true,
|
||||||
|
);
|
||||||
assert.deepEqual(positionsPlan.steps.map((step) => step.kind), [
|
assert.deepEqual(positionsPlan.steps.map((step) => step.kind), [
|
||||||
"collection_trigger",
|
"collection_trigger",
|
||||||
"provider_request",
|
"provider_request",
|
||||||
|
|
@ -54,6 +110,7 @@ assert.deepEqual(
|
||||||
);
|
);
|
||||||
assert.equal(Object.isFrozen(positionsPlan), true);
|
assert.equal(Object.isFrozen(positionsPlan), true);
|
||||||
assert.equal(Object.isFrozen(positionsPlan.steps), true);
|
assert.equal(Object.isFrozen(positionsPlan.steps), true);
|
||||||
|
assert.equal(Object.isFrozen(positionsPlan.graphBlueprint), true);
|
||||||
|
|
||||||
const clonedPlan = compileL2ExecutionPlan(
|
const clonedPlan = compileL2ExecutionPlan(
|
||||||
structuredClone(geliosProviderPackageV8),
|
structuredClone(geliosProviderPackageV8),
|
||||||
|
|
@ -77,6 +134,28 @@ const profilePlan = compileL2ExecutionPlan(geliosProviderPackageV8, profileConne
|
||||||
assert.deepEqual(validateL2ExecutionPlan(profilePlan), { ok: true, errors: [] });
|
assert.deepEqual(validateL2ExecutionPlan(profilePlan), { ok: true, errors: [] });
|
||||||
assert.equal(profilePlan.artifacts.telemetryRegistryDigest, undefined);
|
assert.equal(profilePlan.artifacts.telemetryRegistryDigest, undefined);
|
||||||
assert.notEqual(profilePlan.executionPlanDigest, positionsPlan.executionPlanDigest);
|
assert.notEqual(profilePlan.executionPlanDigest, positionsPlan.executionPlanDigest);
|
||||||
|
assert.deepEqual(profilePlan.graphBlueprint.runtime.mappingRuntime.derivationKinds, []);
|
||||||
|
|
||||||
|
for (const collectionProfile of geliosProviderPackageV8.collectionProfiles) {
|
||||||
|
const connection = instantiateL2Connection(geliosProviderPackageV8, {
|
||||||
|
tenantId: "tenant-all-profiles-fixture",
|
||||||
|
connectionId: `connection-${collectionProfile.id.replaceAll(".", "-")}`,
|
||||||
|
collectionProfileId: collectionProfile.id,
|
||||||
|
providerCredentialRef: "ndc-credref:provider-all-profiles-fixture",
|
||||||
|
});
|
||||||
|
const plan = compileL2ExecutionPlan(
|
||||||
|
geliosProviderPackageV8,
|
||||||
|
connection,
|
||||||
|
collectionProfile.dataProductId === "fleet.positions.current.v5"
|
||||||
|
? { telemetryFieldRegistry: geliosTelemetryFieldRegistryV1 }
|
||||||
|
: {},
|
||||||
|
);
|
||||||
|
assert.deepEqual(validateL2ExecutionPlan(plan), { ok: true, errors: [] });
|
||||||
|
assert.equal(JSON.stringify(plan.graphBlueprint.runtime.mappingRuntime).includes("gelios"), false);
|
||||||
|
const expectedEntrypoints = collectionProfile.mode === "realtime" ? 2 : 1;
|
||||||
|
assert.equal(plan.graphBlueprint.nodes.length, plan.steps.length + expectedEntrypoints - 1);
|
||||||
|
assert.equal(plan.graphBlueprint.edges.length, plan.steps.length + expectedEntrypoints - 2);
|
||||||
|
}
|
||||||
|
|
||||||
const secondConnection = instantiateL2Connection(geliosProviderPackageV8, {
|
const secondConnection = instantiateL2Connection(geliosProviderPackageV8, {
|
||||||
tenantId: "tenant-compiler-fixture",
|
tenantId: "tenant-compiler-fixture",
|
||||||
|
|
@ -115,7 +194,39 @@ assert.equal(
|
||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const authorityEscalatedPlan = structuredClone(positionsPlan);
|
||||||
|
authorityEscalatedPlan.graphBlueprint.runtime.target = "provider_specific_runtime";
|
||||||
|
authorityEscalatedPlan.graphBlueprint.blueprintDigest = digestWithout(
|
||||||
|
authorityEscalatedPlan.graphBlueprint,
|
||||||
|
"blueprintDigest",
|
||||||
|
);
|
||||||
|
authorityEscalatedPlan.executionPlanDigest = digestWithout(
|
||||||
|
authorityEscalatedPlan,
|
||||||
|
"executionPlanDigest",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
validateL2ExecutionPlan(authorityEscalatedPlan).errors.includes("executionPlan.graphBlueprint_mismatch"),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
|
||||||
const serialized = JSON.stringify(positionsPlan);
|
const serialized = JSON.stringify(positionsPlan);
|
||||||
assert.equal(serialized.includes("Bearer "), false);
|
assert.equal(serialized.includes("Bearer "), false);
|
||||||
assert.equal(serialized.includes("access_token"), false);
|
assert.equal(serialized.includes("access_token"), false);
|
||||||
assert.equal(serialized.includes("refresh_token"), false);
|
assert.equal(serialized.includes("refresh_token"), false);
|
||||||
|
|
||||||
|
function digestWithout(value, key) {
|
||||||
|
const descriptor = structuredClone(value);
|
||||||
|
delete descriptor[key];
|
||||||
|
return `sha256:${createHash("sha256").update(JSON.stringify(stableValue(descriptor)), "utf8").digest("hex")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stableValue(value) {
|
||||||
|
if (Array.isArray(value)) return value.map(stableValue);
|
||||||
|
if (!value || typeof value !== "object") return value;
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.keys(value)
|
||||||
|
.filter((key) => value[key] !== undefined)
|
||||||
|
.sort()
|
||||||
|
.map((key) => [key, stableValue(value[key])]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue