SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(provider-contract): define materializable L2 graph blueprints

Browse Source
This commit is contained in:
Codex
2026-07-23 20:22:22 +03:00
parent a5a5644dbf
commit 4cf3a83ef1
7 changed files with 388 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/external-provider-contract/src/index.mjs
+3
View File
@@ -24,9 +24,12 @@ export {
export {
L2_EXECUTION_PLAN_COMPILER_VERSION,
L2_EXECUTION_PLAN_SCHEMA_VERSION,
L2_GRAPH_BLUEPRINT_SCHEMA_VERSION,
L2_MAPPING_RUNTIME_SCHEMA_VERSION,
L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION,
attestL2ExecutionPlanMaterialization,
compileL2ExecutionPlan,
compileL2GraphBlueprint,
validateL2ExecutionPlan,
} from "./l2-execution-plan.mjs";
export {
packages/external-provider-contract/src/l2-execution-plan.mjs
+206 -1
View File
@@ -6,14 +6,23 @@ import {
} from "./telemetry-field-registry.mjs";
export const L2_EXECUTION_PLAN_SCHEMA_VERSION = "nodedc.l2-execution-plan/v1";
export const L2_GRAPH_BLUEPRINT_SCHEMA_VERSION = "nodedc.l2-graph-blueprint/v1";
export const L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION = "nodedc.l2-materialization-receipt/v1";
export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.0.0";
export const L2_MAPPING_RUNTIME_SCHEMA_VERSION = "nodedc.semantic-mapping-runtime/v1";
export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.1.0";
const HASH = /^(?:sha256:)?[a-f0-9]{64}$/;
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const CREDENTIAL_REF = /^ndc-credref:[A-Za-z0-9._:-]{8,255}$/;
const COMPILE_OPTION_KEYS = new Set(["telemetryFieldRegistry"]);
const RECEIPT_OPTION_KEYS = new Set(["graphRevision", "graphDigest", "materializedStepIds"]);
const STEP_RUNTIME_KINDS = Object.freeze({
collection_trigger: "ndc.collection-trigger",
provider_request: "ndc.provider-request",
extract_items: "ndc.extract-items",
semantic_mapping: "ndc.semantic-mapping",
data_product_publish: "ndc.data-product-publish",
});
/**
* Resolve one immutable provider package plus one connection instance into a
@@ -86,6 +95,12 @@ export function compileL2ExecutionPlan(providerPackage, connectionInstance, opti
telemetryProjection,
artifacts,
}));
const graphBlueprint = compileGraphBlueprint({
profile,
template,
mapping,
steps,
});
const planWithoutDigest = {
schemaVersion: L2_EXECUTION_PLAN_SCHEMA_VERSION,
@@ -108,6 +123,7 @@ export function compileL2ExecutionPlan(providerPackage, connectionInstance, opti
collectionProfile: structuredClone(profile),
artifacts,
steps,
graphBlueprint,
};
return deepFreeze({
@@ -128,6 +144,7 @@ export function validateL2ExecutionPlan(value) {
if (ids.some((id) => typeof id !== "string" || !IDENTIFIER.test(id))) errors.push("executionPlan.steps_id_invalid");
if (new Set(ids).size !== ids.length) errors.push("executionPlan.steps_id_must_be_unique");
}
validateGraphBlueprint(value, errors);
if (!HASH.test(String(value.executionPlanDigest || ""))) {
errors.push("executionPlan.executionPlanDigest_invalid");
} else {
@@ -140,6 +157,20 @@ export function validateL2ExecutionPlan(value) {
return result(errors);
}
/**
* Produce the exact provider-neutral graph shape required by one resolved
* execution plan. Runtime adapters may translate these generic node kinds to
* their native graph representation, but may not add provider branches or
* silently weaken the declared execution policies.
*/
export function compileL2GraphBlueprint(executionPlan) {
const validation = validateL2ExecutionPlan(executionPlan);
if (!validation.ok) {
throw new Error(`l2_execution_plan_invalid:${validation.errors.join(",")}`);
}
return executionPlan.graphBlueprint;
}
/**
* Bind a successful Engine graph materialization back to the exact compiled
* plan without teaching this package how Engine stores or renders its graph.
@@ -245,6 +276,180 @@ function compileStep(context) {
throw new Error(`l2_execution_plan_step_kind_unsupported:${String(step.kind)}`);
}
function compileGraphBlueprint({ profile, template, mapping, steps }) {
const nodes = steps.flatMap((step) => {
if (step.kind !== "collection_trigger") return [compileBlueprintNode(step, step.id)];
if (profile.mode !== "realtime") {
return [compileBlueprintNode(step, step.id, "ndc.manual-trigger", { entrypoint: "manual" })];
}
return [
compileBlueprintNode(
step,
`${step.id}.manual`,
"ndc.manual-trigger",
{ entrypoint: "manual" },
),
compileBlueprintNode(
step,
`${step.id}.interval`,
"ndc.interval-trigger",
{ entrypoint: "interval", intervalMs: profile.schedule.intervalMs },
),
];
}).map((node, order) => ({ ...node, order }));
const triggerNodes = nodes.filter((node) => node.stepId === steps[0].id);
const executableNodes = nodes.filter((node) => node.stepId !== steps[0].id);
const edgePairs = [
...triggerNodes.map((node) => ({ from: node.id, to: executableNodes[0].id })),
...executableNodes.slice(1).map((node, index) => ({
from: executableNodes[index].id,
to: node.id,
})),
];
const edges = edgePairs.map((edge, index) => ({
id: `edge.${String(index + 1).padStart(3, "0")}`,
...edge,
channel: "main",
order: 0,
}));
const blueprintWithoutDigest = {
schemaVersion: L2_GRAPH_BLUEPRINT_SCHEMA_VERSION,
runtime: {
target: template.runtime,
instanceMode: template.instanceMode,
executionMode: profile.mode,
entrypoints: profile.mode === "realtime"
? [
{ kind: "manual", purpose: "managed_deploy_and_run" },
{ kind: "interval", intervalMs: profile.schedule.intervalMs, purpose: "scheduled_collection" },
]
: [{ kind: "manual", purpose: "managed_deploy_and_run" }],
envelope: {
schemaVersion: "nodedc.l2-runtime-envelope/v1",
providerResponses: "keyed_by_capability_id",
collectionReceivedAt: "required_iso_timestamp",
canonicalFactsOnlyAtPublish: true,
rawProviderPayloadAtPublish: "forbidden",
maxProviderResponseBytes: 16 * 1024 * 1024,
maxExtractedItems: profile.cardinality.maxCurrentEntities,
},
retry: structuredClone(profile.retry),
batching: structuredClone(profile.batching),
cardinality: structuredClone(profile.cardinality),
failureMode: "fail_closed",
mappingRuntime: compileMappingRuntimeRequirements(mapping),
},
nodes,
edges,
};
return {
...blueprintWithoutDigest,
blueprintDigest: canonicalDigest(blueprintWithoutDigest),
};
}
function compileBlueprintNode(step, id, runtimeKind = STEP_RUNTIME_KINDS[step.kind], adapterConfig) {
return {
id,
stepId: step.id,
runtimeKind,
configDigest: canonicalDigest({
stepConfig: step.config,
...(adapterConfig ? { adapterConfig } : {}),
}),
...(adapterConfig ? { adapterConfig } : {}),
credentialSlots: step.config?.credentialBinding
? [{
role: step.config.credentialBinding,
slot: step.config.credentialBinding,
authority: "engine_managed_opaque_ref",
}]
: [],
};
}
function compileMappingRuntimeRequirements(mapping) {
const expressions = [];
collectMappingExpressions(mapping.fact, expressions);
const geometry = mapping.fact?.geometry;
return {
schemaVersion: L2_MAPPING_RUNTIME_SCHEMA_VERSION,
expressionSources: uniqueSorted(expressions.map(expressionSourceKind)),
coercions: uniqueSorted(expressions.map((expression) => expression.coerce).filter(Boolean)),
derivationKinds: uniqueSorted(Object.values(mapping.derivations || {}).map((item) => item.kind)),
ruleIds: uniqueSorted(
Object.values(mapping.derivations || {}).flatMap((item) => item.rules || []),
),
geometryStrategies: geometry
? [geometry.type === "Point" ? "point_from_longitude_latitude" : geometry.strategy]
: [],
};
}
function collectMappingExpressions(value, found) {
if (Array.isArray(value)) {
value.forEach((item) => collectMappingExpressions(item, found));
return;
}
if (!isPlainObject(value)) return;
if (Array.isArray(value.paths) || value.constant !== undefined || typeof value.derive === "string") {
found.push(value);
}
Object.values(value).forEach((item) => collectMappingExpressions(item, found));
}
function expressionSourceKind(value) {
if (Array.isArray(value.paths)) return value.strategy || "first_non_empty";
if (value.constant !== undefined) return "constant";
return "derive";
}
function validateGraphBlueprint(executionPlan, errors) {
if (!isPlainObject(executionPlan.graphBlueprint)) {
errors.push("executionPlan.graphBlueprint_must_be_object");
return;
}
const blueprint = executionPlan.graphBlueprint;
if (blueprint.schemaVersion !== L2_GRAPH_BLUEPRINT_SCHEMA_VERSION) {
errors.push("executionPlan.graphBlueprint.schemaVersion_mismatch");
}
if (!HASH.test(String(blueprint.blueprintDigest || ""))) {
errors.push("executionPlan.graphBlueprint.blueprintDigest_invalid");
} else {
const descriptor = structuredClone(blueprint);
delete descriptor.blueprintDigest;
if (normalizeDigest(blueprint.blueprintDigest) !== canonicalDigest(descriptor)) {
errors.push("executionPlan.graphBlueprint.blueprintDigest_mismatch");
}
}
if (!Array.isArray(executionPlan.steps) || !isPlainObject(executionPlan.collectionProfile)) return;
try {
const expected = compileGraphBlueprintDescriptor({
profile: executionPlan.collectionProfile,
steps: executionPlan.steps,
mapping: executionPlan.steps.find((step) => step?.kind === "semantic_mapping")?.config?.mappingContract,
});
if (stableJson(blueprint) !== stableJson(expected)) {
errors.push("executionPlan.graphBlueprint_mismatch");
}
} catch {
errors.push("executionPlan.graphBlueprint_unverifiable");
}
}
function compileGraphBlueprintDescriptor({ profile, steps, mapping }) {
if (!isPlainObject(mapping)) throw new Error("graph_blueprint_context_invalid");
const template = {
runtime: "ndc_l2",
instanceMode: "one_connection_per_workflow",
};
return compileGraphBlueprint({ profile, template, mapping, steps });
}
function uniqueSorted(values) {
return [...new Set(values)].sort();
}
function validateConnectionInstance(providerPackage, value) {
if (!isPlainObject(value)) throw new Error("l2_execution_plan_connection_must_be_object");
if (value.schemaVersion !== "nodedc.l2-connection-instance/v1") {
packages/external-provider-contract/src/provider-package.mjs
+39 -6
View File
@@ -144,6 +144,8 @@ const EXPRESSION_KEYS = new Set([
]);
const GEOMETRY_KEYS = new Set([
"type", "longitude", "latitude", "strategy", "paths", "allowedTypes", "omitIfInvalid",
"kindPath", "polygonPointsPath", "corridorPointsPath", "radiusPath",
"coordinateOrder", "circleSegments",
]);
const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]);
const TEMPLATE_KEYS = new Set([
@@ -593,9 +595,18 @@ function validateCollectionProfileTemplate(value, path, errors) {
if (!isPlainObject(value.retry)) {
errors.push(`${path}.retry_must_be_object`);
} else {
rejectUnknownKeys(value.retry, new Set(["maxAttempts", "backoff"]), `${path}.retry`, errors);
rejectUnknownKeys(value.retry, new Set(["maxAttempts", "backoff", "delayMs"]), `${path}.retry`, errors);
if (!Number.isInteger(value.retry.maxAttempts) || value.retry.maxAttempts < 1 || value.retry.maxAttempts > 10) errors.push(`${path}.retry.maxAttempts_invalid`);
if (value.retry.backoff !== "exponential_with_jitter") errors.push(`${path}.retry.backoff_invalid`);
if (!["exponential_with_jitter", "fixed_delay"].includes(value.retry.backoff)) {
errors.push(`${path}.retry.backoff_invalid`);
}
if (value.retry.backoff === "fixed_delay") {
if (!Number.isInteger(value.retry.delayMs) || value.retry.delayMs < 100 || value.retry.delayMs > 60_000) {
errors.push(`${path}.retry.delayMs_invalid`);
}
} else if (value.retry.delayMs !== undefined) {
errors.push(`${path}.retry.delayMs_requires_fixed_delay`);
}
}
}
@@ -672,13 +683,29 @@ function validateMappingContract(value, path, errors) {
errors.push(`${path}.fact.geometry.point_shape_invalid`);
}
} else if (value.fact.geometry.type === "GeoJSON") {
if (value.fact.geometry.strategy !== "gelios_geozone_v1") errors.push(`${path}.fact.geometry.strategy_invalid`);
requiredSourcePathArray(value.fact.geometry.paths, `${path}.fact.geometry.paths`, errors);
if (value.fact.geometry.strategy !== "bounded_zone_geometry_v1") errors.push(`${path}.fact.geometry.strategy_invalid`);
for (const key of ["kindPath", "polygonPointsPath", "corridorPointsPath", "radiusPath"]) {
if (!isCanonicalSourcePath(value.fact.geometry[key])) {
errors.push(`${path}.fact.geometry.${key}_invalid`);
}
}
if (!["latitude_longitude", "longitude_latitude"].includes(value.fact.geometry.coordinateOrder)) {
errors.push(`${path}.fact.geometry.coordinateOrder_invalid`);
}
if (!Number.isInteger(value.fact.geometry.circleSegments)
|| value.fact.geometry.circleSegments < 16
|| value.fact.geometry.circleSegments > 256) {
errors.push(`${path}.fact.geometry.circleSegments_invalid`);
}
if (!sameSet(value.fact.geometry.allowedTypes, ["Polygon", "MultiPolygon"])) {
errors.push(`${path}.fact.geometry.allowedTypes_must_be_polygon_or_multipolygon`);
}
if (value.fact.geometry.omitIfInvalid !== false) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_false`);
if (value.fact.geometry.longitude !== undefined || value.fact.geometry.latitude !== undefined) {
if (
value.fact.geometry.longitude !== undefined
|| value.fact.geometry.latitude !== undefined
|| value.fact.geometry.paths !== undefined
) {
errors.push(`${path}.fact.geometry.geojson_shape_invalid`);
}
} else {
@@ -900,6 +927,9 @@ function validateExpressionAgainstFieldContract(expression, contract, derivation
errors.push(`${path}_required_but_mapping_can_omit_invalid`);
}
if (expression.paths !== undefined) {
if (contract.required === false && expression.omitIfMissing !== true) {
errors.push(`${path}_optional_source_field_must_omit_missing`);
}
const expectedCoerce = new Map([["string", "string"], ["number", "number"], ["boolean", "boolean"]]).get(contract.type);
if (expectedCoerce && expression.coerce !== expectedCoerce) errors.push(`${path}.coerce_must_match_field_contract`);
if (Array.isArray(contract.enum)) errors.push(`${path}.enum_field_must_use_constant_or_derivation`);
@@ -1041,8 +1071,11 @@ function mappingSourcePaths(value, found = []) {
if (isCanonicalSourcePath(path)) found.push(path);
});
}
for (const key of ["kindPath", "polygonPointsPath", "corridorPointsPath", "radiusPath"]) {
if (isCanonicalSourcePath(value[key])) found.push(value[key]);
}
Object.entries(value).forEach(([key, item]) => {
if (key !== "paths") mappingSourcePaths(item, found);
if (key !== "paths" && !key.endsWith("Path")) mappingSourcePaths(item, found);
});
return found;
}