From a5a5644dbfb063ecaca61a5d3c4589aa736eb14a Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 23 Jul 2026 19:27:01 +0300 Subject: [PATCH] feat(provider-contract): compile governed L2 execution plans --- packages/external-provider-contract/README.md | 19 + .../external-provider-contract/package.json | 2 +- .../providers/gelios/v8/index.mjs | 1 + .../gelios/v8/telemetry-field-registry.mjs | 75 ++++ .../external-provider-contract/src/index.mjs | 15 + .../src/l2-execution-plan.mjs | 333 ++++++++++++++++++ .../src/telemetry-field-registry.mjs | 271 ++++++++++++++ .../test/l2-execution-plan.test.mjs | 121 +++++++ .../test/telemetry-field-registry.test.mjs | 80 +++++ 9 files changed, 916 insertions(+), 1 deletion(-) create mode 100644 packages/external-provider-contract/providers/gelios/v8/telemetry-field-registry.mjs create mode 100644 packages/external-provider-contract/src/l2-execution-plan.mjs create mode 100644 packages/external-provider-contract/src/telemetry-field-registry.mjs create mode 100644 packages/external-provider-contract/test/l2-execution-plan.test.mjs create mode 100644 packages/external-provider-contract/test/telemetry-field-registry.test.mjs diff --git a/packages/external-provider-contract/README.md b/packages/external-provider-contract/README.md index 9d92f50..6ead110 100644 --- a/packages/external-provider-contract/README.md +++ b/packages/external-provider-contract/README.md @@ -48,6 +48,25 @@ read-capability передаёт все entities, которые provider воз boundary запрещены; сортировка и видимость принадлежат Data Product consumer и Foundry. +`compileL2ExecutionPlan` превращает package + connection instance в +детерминированный `nodedc.l2-execution-plan/v1`: разрешает generic шаги +collection/request/extract/mapping/publish, прикладывает полные декларативные +request/mapping contracts и фиксирует SHA-256 digests package, profile, +template, mapping, field policy, Data Product и telemetry registry. В +компиляторе нет веток по provider ID. Provider-specific URL, source paths и +параметры являются данными immutable package. После материализации Engine +`attestL2ExecutionPlanMaterialization` связывает exact plan digest с exact +graph revision/digest и точным списком materialized steps. + +Динамическая телеметрия проходит отдельный +`nodedc.telemetry-field-registry/v1`. Реестр хранит только имена source +parameters, типы, статус классификации, sensitivity и разрешённые surfaces — +никогда значения. Только `approved + operational` entries могут войти в +Data Product/analytics/Foundry projection. `observed`, PII/identifier, +command и restricted entries остаются audit-only; wildcard source key +запрещён. Поэтому появление нового параметра у provider не меняет ядро и не +открывает его автоматически потребителям. + Текущий production-shaped package — `providers/gelios/v5`. Он сохраняет два официальных Gelios REST safe-read: `GET /api/v1/users/me/monitoring-config` и `GET /api/v1/units?incltrip=true`, а также точный immutable output diff --git a/packages/external-provider-contract/package.json b/packages/external-provider-contract/package.json index 2e6a07b..613baaf 100644 --- a/packages/external-provider-contract/package.json +++ b/packages/external-provider-contract/package.json @@ -9,6 +9,6 @@ "./providers/*": "./providers/*/index.mjs" }, "scripts": { - "check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/provider-capability-catalog.test.mjs && node test/zone-source.test.mjs && node test/engine-private-extension.test.mjs" + "check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/provider-capability-catalog.test.mjs && node test/telemetry-field-registry.test.mjs && node test/l2-execution-plan.test.mjs && node test/zone-source.test.mjs && node test/engine-private-extension.test.mjs" } } diff --git a/packages/external-provider-contract/providers/gelios/v8/index.mjs b/packages/external-provider-contract/providers/gelios/v8/index.mjs index b2ad6e2..51320c1 100644 --- a/packages/external-provider-contract/providers/gelios/v8/index.mjs +++ b/packages/external-provider-contract/providers/gelios/v8/index.mjs @@ -6,3 +6,4 @@ export { GELIOS_UNIT_PROFILE_ONTOLOGY_REVISION, geliosProviderPackageV8, } from "./package.mjs"; +export { geliosTelemetryFieldRegistryV1 } from "./telemetry-field-registry.mjs"; diff --git a/packages/external-provider-contract/providers/gelios/v8/telemetry-field-registry.mjs b/packages/external-provider-contract/providers/gelios/v8/telemetry-field-registry.mjs new file mode 100644 index 0000000..ee29178 --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v8/telemetry-field-registry.mjs @@ -0,0 +1,75 @@ +import { + TELEMETRY_FIELD_REGISTRY_SCHEMA_VERSION, +} from "../../../src/telemetry-field-registry.mjs"; +import { + GELIOS_PROVIDER_PACKAGE_ID, + GELIOS_PROVIDER_PACKAGE_VERSION, +} from "./package.mjs"; +import { + GELIOS_POSITIONS_DATA_PRODUCT_ID, +} from "../v7/package.mjs"; + +const value = { + schemaVersion: TELEMETRY_FIELD_REGISTRY_SCHEMA_VERSION, + id: "gelios.units.current.telemetry.v1", + version: "1.0.0", + providerPackage: { + id: GELIOS_PROVIDER_PACKAGE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + }, + dataProductId: GELIOS_POSITIONS_DATA_PRODUCT_ID, + sourceNamespace: "gelios.units.configured_sensor_parameter", + entries: [ + { + id: "gelios.sensor-parameter.in0", + sourceKey: "in0", + status: "approved", + sensitivity: "operational", + valueType: "number", + semanticReading: { + id: "sensor.param.in0", + labelSource: "provider_configured", + unitSource: "provider_configured", + }, + allowedSurfaces: ["audit", "data_product", "analytics", "foundry"], + }, + { + id: "gelios.sensor-parameter.in1", + sourceKey: "in1", + status: "approved", + sensitivity: "operational", + valueType: "number", + semanticReading: { + id: "sensor.param.in1", + labelSource: "provider_configured", + unitSource: "provider_configured", + }, + allowedSurfaces: ["audit", "data_product", "analytics", "foundry"], + }, + { + id: "gelios.sensor-parameter.in-underscore-0", + sourceKey: "in_0", + status: "observed", + sensitivity: "unclassified", + valueType: "unknown", + allowedSurfaces: ["audit"], + }, + { + id: "gelios.sensor-parameter.in-underscore-1", + sourceKey: "in_1", + status: "observed", + sensitivity: "unclassified", + valueType: "unknown", + allowedSurfaces: ["audit"], + }, + ], +}; + +export const geliosTelemetryFieldRegistryV1 = deepFreeze(value); + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/src/index.mjs b/packages/external-provider-contract/src/index.mjs index 5a45300..c2c725f 100644 --- a/packages/external-provider-contract/src/index.mjs +++ b/packages/external-provider-contract/src/index.mjs @@ -14,6 +14,21 @@ export { TELEMETRY_READINGS_MAX_ITEMS, isBoundedTelemetryReadings, } from "./telemetry-readings.mjs"; +export { + TELEMETRY_FIELD_PROJECTION_SCHEMA_VERSION, + TELEMETRY_FIELD_REGISTRY_SCHEMA_VERSION, + compileTelemetryFieldProjection, + digestTelemetryFieldRegistry, + validateTelemetryFieldRegistry, +} from "./telemetry-field-registry.mjs"; +export { + L2_EXECUTION_PLAN_COMPILER_VERSION, + L2_EXECUTION_PLAN_SCHEMA_VERSION, + L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION, + attestL2ExecutionPlanMaterialization, + compileL2ExecutionPlan, + validateL2ExecutionPlan, +} from "./l2-execution-plan.mjs"; export { ZONE_SOURCE_ADAPTERS, ZONE_SOURCE_GENERATION_SCHEMA_VERSION, diff --git a/packages/external-provider-contract/src/l2-execution-plan.mjs b/packages/external-provider-contract/src/l2-execution-plan.mjs new file mode 100644 index 0000000..e340512 --- /dev/null +++ b/packages/external-provider-contract/src/l2-execution-plan.mjs @@ -0,0 +1,333 @@ +import { createHash } from "node:crypto"; +import { validateProviderPackage } from "./provider-package.mjs"; +import { + compileTelemetryFieldProjection, + validateTelemetryFieldRegistry, +} from "./telemetry-field-registry.mjs"; + +export const L2_EXECUTION_PLAN_SCHEMA_VERSION = "nodedc.l2-execution-plan/v1"; +export const L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION = "nodedc.l2-materialization-receipt/v1"; +export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.0.0"; + +const HASH = /^(?:sha256:)?[a-f0-9]{64}$/; +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const CREDENTIAL_REF = /^ndc-credref:[A-Za-z0-9._:-]{8,255}$/; +const COMPILE_OPTION_KEYS = new Set(["telemetryFieldRegistry"]); +const RECEIPT_OPTION_KEYS = new Set(["graphRevision", "graphDigest", "materializedStepIds"]); + +/** + * Resolve one immutable provider package plus one connection instance into a + * deterministic, provider-neutral execution plan. Provider-specific knowledge + * stays in package/registry data; compiler branching is only by generic step + * kind. + */ +export function compileL2ExecutionPlan(providerPackage, connectionInstance, options = {}) { + const packageValidation = validateProviderPackage(providerPackage); + if (!packageValidation.ok) { + throw new Error(`provider_package_invalid:${packageValidation.errors.join(",")}`); + } + validateCompileOptions(options); + validateConnectionInstance(providerPackage, connectionInstance); + + const profile = requireById( + providerPackage.collectionProfiles, + connectionInstance.collectionProfileId, + "collection_profile", + ); + const template = requireById(providerPackage.l2Templates, profile.l2TemplateId, "l2_template"); + if (template.id !== connectionInstance.l2TemplateId) { + throw new Error("l2_execution_plan_connection_template_mismatch"); + } + const mapping = requireById(providerPackage.mappingContracts, profile.mappingContractId, "mapping_contract"); + const fieldPolicy = requireById(providerPackage.fieldPolicies, profile.fieldPolicyId, "field_policy"); + const dataProduct = requireById(providerPackage.dataProducts, profile.dataProductId, "data_product"); + const requiresTelemetryRegistry = Object.values(dataProduct.fieldContracts || {}) + .some((contract) => contract?.type === "telemetry_readings"); + + let telemetryProjection; + if (options.telemetryFieldRegistry !== undefined) { + const registryValidation = validateTelemetryFieldRegistry(options.telemetryFieldRegistry); + if (!registryValidation.ok) { + throw new Error(`telemetry_field_registry_invalid:${registryValidation.errors.join(",")}`); + } + telemetryProjection = compileTelemetryFieldProjection(options.telemetryFieldRegistry, { + providerPackageId: providerPackage.id, + providerPackageVersion: providerPackage.version, + dataProductId: dataProduct.id, + surface: "data_product", + }); + } else if (requiresTelemetryRegistry) { + throw new Error("l2_execution_plan_telemetry_registry_required"); + } + if (requiresTelemetryRegistry && telemetryProjection.entries.length === 0) { + throw new Error("l2_execution_plan_telemetry_projection_empty"); + } + if (!requiresTelemetryRegistry && telemetryProjection !== undefined) { + throw new Error("l2_execution_plan_telemetry_registry_not_applicable"); + } + + const artifacts = { + packageDigest: canonicalDigest(providerPackage), + collectionProfileDigest: canonicalDigest(profile), + l2TemplateDigest: canonicalDigest(template), + mappingContractDigest: canonicalDigest(mapping), + fieldPolicyDigest: canonicalDigest(fieldPolicy), + dataProductDigest: canonicalDigest(dataProduct), + ...(telemetryProjection ? { telemetryRegistryDigest: telemetryProjection.registry.digest } : {}), + }; + + const steps = template.steps.map((step) => compileStep({ + step, + providerPackage, + profile, + mapping, + fieldPolicy, + dataProduct, + telemetryProjection, + artifacts, + })); + + const planWithoutDigest = { + schemaVersion: L2_EXECUTION_PLAN_SCHEMA_VERSION, + compilerVersion: L2_EXECUTION_PLAN_COMPILER_VERSION, + package: { + id: providerPackage.id, + providerId: providerPackage.providerId, + version: providerPackage.version, + }, + connection: { + tenantId: connectionInstance.tenantId, + connectionId: connectionInstance.connectionId, + collectionProfileId: profile.id, + l2TemplateId: template.id, + }, + bindings: { + provider: structuredClone(connectionInstance.credentialRefs.provider), + publisher: structuredClone(connectionInstance.systemBindings.publisher), + }, + collectionProfile: structuredClone(profile), + artifacts, + steps, + }; + + return deepFreeze({ + ...planWithoutDigest, + executionPlanDigest: canonicalDigest(planWithoutDigest), + }); +} + +export function validateL2ExecutionPlan(value) { + const errors = []; + if (!isPlainObject(value)) return result(["executionPlan_must_be_object"]); + if (value.schemaVersion !== L2_EXECUTION_PLAN_SCHEMA_VERSION) errors.push("executionPlan.schemaVersion_mismatch"); + if (value.compilerVersion !== L2_EXECUTION_PLAN_COMPILER_VERSION) errors.push("executionPlan.compilerVersion_mismatch"); + if (!Array.isArray(value.steps) || value.steps.length < 5) { + errors.push("executionPlan.steps_invalid"); + } else { + const ids = value.steps.map((step) => step?.id); + if (ids.some((id) => typeof id !== "string" || !IDENTIFIER.test(id))) errors.push("executionPlan.steps_id_invalid"); + if (new Set(ids).size !== ids.length) errors.push("executionPlan.steps_id_must_be_unique"); + } + if (!HASH.test(String(value.executionPlanDigest || ""))) { + errors.push("executionPlan.executionPlanDigest_invalid"); + } else { + const descriptor = structuredClone(value); + delete descriptor.executionPlanDigest; + if (normalizeDigest(value.executionPlanDigest) !== canonicalDigest(descriptor)) { + errors.push("executionPlan.executionPlanDigest_mismatch"); + } + } + return result(errors); +} + +/** + * Bind a successful Engine graph materialization back to the exact compiled + * plan without teaching this package how Engine stores or renders its graph. + */ +export function attestL2ExecutionPlanMaterialization(executionPlan, options) { + const validation = validateL2ExecutionPlan(executionPlan); + if (!validation.ok) { + throw new Error(`l2_execution_plan_invalid:${validation.errors.join(",")}`); + } + if (!isPlainObject(options)) throw new Error("l2_materialization_options_must_be_object"); + const errors = []; + rejectUnknownKeys(options, RECEIPT_OPTION_KEYS, "options", errors); + if (typeof options.graphRevision !== "string" || !options.graphRevision.trim() || options.graphRevision.length > 160) { + errors.push("options.graphRevision_invalid"); + } + if (!HASH.test(String(options.graphDigest || ""))) errors.push("options.graphDigest_invalid"); + const expectedStepIds = executionPlan.steps.map((step) => step.id); + if (!Array.isArray(options.materializedStepIds) + || options.materializedStepIds.length !== expectedStepIds.length + || options.materializedStepIds.some((id, index) => id !== expectedStepIds[index])) { + errors.push("options.materializedStepIds_must_exactly_match_execution_plan"); + } + if (errors.length) { + throw new Error(`l2_materialization_invalid:${[...new Set(errors)].join(",")}`); + } + + const receiptWithoutDigest = { + schemaVersion: L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION, + compilerVersion: executionPlan.compilerVersion, + executionPlanDigest: executionPlan.executionPlanDigest, + graphRevision: options.graphRevision, + graphDigest: normalizeDigest(options.graphDigest), + materializedStepIds: [...options.materializedStepIds], + }; + return deepFreeze({ + ...receiptWithoutDigest, + receiptDigest: canonicalDigest(receiptWithoutDigest), + }); +} + +function compileStep(context) { + const { step, providerPackage, profile, mapping, fieldPolicy, dataProduct, telemetryProjection, artifacts } = context; + const base = { id: step.id, kind: step.kind }; + if (step.kind === "collection_trigger") { + return { + ...base, + config: { + mode: profile.mode, + ...(profile.schedule ? { schedule: structuredClone(profile.schedule) } : {}), + }, + }; + } + if (step.kind === "provider_request") { + const capability = requireById(providerPackage.capabilities, step.capabilityId, "capability"); + return { + ...base, + config: { + capabilityId: capability.id, + capabilityDigest: canonicalDigest(capability), + authModeId: capability.authModeId, + credentialBinding: "provider", + request: structuredClone(capability.request), + }, + }; + } + if (step.kind === "extract_items") { + const capability = requireById(providerPackage.capabilities, step.capabilityId, "capability"); + return { + ...base, + config: { + capabilityId: capability.id, + capabilityDigest: canonicalDigest(capability), + response: structuredClone(capability.request.response), + }, + }; + } + if (step.kind === "semantic_mapping") { + return { + ...base, + config: { + mappingContractId: mapping.id, + mappingContractDigest: artifacts.mappingContractDigest, + fieldPolicyId: fieldPolicy.id, + fieldPolicyDigest: artifacts.fieldPolicyDigest, + dataProductId: dataProduct.id, + dataProductDigest: artifacts.dataProductDigest, + mappingContract: structuredClone(mapping), + ...(telemetryProjection ? { telemetryProjection: structuredClone(telemetryProjection) } : {}), + }, + }; + } + if (step.kind === "data_product_publish") { + return { + ...base, + config: { + dataProductId: dataProduct.id, + dataProductDigest: artifacts.dataProductDigest, + nodeType: step.nodeType, + credentialBinding: "publisher", + }, + }; + } + throw new Error(`l2_execution_plan_step_kind_unsupported:${String(step.kind)}`); +} + +function validateConnectionInstance(providerPackage, value) { + if (!isPlainObject(value)) throw new Error("l2_execution_plan_connection_must_be_object"); + if (value.schemaVersion !== "nodedc.l2-connection-instance/v1") { + throw new Error("l2_execution_plan_connection_schema_mismatch"); + } + if (value.package?.id !== providerPackage.id + || value.package?.providerId !== providerPackage.providerId + || value.package?.version !== providerPackage.version) { + throw new Error("l2_execution_plan_connection_package_mismatch"); + } + if (typeof value.tenantId !== "string" || !IDENTIFIER.test(value.tenantId)) { + throw new Error("l2_execution_plan_tenant_id_invalid"); + } + if (typeof value.connectionId !== "string" || !IDENTIFIER.test(value.connectionId)) { + throw new Error("l2_execution_plan_connection_id_invalid"); + } + if (!CREDENTIAL_REF.test(String(value.credentialRefs?.provider?.reference || "")) + || value.credentialRefs?.provider?.owner !== "ndc_l2_credentials") { + throw new Error("l2_execution_plan_provider_credential_ref_invalid"); + } + const publisher = value.systemBindings?.publisher; + if (publisher?.owner !== "ndc_l2_credentials" + || publisher?.management !== "control_plane_managed" + || publisher?.desiredState !== "bound" + || publisher?.status !== "unresolved") { + throw new Error("l2_execution_plan_publisher_binding_invalid"); + } +} + +function validateCompileOptions(options) { + if (!isPlainObject(options)) throw new Error("l2_execution_plan_options_must_be_object"); + const errors = []; + rejectUnknownKeys(options, COMPILE_OPTION_KEYS, "options", errors); + if (errors.length) throw new Error(`l2_execution_plan_options_invalid:${errors.join(",")}`); +} + +function requireById(items, id, type) { + const value = Array.isArray(items) ? items.find((item) => item?.id === id) : undefined; + if (!value) throw new Error(`l2_execution_plan_${type}_missing:${String(id)}`); + return value; +} + +function canonicalDigest(value) { + return `sha256:${createHash("sha256").update(stableJson(value), "utf8").digest("hex")}`; +} + +function normalizeDigest(value) { + const text = String(value); + return text.startsWith("sha256:") ? text : `sha256:${text}`; +} + +function stableJson(value) { + return JSON.stringify(stableValue(value)); +} + +function stableValue(value) { + if (Array.isArray(value)) return value.map(stableValue); + if (!isPlainObject(value)) return value; + return Object.fromEntries( + Object.keys(value) + .filter((key) => value[key] !== undefined) + .sort() + .map((key) => [key, stableValue(value[key])]), + ); +} + +function rejectUnknownKeys(value, allowed, path, errors) { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`); + } +} + +function result(errors) { + return { ok: errors.length === 0, errors: [...new Set(errors)] }; +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/src/telemetry-field-registry.mjs b/packages/external-provider-contract/src/telemetry-field-registry.mjs new file mode 100644 index 0000000..a0981b3 --- /dev/null +++ b/packages/external-provider-contract/src/telemetry-field-registry.mjs @@ -0,0 +1,271 @@ +import { createHash } from "node:crypto"; + +export const TELEMETRY_FIELD_REGISTRY_SCHEMA_VERSION = "nodedc.telemetry-field-registry/v1"; +export const TELEMETRY_FIELD_PROJECTION_SCHEMA_VERSION = "nodedc.telemetry-field-projection/v1"; + +const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; +const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; +const SOURCE_KEY = /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/; +const VALUE_TYPES = new Set(["unknown", "number", "boolean", "string"]); +const STATUSES = new Set(["observed", "classified", "approved", "restricted", "deprecated"]); +const SENSITIVITIES = new Set(["operational", "unclassified", "identifier", "personal", "command", "restricted"]); +const SURFACES = new Set(["audit", "data_product", "analytics", "foundry"]); +const REGISTRY_KEYS = new Set([ + "schemaVersion", + "id", + "version", + "providerPackage", + "dataProductId", + "sourceNamespace", + "entries", +]); +const PACKAGE_KEYS = new Set(["id", "version"]); +const ENTRY_KEYS = new Set([ + "id", + "sourceKey", + "status", + "sensitivity", + "valueType", + "semanticReading", + "allowedSurfaces", +]); +const READING_KEYS = new Set(["id", "labelSource", "label", "unitSource", "unit"]); +const LABEL_SOURCES = new Set(["registry", "provider_configured"]); +const UNIT_SOURCES = new Set(["none", "registry", "provider_configured"]); +const PROJECTION_OPTION_KEYS = new Set([ + "providerPackageId", + "providerPackageVersion", + "dataProductId", + "surface", +]); + +/** + * Validate a values-free, provider-scoped registry that classifies dynamic + * telemetry parameters before any Data Product or interface can expose them. + */ +export function validateTelemetryFieldRegistry(value) { + const errors = []; + if (!isPlainObject(value)) return result(["registry_must_be_object"]); + rejectUnknownKeys(value, REGISTRY_KEYS, "registry", errors); + if (value.schemaVersion !== TELEMETRY_FIELD_REGISTRY_SCHEMA_VERSION) { + errors.push("registry.schemaVersion_mismatch"); + } + requiredIdentifier(value.id, "registry.id", errors); + requiredSemver(value.version, "registry.version", errors); + requiredIdentifier(value.dataProductId, "registry.dataProductId", errors); + requiredIdentifier(value.sourceNamespace, "registry.sourceNamespace", errors); + + if (!isPlainObject(value.providerPackage)) { + errors.push("registry.providerPackage_must_be_object"); + } else { + rejectUnknownKeys(value.providerPackage, PACKAGE_KEYS, "registry.providerPackage", errors); + requiredIdentifier(value.providerPackage.id, "registry.providerPackage.id", errors); + requiredSemver(value.providerPackage.version, "registry.providerPackage.version", errors); + } + + if (!Array.isArray(value.entries) || value.entries.length === 0 || value.entries.length > 512) { + errors.push("registry.entries_must_be_nonempty_bounded_array"); + } else { + const entryIds = new Set(); + const sourceKeys = new Set(); + const approvedReadingIds = new Set(); + value.entries.forEach((entry, index) => { + const path = `registry.entries[${index}]`; + if (!isPlainObject(entry)) { + errors.push(`${path}_must_be_object`); + return; + } + rejectUnknownKeys(entry, ENTRY_KEYS, path, errors); + requiredIdentifier(entry.id, `${path}.id`, errors); + if (entryIds.has(entry.id)) errors.push(`${path}.id_must_be_unique`); + entryIds.add(entry.id); + if (typeof entry.sourceKey !== "string" || !SOURCE_KEY.test(entry.sourceKey)) { + errors.push(`${path}.sourceKey_invalid`); + } + if (sourceKeys.has(entry.sourceKey)) errors.push(`${path}.sourceKey_must_be_unique`); + sourceKeys.add(entry.sourceKey); + if (!STATUSES.has(entry.status)) errors.push(`${path}.status_invalid`); + if (!SENSITIVITIES.has(entry.sensitivity)) errors.push(`${path}.sensitivity_invalid`); + if (!VALUE_TYPES.has(entry.valueType)) errors.push(`${path}.valueType_invalid`); + validateSurfaces(entry.allowedSurfaces, path, errors); + validateSemanticReading(entry.semanticReading, path, errors); + + const surfaces = new Set(Array.isArray(entry.allowedSurfaces) ? entry.allowedSurfaces : []); + const externallyExposed = [...surfaces].some((surface) => surface !== "audit"); + if (entry.status === "approved") { + if (!isPlainObject(entry.semanticReading)) errors.push(`${path}.approved_requires_semantic_reading`); + if (entry.sensitivity !== "operational") errors.push(`${path}.approved_must_be_operational`); + if (entry.valueType === "unknown") errors.push(`${path}.approved_valueType_must_be_classified`); + if (!surfaces.has("data_product")) errors.push(`${path}.approved_requires_data_product_surface`); + const readingId = entry.semanticReading?.id; + if (typeof readingId === "string") { + if (approvedReadingIds.has(readingId)) errors.push(`${path}.semanticReading.id_must_be_unique`); + approvedReadingIds.add(readingId); + } + } else if (externallyExposed) { + errors.push(`${path}.nonapproved_surface_must_be_audit_only`); + } + if (new Set(["observed", "restricted"]).has(entry.status) && entry.semanticReading !== undefined) { + errors.push(`${path}.${entry.status}_must_not_claim_semantic_reading`); + } + if (entry.status === "observed" && entry.sensitivity !== "unclassified") { + errors.push(`${path}.observed_must_be_unclassified`); + } + if (entry.status === "restricted" && !new Set(["identifier", "personal", "command", "restricted"]).has(entry.sensitivity)) { + errors.push(`${path}.restricted_sensitivity_invalid`); + } + }); + } + + return result(errors); +} + +/** + * Compile an exact, values-free allowlist for one consumer surface. Unknown, + * observed, restricted and deprecated provider parameters are omitted. + */ +export function compileTelemetryFieldProjection(registry, options) { + const validation = validateTelemetryFieldRegistry(registry); + if (!validation.ok) { + throw new Error(`telemetry_field_registry_invalid:${validation.errors.join(",")}`); + } + if (!isPlainObject(options)) throw new Error("telemetry_projection_options_must_be_object"); + const optionErrors = []; + rejectUnknownKeys(options, PROJECTION_OPTION_KEYS, "options", optionErrors); + requiredIdentifier(options.providerPackageId, "options.providerPackageId", optionErrors); + requiredSemver(options.providerPackageVersion, "options.providerPackageVersion", optionErrors); + requiredIdentifier(options.dataProductId, "options.dataProductId", optionErrors); + if (!SURFACES.has(options.surface)) optionErrors.push("options.surface_invalid"); + if (options.providerPackageId !== registry.providerPackage.id) optionErrors.push("options.providerPackageId_mismatch"); + if (options.providerPackageVersion !== registry.providerPackage.version) optionErrors.push("options.providerPackageVersion_mismatch"); + if (options.dataProductId !== registry.dataProductId) optionErrors.push("options.dataProductId_mismatch"); + if (optionErrors.length) { + throw new Error(`telemetry_projection_invalid:${[...new Set(optionErrors)].join(",")}`); + } + + const entries = registry.entries + .filter((entry) => entry.status === "approved" && entry.allowedSurfaces.includes(options.surface)) + .map((entry) => ({ + registryEntryId: entry.id, + sourceNamespace: registry.sourceNamespace, + sourceKey: entry.sourceKey, + valueType: entry.valueType, + reading: structuredClone(entry.semanticReading), + })) + .sort((left, right) => left.sourceKey.localeCompare(right.sourceKey)); + + return deepFreeze({ + schemaVersion: TELEMETRY_FIELD_PROJECTION_SCHEMA_VERSION, + registry: { + id: registry.id, + version: registry.version, + digest: digestTelemetryFieldRegistry(registry), + }, + providerPackage: structuredClone(registry.providerPackage), + dataProductId: registry.dataProductId, + surface: options.surface, + entries, + }); +} + +export function digestTelemetryFieldRegistry(registry) { + const validation = validateTelemetryFieldRegistry(registry); + if (!validation.ok) { + throw new Error(`telemetry_field_registry_invalid:${validation.errors.join(",")}`); + } + return canonicalDigest(registry); +} + +function validateSurfaces(value, path, errors) { + if (!Array.isArray(value) || value.length === 0 || value.length > SURFACES.size) { + errors.push(`${path}.allowedSurfaces_must_be_nonempty_bounded_array`); + return; + } + if (new Set(value).size !== value.length) errors.push(`${path}.allowedSurfaces_must_be_unique`); + value.forEach((surface) => { + if (!SURFACES.has(surface)) errors.push(`${path}.allowedSurfaces_contains_invalid_surface`); + }); +} + +function validateSemanticReading(value, path, errors) { + if (value === undefined) return; + if (!isPlainObject(value)) { + errors.push(`${path}.semanticReading_must_be_object`); + return; + } + rejectUnknownKeys(value, READING_KEYS, `${path}.semanticReading`, errors); + requiredIdentifier(value.id, `${path}.semanticReading.id`, errors); + if (!LABEL_SOURCES.has(value.labelSource)) { + errors.push(`${path}.semanticReading.labelSource_invalid`); + } + if (value.labelSource === "registry" + && (typeof value.label !== "string" || !value.label.trim() || value.label.length > 160)) { + errors.push(`${path}.semanticReading.label_invalid`); + } + if (value.labelSource === "provider_configured" + && value.label !== undefined + && (typeof value.label !== "string" || !value.label.trim() || value.label.length > 160)) { + errors.push(`${path}.semanticReading.label_invalid`); + } + if (!UNIT_SOURCES.has(value.unitSource)) { + errors.push(`${path}.semanticReading.unitSource_invalid`); + } + if (value.unitSource === "registry" + && (typeof value.unit !== "string" || !value.unit.trim() || value.unit.length > 32)) { + errors.push(`${path}.semanticReading.unit_invalid`); + } + if (value.unitSource === "none" && value.unit !== undefined) { + errors.push(`${path}.semanticReading.unit_not_allowed`); + } + if (value.unit !== undefined && (typeof value.unit !== "string" || value.unit.length > 32)) { + errors.push(`${path}.semanticReading.unit_invalid`); + } +} + +function canonicalDigest(value) { + return `sha256:${createHash("sha256").update(stableJson(value), "utf8").digest("hex")}`; +} + +function stableJson(value) { + return JSON.stringify(stableValue(value)); +} + +function stableValue(value) { + if (Array.isArray(value)) return value.map(stableValue); + if (!isPlainObject(value)) return value; + return Object.fromEntries( + Object.keys(value) + .filter((key) => value[key] !== undefined) + .sort() + .map((key) => [key, stableValue(value[key])]), + ); +} + +function rejectUnknownKeys(value, allowed, path, errors) { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`); + } +} + +function requiredIdentifier(value, path, errors) { + if (typeof value !== "string" || !IDENTIFIER.test(value)) errors.push(`${path}_invalid`); +} + +function requiredSemver(value, path, errors) { + if (typeof value !== "string" || !SEMVER.test(value)) errors.push(`${path}_invalid`); +} + +function result(errors) { + return { ok: errors.length === 0, errors: [...new Set(errors)] }; +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/test/l2-execution-plan.test.mjs b/packages/external-provider-contract/test/l2-execution-plan.test.mjs new file mode 100644 index 0000000..73eff0a --- /dev/null +++ b/packages/external-provider-contract/test/l2-execution-plan.test.mjs @@ -0,0 +1,121 @@ +import assert from "node:assert/strict"; +import { + L2_EXECUTION_PLAN_SCHEMA_VERSION, + L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION, + attestL2ExecutionPlanMaterialization, + compileL2ExecutionPlan, + instantiateL2Connection, + validateL2ExecutionPlan, +} from "../src/index.mjs"; +import { + geliosProviderPackageV8, + geliosTelemetryFieldRegistryV1, +} from "../providers/gelios/v8/index.mjs"; + +const positionsConnection = instantiateL2Connection(geliosProviderPackageV8, { + tenantId: "tenant-compiler-fixture", + connectionId: "gelios-compiler-fixture", + collectionProfileId: "gelios.positions.current.realtime.v7", + providerCredentialRef: "ndc-credref:provider-compiler-fixture-0001", +}); +const positionsPlan = compileL2ExecutionPlan( + geliosProviderPackageV8, + positionsConnection, + { telemetryFieldRegistry: geliosTelemetryFieldRegistryV1 }, +); + +assert.equal(positionsPlan.schemaVersion, L2_EXECUTION_PLAN_SCHEMA_VERSION); +assert.deepEqual(validateL2ExecutionPlan(positionsPlan), { ok: true, errors: [] }); +assert.match(positionsPlan.executionPlanDigest, /^sha256:[a-f0-9]{64}$/); +assert.match(positionsPlan.artifacts.packageDigest, /^sha256:[a-f0-9]{64}$/); +assert.match(positionsPlan.artifacts.mappingContractDigest, /^sha256:[a-f0-9]{64}$/); +assert.match(positionsPlan.artifacts.telemetryRegistryDigest, /^sha256:[a-f0-9]{64}$/); +assert.deepEqual(positionsPlan.steps.map((step) => step.kind), [ + "collection_trigger", + "provider_request", + "provider_request", + "extract_items", + "semantic_mapping", + "data_product_publish", +]); +const unitsRequest = positionsPlan.steps.find( + (step) => step.kind === "provider_request" && step.config.capabilityId === "gelios.units.current.read", +); +assert.deepEqual(unitsRequest.config.request.query, { + incltrip: "true", + inclcntrs: "true", + inclsnsrs: "true", + incllsv: "true", +}); +const semanticMapping = positionsPlan.steps.find((step) => step.kind === "semantic_mapping"); +assert.deepEqual( + semanticMapping.config.telemetryProjection.entries.map((entry) => entry.reading.id), + ["sensor.param.in0", "sensor.param.in1"], +); +assert.equal(Object.isFrozen(positionsPlan), true); +assert.equal(Object.isFrozen(positionsPlan.steps), true); + +const clonedPlan = compileL2ExecutionPlan( + structuredClone(geliosProviderPackageV8), + structuredClone(positionsConnection), + { telemetryFieldRegistry: structuredClone(geliosTelemetryFieldRegistryV1) }, +); +assert.equal(clonedPlan.executionPlanDigest, positionsPlan.executionPlanDigest); + +assert.throws( + () => compileL2ExecutionPlan(geliosProviderPackageV8, positionsConnection), + /telemetry_registry_required/, +); + +const profileConnection = instantiateL2Connection(geliosProviderPackageV8, { + tenantId: "tenant-compiler-fixture", + connectionId: "gelios-profile-compiler-fixture", + collectionProfileId: "gelios.units.profile.cold.v1", + providerCredentialRef: "ndc-credref:provider-compiler-fixture-0001", +}); +const profilePlan = compileL2ExecutionPlan(geliosProviderPackageV8, profileConnection); +assert.deepEqual(validateL2ExecutionPlan(profilePlan), { ok: true, errors: [] }); +assert.equal(profilePlan.artifacts.telemetryRegistryDigest, undefined); +assert.notEqual(profilePlan.executionPlanDigest, positionsPlan.executionPlanDigest); + +const secondConnection = instantiateL2Connection(geliosProviderPackageV8, { + tenantId: "tenant-compiler-fixture", + connectionId: "gelios-compiler-fixture-two", + collectionProfileId: "gelios.positions.current.realtime.v7", + providerCredentialRef: "ndc-credref:provider-compiler-fixture-0002", +}); +const secondPlan = compileL2ExecutionPlan( + geliosProviderPackageV8, + secondConnection, + { telemetryFieldRegistry: geliosTelemetryFieldRegistryV1 }, +); +assert.notEqual(secondPlan.executionPlanDigest, positionsPlan.executionPlanDigest); + +const receipt = attestL2ExecutionPlanMaterialization(positionsPlan, { + graphRevision: "engine-revision-fixture-0001", + graphDigest: "a".repeat(64), + materializedStepIds: positionsPlan.steps.map((step) => step.id), +}); +assert.equal(receipt.schemaVersion, L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION); +assert.equal(receipt.executionPlanDigest, positionsPlan.executionPlanDigest); +assert.equal(receipt.graphDigest, `sha256:${"a".repeat(64)}`); +assert.match(receipt.receiptDigest, /^sha256:[a-f0-9]{64}$/); +assert.equal(Object.isFrozen(receipt), true); + +assert.throws(() => attestL2ExecutionPlanMaterialization(positionsPlan, { + graphRevision: "engine-revision-fixture-0001", + graphDigest: "a".repeat(64), + materializedStepIds: positionsPlan.steps.slice(1).map((step) => step.id), +}), /materializedStepIds_must_exactly_match/); + +const tamperedPlan = structuredClone(positionsPlan); +tamperedPlan.steps[0].config.mode = "history"; +assert.equal( + validateL2ExecutionPlan(tamperedPlan).errors.includes("executionPlan.executionPlanDigest_mismatch"), + true, +); + +const serialized = JSON.stringify(positionsPlan); +assert.equal(serialized.includes("Bearer "), false); +assert.equal(serialized.includes("access_token"), false); +assert.equal(serialized.includes("refresh_token"), false); diff --git a/packages/external-provider-contract/test/telemetry-field-registry.test.mjs b/packages/external-provider-contract/test/telemetry-field-registry.test.mjs new file mode 100644 index 0000000..593762d --- /dev/null +++ b/packages/external-provider-contract/test/telemetry-field-registry.test.mjs @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { + TELEMETRY_FIELD_PROJECTION_SCHEMA_VERSION, + compileTelemetryFieldProjection, + digestTelemetryFieldRegistry, + validateTelemetryFieldRegistry, +} from "../src/index.mjs"; +import { + geliosTelemetryFieldRegistryV1, +} from "../providers/gelios/v8/index.mjs"; + +assert.deepEqual(validateTelemetryFieldRegistry(geliosTelemetryFieldRegistryV1), { + ok: true, + errors: [], +}); +assert.match(digestTelemetryFieldRegistry(geliosTelemetryFieldRegistryV1), /^sha256:[a-f0-9]{64}$/); + +const projection = compileTelemetryFieldProjection(geliosTelemetryFieldRegistryV1, { + providerPackageId: "gelios.provider.v8", + providerPackageVersion: "8.0.0", + dataProductId: "fleet.positions.current.v5", + surface: "data_product", +}); +assert.equal(projection.schemaVersion, TELEMETRY_FIELD_PROJECTION_SCHEMA_VERSION); +assert.deepEqual(projection.entries.map((entry) => entry.sourceKey), ["in0", "in1"]); +assert.deepEqual(projection.entries.map((entry) => entry.reading.id), [ + "sensor.param.in0", + "sensor.param.in1", +]); +assert.equal(projection.entries.some((entry) => entry.sourceKey === "in_0"), false); +assert.equal(Object.isFrozen(projection), true); +assert.equal(Object.isFrozen(projection.entries), true); + +const observedExposed = structuredClone(geliosTelemetryFieldRegistryV1); +observedExposed.entries.find((entry) => entry.sourceKey === "in_0").allowedSurfaces.push("foundry"); +assert.equal( + validateTelemetryFieldRegistry(observedExposed).errors.includes( + "registry.entries[2].nonapproved_surface_must_be_audit_only", + ), + true, +); + +const approvedUnclassified = structuredClone(geliosTelemetryFieldRegistryV1); +approvedUnclassified.entries[0].sensitivity = "unclassified"; +assert.equal( + validateTelemetryFieldRegistry(approvedUnclassified).errors.includes( + "registry.entries[0].approved_must_be_operational", + ), + true, +); + +const wildcardSource = structuredClone(geliosTelemetryFieldRegistryV1); +wildcardSource.entries[0].sourceKey = "*"; +assert.equal( + validateTelemetryFieldRegistry(wildcardSource).errors.includes( + "registry.entries[0].sourceKey_invalid", + ), + true, +); + +const duplicateReading = structuredClone(geliosTelemetryFieldRegistryV1); +duplicateReading.entries[1].semanticReading.id = duplicateReading.entries[0].semanticReading.id; +assert.equal( + validateTelemetryFieldRegistry(duplicateReading).errors.includes( + "registry.entries[1].semanticReading.id_must_be_unique", + ), + true, +); + +assert.throws(() => compileTelemetryFieldProjection(geliosTelemetryFieldRegistryV1, { + providerPackageId: "another.provider.v1", + providerPackageVersion: "8.0.0", + dataProductId: "fleet.positions.current.v5", + surface: "data_product", +}), /providerPackageId_mismatch/); + +const serialized = JSON.stringify(geliosTelemetryFieldRegistryV1); +for (const forbidden of ["phone", "imei", "commands", "access_token", "refresh_token"]) { + assert.equal(serialized.toLowerCase().includes(forbidden), false); +}