export const DATA_PRODUCT_PUBLISH_SCHEMA_VERSION = "nodedc.data-product.publish/v1"; export const DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION = "nodedc.data-product.snapshot/v1"; export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1"; export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1"; import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs"; import { validateGeoJsonGeometry } from "./geometry.mjs"; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; const CURSOR = /^(?:0|[1-9]\d*)$/; const MAX_BATCH_SEQUENCE = 2_147_483_647; const MAX_FACT_ATTRIBUTES_BYTES = 64 * 1024; /** * Wire form accepted from an NDC Data Product Publish node. * * Scope, provider identity, product version, ontology revision and storage * policy are deliberately absent: the Data Plane materializes them from the * opaque writer grant and its product registry. */ export function validateDataProductPublish(value, { maxFacts = 5000, maxAttributesBytes = 64 * 1024 } = {}) { const errors = []; const attributesCeiling = Number.isInteger(maxAttributesBytes) && maxAttributesBytes > 0 ? Math.min(maxAttributesBytes, MAX_FACT_ATTRIBUTES_BYTES) : MAX_FACT_ATTRIBUTES_BYTES; if (!isPlainObject(value)) return result(["publish_must_be_object"]); if (value.schemaVersion !== DATA_PRODUCT_PUBLISH_SCHEMA_VERSION) errors.push("schemaVersion_mismatch"); rejectUnknownKeys(value, new Set(["schemaVersion", "batch", "facts"]), "publish", errors); if (!isPlainObject(value.batch)) { errors.push("batch_must_be_object"); } else { rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey", "mode", "generationAt"]), "batch", errors); requiredIdentifier(value.batch.runId, "batch.runId", errors); requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors); if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) { errors.push("batch.sequence_must_be_integer_0_to_2147483647"); } const mode = value.batch.mode === undefined ? "upsert" : value.batch.mode; if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid"); if (mode === "replace") requiredIsoTimestamp(value.batch.generationAt, "batch.generationAt", errors); if (mode === "upsert" && value.batch.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert"); } if (!Array.isArray(value.facts)) { errors.push("facts_must_be_array"); } else if (value.facts.length === 0 && value.batch?.mode !== "replace") { errors.push("facts_must_be_nonempty_array_for_upsert"); } else if (value.facts.length > maxFacts) { errors.push("facts_limit_exceeded"); } else { const entityKeys = new Set(); value.facts.forEach((fact, index) => { validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling }); if (value.batch?.mode === "replace" && !Number.isNaN(Date.parse(fact?.observedAt)) && Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) { errors.push(`facts[${index}].observedAt_must_equal_generationAt`); } if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return; const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`; if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key"); entityKeys.add(entityKey); }); } if (containsSecretLikeMaterial(value)) errors.push("publish_must_not_contain_secret_material"); return result(errors); } export function validateDataProductSnapshot(value) { const errors = envelopeErrors(value, DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, "snapshot"); rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "cursor", "facts", "nextPageCursor"]), "snapshot", errors); requiredCursor(value?.cursor, "cursor", errors); requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors); if (!Array.isArray(value?.facts)) { errors.push("facts_must_be_array"); } else { value.facts.forEach((fact, index) => validateCanonicalFact(fact, `facts[${index}]`, errors)); } if (value?.nextPageCursor !== undefined) requiredString(value.nextPageCursor, "nextPageCursor", errors); if (containsSecretLikeMaterial(value)) errors.push("snapshot_must_not_contain_secret_material"); return result(errors); } export function validateDataProductHistory(value) { const errors = envelopeErrors(value, DATA_PRODUCT_HISTORY_SCHEMA_VERSION, "history"); rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "query", "facts", "nextCursor"]), "history", errors); requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors); if (!isPlainObject(value?.query)) { errors.push("query_must_be_object"); } else { rejectUnknownKeys(value.query, new Set(["from", "to", "resolutionMs", "sourceIds", "order"]), "query", errors); requiredIsoTimestamp(value.query.from, "query.from", errors); requiredIsoTimestamp(value.query.to, "query.to", errors); if ( !Number.isNaN(Date.parse(value.query.from)) && !Number.isNaN(Date.parse(value.query.to)) && Date.parse(value.query.from) >= Date.parse(value.query.to) ) errors.push("query.range_invalid"); if (!Number.isInteger(value.query.resolutionMs) || value.query.resolutionMs < 1000) { errors.push("query.resolutionMs_invalid"); } if (!Array.isArray(value.query.sourceIds)) { errors.push("query.sourceIds_must_be_array"); } else { value.query.sourceIds.forEach((sourceId, index) => requiredIdentifier(sourceId, `query.sourceIds[${index}]`, errors)); if (JSON.stringify(value.query.sourceIds) !== JSON.stringify([...new Set(value.query.sourceIds)].sort())) { errors.push("query.sourceIds_must_be_unique_and_sorted"); } } if (value.query.order !== "asc") errors.push("query.order_must_be_asc"); } if (!Array.isArray(value?.facts)) { errors.push("facts_must_be_array"); } else { let previousOrderKey = null; value.facts.forEach((fact, index) => { validateCanonicalFact(fact, `facts[${index}]`, errors, { allowBucketStart: true }); if (isPlainObject(fact)) { requiredIsoTimestamp(fact.bucketStart, `facts[${index}].bucketStart`, errors); const bucketTime = Date.parse(fact.bucketStart); if ( !Number.isNaN(bucketTime) && isPlainObject(value?.query) && ( bucketTime < Date.parse(value.query.from) || bucketTime >= Date.parse(value.query.to) ) ) errors.push(`facts[${index}].bucketStart_outside_query_range`); const orderKey = `${fact.bucketStart}\u0000${fact.sourceId}\u0000${fact.semanticType}`; if (previousOrderKey !== null && orderKey <= previousOrderKey) { errors.push(`facts[${index}]_not_strictly_ordered`); } previousOrderKey = orderKey; } }); } if (value?.nextCursor !== undefined) { requiredString(value.nextCursor, "nextCursor", errors); if (!/^[A-Za-z0-9_-]{1,1024}$/.test(String(value.nextCursor || ""))) errors.push("nextCursor_invalid"); } if (containsSecretLikeMaterial(value)) errors.push("history_must_not_contain_secret_material"); return result(errors); } export function validateDataProductPatch(value) { const errors = envelopeErrors(value, DATA_PRODUCT_PATCH_SCHEMA_VERSION, "patch"); rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "cursor", "previousCursor", "emittedAt", "operations"]), "patch", errors); requiredCursor(value?.cursor, "cursor", errors); requiredCursor(value?.previousCursor, "previousCursor", errors); requiredIsoTimestamp(value?.emittedAt, "emittedAt", errors); if (!Array.isArray(value?.operations) || value.operations.length === 0) { errors.push("operations_must_be_nonempty_array"); } else { value.operations.forEach((operation, index) => { if (!isPlainObject(operation) || !new Set(["upsert", "remove"]).has(operation.op)) { errors.push(`operations[${index}].op_invalid`); return; } if (operation.op === "upsert") { rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors); validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors); } else { rejectUnknownKeys(operation, new Set(["op", "sourceId", "semanticType"]), `operations[${index}]`, errors); requiredIdentifier(operation.sourceId, `operations[${index}].sourceId`, errors); requiredIdentifier(operation.semanticType, `operations[${index}].semanticType`, errors); } }); } if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material"); return result(errors); } function envelopeErrors(value, schemaVersion, label) { const errors = []; if (!isPlainObject(value)) return [`${label}_must_be_object`]; if (value.schemaVersion !== schemaVersion) errors.push("schemaVersion_mismatch"); if (!isPlainObject(value.dataProduct)) { errors.push("dataProduct_must_be_object"); } else { rejectUnknownKeys(value.dataProduct, new Set(["id", "version"]), "dataProduct", errors); requiredIdentifier(value.dataProduct.id, "dataProduct.id", errors); requiredString(value.dataProduct.version, "dataProduct.version", errors); if (value.dataProduct.version && !SEMVER.test(value.dataProduct.version)) errors.push("dataProduct.version_must_be_semver"); } return errors; } function validateFact(value, path, errors, { maxAttributesBytes, canonical = false, allowBucketStart = false }) { if (!isPlainObject(value)) { errors.push(`${path}_must_be_object`); return; } const allowedKeys = new Set(["sourceId", "semanticType", "observedAt", "attributes", "geometry"]); if (canonical) allowedKeys.add("receivedAt"); if (allowBucketStart) allowedKeys.add("bucketStart"); rejectUnknownKeys(value, allowedKeys, path, errors); requiredIdentifier(value.sourceId, `${path}.sourceId`, errors); requiredIdentifier(value.semanticType, `${path}.semanticType`, errors); requiredIsoTimestamp(value.observedAt, `${path}.observedAt`, errors); if (value.attributes !== undefined) { if (!isPlainObject(value.attributes)) { errors.push(`${path}.attributes_must_be_object`); } else if (serializedByteLength(value.attributes) > maxAttributesBytes) { errors.push(`${path}.attributes_size_exceeded`); } } if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors); } function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) { validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true, allowBucketStart }); if (!isPlainObject(value)) return; requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors); } function rejectUnknownKeys(value, allowed, path, errors) { if (!isPlainObject(value)) return; for (const key of Object.keys(value)) { if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`); } } function requiredIdentifier(value, path, errors) { if (typeof value !== "string" || !IDENTIFIER.test(value)) errors.push(`${path}_invalid`); } function requiredString(value, path, errors) { if (typeof value !== "string" || !value.trim()) errors.push(`${path}_required`); } function requiredCursor(value, path, errors) { if (typeof value !== "string" || !CURSOR.test(value)) errors.push(`${path}_invalid`); } function requiredIsoTimestamp(value, path, errors) { if (typeof value !== "string" || Number.isNaN(Date.parse(value))) errors.push(`${path}_invalid_timestamp`); } function isPlainObject(value) { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } function serializedByteLength(value) { try { return Buffer.byteLength(JSON.stringify(value)); } catch { return Number.POSITIVE_INFINITY; } } function containsSecretLikeMaterial(value) { if (typeof value === "string") return SECRET_LIKE_VALUE.test(value); if (Array.isArray(value)) return value.some(containsSecretLikeMaterial); if (!isPlainObject(value)) return false; return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeMaterial(child)); } function result(errors) { return Object.freeze({ ok: errors.length === 0, errors: Object.freeze([...new Set(errors)]) }); }