264 lines
12 KiB
JavaScript
264 lines
12 KiB
JavaScript
export const DATA_PRODUCT_PUBLISH_SCHEMA_VERSION = "nodedc.data-product.publish/v1";
|
|
export const DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION = "nodedc.data-product.snapshot/v1";
|
|
export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1";
|
|
export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1";
|
|
|
|
import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs";
|
|
import { validateGeoJsonGeometry } from "./geometry.mjs";
|
|
|
|
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
|
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
|
const CURSOR = /^(?:0|[1-9]\d*)$/;
|
|
const MAX_BATCH_SEQUENCE = 2_147_483_647;
|
|
const MAX_FACT_ATTRIBUTES_BYTES = 64 * 1024;
|
|
|
|
/**
|
|
* Wire form accepted from an NDC Data Product Publish node.
|
|
*
|
|
* Scope, provider identity, product version, ontology revision and storage
|
|
* policy are deliberately absent: the Data Plane materializes them from the
|
|
* opaque writer grant and its product registry.
|
|
*/
|
|
export function validateDataProductPublish(value, { maxFacts = 5000, maxAttributesBytes = 64 * 1024 } = {}) {
|
|
const errors = [];
|
|
const attributesCeiling = Number.isInteger(maxAttributesBytes) && maxAttributesBytes > 0
|
|
? Math.min(maxAttributesBytes, MAX_FACT_ATTRIBUTES_BYTES)
|
|
: MAX_FACT_ATTRIBUTES_BYTES;
|
|
if (!isPlainObject(value)) return result(["publish_must_be_object"]);
|
|
if (value.schemaVersion !== DATA_PRODUCT_PUBLISH_SCHEMA_VERSION) errors.push("schemaVersion_mismatch");
|
|
rejectUnknownKeys(value, new Set(["schemaVersion", "batch", "facts"]), "publish", errors);
|
|
|
|
if (!isPlainObject(value.batch)) {
|
|
errors.push("batch_must_be_object");
|
|
} else {
|
|
rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey", "mode", "generationAt"]), "batch", errors);
|
|
requiredIdentifier(value.batch.runId, "batch.runId", errors);
|
|
requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors);
|
|
if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) {
|
|
errors.push("batch.sequence_must_be_integer_0_to_2147483647");
|
|
}
|
|
const mode = value.batch.mode === undefined ? "upsert" : value.batch.mode;
|
|
if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid");
|
|
if (mode === "replace") requiredIsoTimestamp(value.batch.generationAt, "batch.generationAt", errors);
|
|
if (mode === "upsert" && value.batch.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert");
|
|
}
|
|
|
|
if (!Array.isArray(value.facts)) {
|
|
errors.push("facts_must_be_array");
|
|
} else if (value.facts.length === 0 && value.batch?.mode !== "replace") {
|
|
errors.push("facts_must_be_nonempty_array_for_upsert");
|
|
} else if (value.facts.length > maxFacts) {
|
|
errors.push("facts_limit_exceeded");
|
|
} else {
|
|
const entityKeys = new Set();
|
|
value.facts.forEach((fact, index) => {
|
|
validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling });
|
|
if (value.batch?.mode === "replace"
|
|
&& !Number.isNaN(Date.parse(fact?.observedAt))
|
|
&& Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) {
|
|
errors.push(`facts[${index}].observedAt_must_equal_generationAt`);
|
|
}
|
|
if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return;
|
|
const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`;
|
|
if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key");
|
|
entityKeys.add(entityKey);
|
|
});
|
|
}
|
|
|
|
if (containsSecretLikeMaterial(value)) errors.push("publish_must_not_contain_secret_material");
|
|
return result(errors);
|
|
}
|
|
|
|
export function validateDataProductSnapshot(value) {
|
|
const errors = envelopeErrors(value, DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, "snapshot");
|
|
rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "cursor", "facts", "nextPageCursor"]), "snapshot", errors);
|
|
requiredCursor(value?.cursor, "cursor", errors);
|
|
requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors);
|
|
if (!Array.isArray(value?.facts)) {
|
|
errors.push("facts_must_be_array");
|
|
} else {
|
|
value.facts.forEach((fact, index) => validateCanonicalFact(fact, `facts[${index}]`, errors));
|
|
}
|
|
if (value?.nextPageCursor !== undefined) requiredString(value.nextPageCursor, "nextPageCursor", errors);
|
|
if (containsSecretLikeMaterial(value)) errors.push("snapshot_must_not_contain_secret_material");
|
|
return result(errors);
|
|
}
|
|
|
|
export function validateDataProductHistory(value) {
|
|
const errors = envelopeErrors(value, DATA_PRODUCT_HISTORY_SCHEMA_VERSION, "history");
|
|
rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "query", "facts", "nextCursor"]), "history", errors);
|
|
requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors);
|
|
if (!isPlainObject(value?.query)) {
|
|
errors.push("query_must_be_object");
|
|
} else {
|
|
rejectUnknownKeys(value.query, new Set(["from", "to", "resolutionMs", "sourceIds", "order"]), "query", errors);
|
|
requiredIsoTimestamp(value.query.from, "query.from", errors);
|
|
requiredIsoTimestamp(value.query.to, "query.to", errors);
|
|
if (
|
|
!Number.isNaN(Date.parse(value.query.from))
|
|
&& !Number.isNaN(Date.parse(value.query.to))
|
|
&& Date.parse(value.query.from) >= Date.parse(value.query.to)
|
|
) errors.push("query.range_invalid");
|
|
if (!Number.isInteger(value.query.resolutionMs) || value.query.resolutionMs < 1000) {
|
|
errors.push("query.resolutionMs_invalid");
|
|
}
|
|
if (!Array.isArray(value.query.sourceIds)) {
|
|
errors.push("query.sourceIds_must_be_array");
|
|
} else {
|
|
value.query.sourceIds.forEach((sourceId, index) => requiredIdentifier(sourceId, `query.sourceIds[${index}]`, errors));
|
|
if (JSON.stringify(value.query.sourceIds) !== JSON.stringify([...new Set(value.query.sourceIds)].sort())) {
|
|
errors.push("query.sourceIds_must_be_unique_and_sorted");
|
|
}
|
|
}
|
|
if (value.query.order !== "asc") errors.push("query.order_must_be_asc");
|
|
}
|
|
if (!Array.isArray(value?.facts)) {
|
|
errors.push("facts_must_be_array");
|
|
} else {
|
|
let previousOrderKey = null;
|
|
value.facts.forEach((fact, index) => {
|
|
validateCanonicalFact(fact, `facts[${index}]`, errors, { allowBucketStart: true });
|
|
if (isPlainObject(fact)) {
|
|
requiredIsoTimestamp(fact.bucketStart, `facts[${index}].bucketStart`, errors);
|
|
const bucketTime = Date.parse(fact.bucketStart);
|
|
if (
|
|
!Number.isNaN(bucketTime)
|
|
&& isPlainObject(value?.query)
|
|
&& (
|
|
bucketTime < Date.parse(value.query.from)
|
|
|| bucketTime >= Date.parse(value.query.to)
|
|
)
|
|
) errors.push(`facts[${index}].bucketStart_outside_query_range`);
|
|
const orderKey = `${fact.bucketStart}\u0000${fact.sourceId}\u0000${fact.semanticType}`;
|
|
if (previousOrderKey !== null && orderKey <= previousOrderKey) {
|
|
errors.push(`facts[${index}]_not_strictly_ordered`);
|
|
}
|
|
previousOrderKey = orderKey;
|
|
}
|
|
});
|
|
}
|
|
if (value?.nextCursor !== undefined) {
|
|
requiredString(value.nextCursor, "nextCursor", errors);
|
|
if (!/^[A-Za-z0-9_-]{1,1024}$/.test(String(value.nextCursor || ""))) errors.push("nextCursor_invalid");
|
|
}
|
|
if (containsSecretLikeMaterial(value)) errors.push("history_must_not_contain_secret_material");
|
|
return result(errors);
|
|
}
|
|
|
|
export function validateDataProductPatch(value) {
|
|
const errors = envelopeErrors(value, DATA_PRODUCT_PATCH_SCHEMA_VERSION, "patch");
|
|
rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "cursor", "previousCursor", "emittedAt", "operations"]), "patch", errors);
|
|
requiredCursor(value?.cursor, "cursor", errors);
|
|
requiredCursor(value?.previousCursor, "previousCursor", errors);
|
|
requiredIsoTimestamp(value?.emittedAt, "emittedAt", errors);
|
|
if (!Array.isArray(value?.operations) || value.operations.length === 0) {
|
|
errors.push("operations_must_be_nonempty_array");
|
|
} else {
|
|
value.operations.forEach((operation, index) => {
|
|
if (!isPlainObject(operation) || !new Set(["upsert", "remove"]).has(operation.op)) {
|
|
errors.push(`operations[${index}].op_invalid`);
|
|
return;
|
|
}
|
|
if (operation.op === "upsert") {
|
|
rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors);
|
|
validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors);
|
|
} else {
|
|
rejectUnknownKeys(operation, new Set(["op", "sourceId", "semanticType"]), `operations[${index}]`, errors);
|
|
requiredIdentifier(operation.sourceId, `operations[${index}].sourceId`, errors);
|
|
requiredIdentifier(operation.semanticType, `operations[${index}].semanticType`, errors);
|
|
}
|
|
});
|
|
}
|
|
if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material");
|
|
return result(errors);
|
|
}
|
|
|
|
function envelopeErrors(value, schemaVersion, label) {
|
|
const errors = [];
|
|
if (!isPlainObject(value)) return [`${label}_must_be_object`];
|
|
if (value.schemaVersion !== schemaVersion) errors.push("schemaVersion_mismatch");
|
|
if (!isPlainObject(value.dataProduct)) {
|
|
errors.push("dataProduct_must_be_object");
|
|
} else {
|
|
rejectUnknownKeys(value.dataProduct, new Set(["id", "version"]), "dataProduct", errors);
|
|
requiredIdentifier(value.dataProduct.id, "dataProduct.id", errors);
|
|
requiredString(value.dataProduct.version, "dataProduct.version", errors);
|
|
if (value.dataProduct.version && !SEMVER.test(value.dataProduct.version)) errors.push("dataProduct.version_must_be_semver");
|
|
}
|
|
return errors;
|
|
}
|
|
|
|
function validateFact(value, path, errors, { maxAttributesBytes, canonical = false, allowBucketStart = false }) {
|
|
if (!isPlainObject(value)) {
|
|
errors.push(`${path}_must_be_object`);
|
|
return;
|
|
}
|
|
const allowedKeys = new Set(["sourceId", "semanticType", "observedAt", "attributes", "geometry"]);
|
|
if (canonical) allowedKeys.add("receivedAt");
|
|
if (allowBucketStart) allowedKeys.add("bucketStart");
|
|
rejectUnknownKeys(value, allowedKeys, path, errors);
|
|
requiredIdentifier(value.sourceId, `${path}.sourceId`, errors);
|
|
requiredIdentifier(value.semanticType, `${path}.semanticType`, errors);
|
|
requiredIsoTimestamp(value.observedAt, `${path}.observedAt`, errors);
|
|
if (value.attributes !== undefined) {
|
|
if (!isPlainObject(value.attributes)) {
|
|
errors.push(`${path}.attributes_must_be_object`);
|
|
} else if (serializedByteLength(value.attributes) > maxAttributesBytes) {
|
|
errors.push(`${path}.attributes_size_exceeded`);
|
|
}
|
|
}
|
|
if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors);
|
|
}
|
|
|
|
function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) {
|
|
validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true, allowBucketStart });
|
|
if (!isPlainObject(value)) return;
|
|
requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors);
|
|
}
|
|
|
|
function rejectUnknownKeys(value, allowed, path, errors) {
|
|
if (!isPlainObject(value)) return;
|
|
for (const key of Object.keys(value)) {
|
|
if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`);
|
|
}
|
|
}
|
|
|
|
function requiredIdentifier(value, path, errors) {
|
|
if (typeof value !== "string" || !IDENTIFIER.test(value)) errors.push(`${path}_invalid`);
|
|
}
|
|
|
|
function requiredString(value, path, errors) {
|
|
if (typeof value !== "string" || !value.trim()) errors.push(`${path}_required`);
|
|
}
|
|
|
|
function requiredCursor(value, path, errors) {
|
|
if (typeof value !== "string" || !CURSOR.test(value)) errors.push(`${path}_invalid`);
|
|
}
|
|
|
|
function requiredIsoTimestamp(value, path, errors) {
|
|
if (typeof value !== "string" || Number.isNaN(Date.parse(value))) errors.push(`${path}_invalid_timestamp`);
|
|
}
|
|
|
|
function isPlainObject(value) {
|
|
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function serializedByteLength(value) {
|
|
try {
|
|
return Buffer.byteLength(JSON.stringify(value));
|
|
} catch {
|
|
return Number.POSITIVE_INFINITY;
|
|
}
|
|
}
|
|
|
|
function containsSecretLikeMaterial(value) {
|
|
if (typeof value === "string") return SECRET_LIKE_VALUE.test(value);
|
|
if (Array.isArray(value)) return value.some(containsSecretLikeMaterial);
|
|
if (!isPlainObject(value)) return false;
|
|
return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeMaterial(child));
|
|
}
|
|
|
|
function result(errors) {
|
|
return Object.freeze({ ok: errors.length === 0, errors: Object.freeze([...new Set(errors)]) });
|
|
}
|