feat(platform): add managed data product history plane

packages/external-provider-contract/src/data-plane.mjs

@@ -1,9 +1,11 @@
 export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
 export { validateIntakeBatch } from "./intake-batch.mjs";
 export {
+  DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
   DATA_PRODUCT_PATCH_SCHEMA_VERSION,
   DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
   DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
+  validateDataProductHistory,
   validateDataProductPatch,
   validateDataProductPublish,
   validateDataProductSnapshot,

packages/external-provider-contract/src/data-product.mjs

@@ -1,6 +1,7 @@
 export const DATA_PRODUCT_PUBLISH_SCHEMA_VERSION = "nodedc.data-product.publish/v1";
 export const DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION = "nodedc.data-product.snapshot/v1";
 export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1";
+export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1";
 
 import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs";
 
@@ -71,6 +72,67 @@ export function validateDataProductSnapshot(value) {
   return result(errors);
 }
 
+export function validateDataProductHistory(value) {
+  const errors = envelopeErrors(value, DATA_PRODUCT_HISTORY_SCHEMA_VERSION, "history");
+  rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "query", "facts", "nextCursor"]), "history", errors);
+  requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors);
+  if (!isPlainObject(value?.query)) {
+    errors.push("query_must_be_object");
+  } else {
+    rejectUnknownKeys(value.query, new Set(["from", "to", "resolutionMs", "sourceIds", "order"]), "query", errors);
+    requiredIsoTimestamp(value.query.from, "query.from", errors);
+    requiredIsoTimestamp(value.query.to, "query.to", errors);
+    if (
+      !Number.isNaN(Date.parse(value.query.from))
+      && !Number.isNaN(Date.parse(value.query.to))
+      && Date.parse(value.query.from) >= Date.parse(value.query.to)
+    ) errors.push("query.range_invalid");
+    if (!Number.isInteger(value.query.resolutionMs) || value.query.resolutionMs < 1000) {
+      errors.push("query.resolutionMs_invalid");
+    }
+    if (!Array.isArray(value.query.sourceIds)) {
+      errors.push("query.sourceIds_must_be_array");
+    } else {
+      value.query.sourceIds.forEach((sourceId, index) => requiredIdentifier(sourceId, `query.sourceIds[${index}]`, errors));
+      if (JSON.stringify(value.query.sourceIds) !== JSON.stringify([...new Set(value.query.sourceIds)].sort())) {
+        errors.push("query.sourceIds_must_be_unique_and_sorted");
+      }
+    }
+    if (value.query.order !== "asc") errors.push("query.order_must_be_asc");
+  }
+  if (!Array.isArray(value?.facts)) {
+    errors.push("facts_must_be_array");
+  } else {
+    let previousOrderKey = null;
+    value.facts.forEach((fact, index) => {
+      validateCanonicalFact(fact, `facts[${index}]`, errors, { allowBucketStart: true });
+      if (isPlainObject(fact)) {
+        requiredIsoTimestamp(fact.bucketStart, `facts[${index}].bucketStart`, errors);
+        const bucketTime = Date.parse(fact.bucketStart);
+        if (
+          !Number.isNaN(bucketTime)
+          && isPlainObject(value?.query)
+          && (
+            bucketTime < Date.parse(value.query.from)
+            || bucketTime >= Date.parse(value.query.to)
+          )
+        ) errors.push(`facts[${index}].bucketStart_outside_query_range`);
+        const orderKey = `${fact.bucketStart}\u0000${fact.sourceId}\u0000${fact.semanticType}`;
+        if (previousOrderKey !== null && orderKey <= previousOrderKey) {
+          errors.push(`facts[${index}]_not_strictly_ordered`);
+        }
+        previousOrderKey = orderKey;
+      }
+    });
+  }
+  if (value?.nextCursor !== undefined) {
+    requiredString(value.nextCursor, "nextCursor", errors);
+    if (!/^[A-Za-z0-9_-]{1,1024}$/.test(String(value.nextCursor || ""))) errors.push("nextCursor_invalid");
+  }
+  if (containsSecretLikeMaterial(value)) errors.push("history_must_not_contain_secret_material");
+  return result(errors);
+}
+
 export function validateDataProductPatch(value) {
   const errors = envelopeErrors(value, DATA_PRODUCT_PATCH_SCHEMA_VERSION, "patch");
   rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "cursor", "previousCursor", "emittedAt", "operations"]), "patch", errors);
@@ -108,13 +170,14 @@ function envelopeErrors(value, schemaVersion, label) {
   return errors;
 }
 
-function validateFact(value, path, errors, { maxAttributesBytes, canonical = false }) {
+function validateFact(value, path, errors, { maxAttributesBytes, canonical = false, allowBucketStart = false }) {
   if (!isPlainObject(value)) {
     errors.push(`${path}_must_be_object`);
     return;
   }
   const allowedKeys = new Set(["sourceId", "semanticType", "observedAt", "attributes", "geometry"]);
   if (canonical) allowedKeys.add("receivedAt");
+  if (allowBucketStart) allowedKeys.add("bucketStart");
   rejectUnknownKeys(value, allowedKeys, path, errors);
   requiredIdentifier(value.sourceId, `${path}.sourceId`, errors);
   requiredIdentifier(value.semanticType, `${path}.semanticType`, errors);
@@ -129,8 +192,8 @@ function validateFact(value, path, errors, { maxAttributesBytes, canonical = fal
   if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors);
 }
 
-function validateCanonicalFact(value, path, errors) {
-  validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true });
+function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) {
+  validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true, allowBucketStart });
   if (!isPlainObject(value)) return;
   requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors);
 }

packages/external-provider-contract/src/index.mjs

@@ -19,9 +19,11 @@ export {
 } from "./provider-package.mjs";
 
 export {
+  DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
   DATA_PRODUCT_PATCH_SCHEMA_VERSION,
   DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
   DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
+  validateDataProductHistory,
   validateDataProductPatch,
   validateDataProductPublish,
   validateDataProductSnapshot,