feat(platform): add managed data product history plane
This commit is contained in:
@@ -91,6 +91,8 @@ Provider-neutral runtime использует отдельные wire schemas:
|
||||
- `nodedc.data-product.publish/v1` — unscoped publish request от
|
||||
`NDC Data Product Publish`; содержит только batch identity и canonical facts;
|
||||
- `nodedc.data-product.snapshot/v1` — согласованный current snapshot с cursor;
|
||||
- `nodedc.data-product.history/v1` — bounded Timescale history window с
|
||||
`from/to`, provider-neutral `sourceIds`, resolution и opaque keyset cursor;
|
||||
- `nodedc.data-product.patch/v1` — committed upsert operations из durable
|
||||
outbox с `previousCursor`/`cursor`.
|
||||
|
||||
@@ -109,9 +111,11 @@ binding; shared internal bearer и caller-provided scope headers являютс
|
||||
stream с усечённой базой. `nextPageCursor` зарезервирован для будущего отдельно
|
||||
версионируемого query contract и текущим bounded runtime не выдаётся.
|
||||
|
||||
Для большей cardinality definition заранее раскладывается по стабильным
|
||||
partition Data Products с независимыми snapshot/patch cursors либо использует
|
||||
будущий query contract с единым snapshot barrier и continuation semantics.
|
||||
Для большей current cardinality definition заранее раскладывается по стабильным
|
||||
partition Data Products с независимыми snapshot/patch cursors. History является
|
||||
отдельным immutable-window query: cursor привязан digest-ом к exact
|
||||
`from/to/resolution/sourceIds`, поэтому его нельзя переиспользовать с другим
|
||||
запросом; offset pagination запрещена.
|
||||
Offset/source-ID pagination поверх меняющегося current snapshot запрещена:
|
||||
между страницами она способна потерять или задублировать изменения относительно
|
||||
patch cursor.
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
|
||||
export { validateIntakeBatch } from "./intake-batch.mjs";
|
||||
export {
|
||||
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||
validateDataProductHistory,
|
||||
validateDataProductPatch,
|
||||
validateDataProductPublish,
|
||||
validateDataProductSnapshot,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
export const DATA_PRODUCT_PUBLISH_SCHEMA_VERSION = "nodedc.data-product.publish/v1";
|
||||
export const DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION = "nodedc.data-product.snapshot/v1";
|
||||
export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1";
|
||||
export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1";
|
||||
|
||||
import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs";
|
||||
|
||||
@@ -71,6 +72,67 @@ export function validateDataProductSnapshot(value) {
|
||||
return result(errors);
|
||||
}
|
||||
|
||||
export function validateDataProductHistory(value) {
|
||||
const errors = envelopeErrors(value, DATA_PRODUCT_HISTORY_SCHEMA_VERSION, "history");
|
||||
rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "query", "facts", "nextCursor"]), "history", errors);
|
||||
requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors);
|
||||
if (!isPlainObject(value?.query)) {
|
||||
errors.push("query_must_be_object");
|
||||
} else {
|
||||
rejectUnknownKeys(value.query, new Set(["from", "to", "resolutionMs", "sourceIds", "order"]), "query", errors);
|
||||
requiredIsoTimestamp(value.query.from, "query.from", errors);
|
||||
requiredIsoTimestamp(value.query.to, "query.to", errors);
|
||||
if (
|
||||
!Number.isNaN(Date.parse(value.query.from))
|
||||
&& !Number.isNaN(Date.parse(value.query.to))
|
||||
&& Date.parse(value.query.from) >= Date.parse(value.query.to)
|
||||
) errors.push("query.range_invalid");
|
||||
if (!Number.isInteger(value.query.resolutionMs) || value.query.resolutionMs < 1000) {
|
||||
errors.push("query.resolutionMs_invalid");
|
||||
}
|
||||
if (!Array.isArray(value.query.sourceIds)) {
|
||||
errors.push("query.sourceIds_must_be_array");
|
||||
} else {
|
||||
value.query.sourceIds.forEach((sourceId, index) => requiredIdentifier(sourceId, `query.sourceIds[${index}]`, errors));
|
||||
if (JSON.stringify(value.query.sourceIds) !== JSON.stringify([...new Set(value.query.sourceIds)].sort())) {
|
||||
errors.push("query.sourceIds_must_be_unique_and_sorted");
|
||||
}
|
||||
}
|
||||
if (value.query.order !== "asc") errors.push("query.order_must_be_asc");
|
||||
}
|
||||
if (!Array.isArray(value?.facts)) {
|
||||
errors.push("facts_must_be_array");
|
||||
} else {
|
||||
let previousOrderKey = null;
|
||||
value.facts.forEach((fact, index) => {
|
||||
validateCanonicalFact(fact, `facts[${index}]`, errors, { allowBucketStart: true });
|
||||
if (isPlainObject(fact)) {
|
||||
requiredIsoTimestamp(fact.bucketStart, `facts[${index}].bucketStart`, errors);
|
||||
const bucketTime = Date.parse(fact.bucketStart);
|
||||
if (
|
||||
!Number.isNaN(bucketTime)
|
||||
&& isPlainObject(value?.query)
|
||||
&& (
|
||||
bucketTime < Date.parse(value.query.from)
|
||||
|| bucketTime >= Date.parse(value.query.to)
|
||||
)
|
||||
) errors.push(`facts[${index}].bucketStart_outside_query_range`);
|
||||
const orderKey = `${fact.bucketStart}\u0000${fact.sourceId}\u0000${fact.semanticType}`;
|
||||
if (previousOrderKey !== null && orderKey <= previousOrderKey) {
|
||||
errors.push(`facts[${index}]_not_strictly_ordered`);
|
||||
}
|
||||
previousOrderKey = orderKey;
|
||||
}
|
||||
});
|
||||
}
|
||||
if (value?.nextCursor !== undefined) {
|
||||
requiredString(value.nextCursor, "nextCursor", errors);
|
||||
if (!/^[A-Za-z0-9_-]{1,1024}$/.test(String(value.nextCursor || ""))) errors.push("nextCursor_invalid");
|
||||
}
|
||||
if (containsSecretLikeMaterial(value)) errors.push("history_must_not_contain_secret_material");
|
||||
return result(errors);
|
||||
}
|
||||
|
||||
export function validateDataProductPatch(value) {
|
||||
const errors = envelopeErrors(value, DATA_PRODUCT_PATCH_SCHEMA_VERSION, "patch");
|
||||
rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "cursor", "previousCursor", "emittedAt", "operations"]), "patch", errors);
|
||||
@@ -108,13 +170,14 @@ function envelopeErrors(value, schemaVersion, label) {
|
||||
return errors;
|
||||
}
|
||||
|
||||
function validateFact(value, path, errors, { maxAttributesBytes, canonical = false }) {
|
||||
function validateFact(value, path, errors, { maxAttributesBytes, canonical = false, allowBucketStart = false }) {
|
||||
if (!isPlainObject(value)) {
|
||||
errors.push(`${path}_must_be_object`);
|
||||
return;
|
||||
}
|
||||
const allowedKeys = new Set(["sourceId", "semanticType", "observedAt", "attributes", "geometry"]);
|
||||
if (canonical) allowedKeys.add("receivedAt");
|
||||
if (allowBucketStart) allowedKeys.add("bucketStart");
|
||||
rejectUnknownKeys(value, allowedKeys, path, errors);
|
||||
requiredIdentifier(value.sourceId, `${path}.sourceId`, errors);
|
||||
requiredIdentifier(value.semanticType, `${path}.semanticType`, errors);
|
||||
@@ -129,8 +192,8 @@ function validateFact(value, path, errors, { maxAttributesBytes, canonical = fal
|
||||
if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors);
|
||||
}
|
||||
|
||||
function validateCanonicalFact(value, path, errors) {
|
||||
validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true });
|
||||
function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) {
|
||||
validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true, allowBucketStart });
|
||||
if (!isPlainObject(value)) return;
|
||||
requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors);
|
||||
}
|
||||
|
||||
@@ -19,9 +19,11 @@ export {
|
||||
} from "./provider-package.mjs";
|
||||
|
||||
export {
|
||||
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||
validateDataProductHistory,
|
||||
validateDataProductPatch,
|
||||
validateDataProductPublish,
|
||||
validateDataProductSnapshot,
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||
validateDataProductHistory,
|
||||
validateDataProductPatch,
|
||||
validateDataProductPublish,
|
||||
validateDataProductSnapshot,
|
||||
@@ -89,6 +91,45 @@ assert.equal(validateDataProductSnapshot({
|
||||
...snapshot,
|
||||
facts: [{ ...canonicalFact, attributes: { status: "ndc_edprb_forbidden-reader-token" } }],
|
||||
}).errors.includes("snapshot_must_not_contain_secret_material"), true);
|
||||
|
||||
const history = {
|
||||
schemaVersion: DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
||||
dataProduct: { id: "fleet.positions.current.v1", version: "1.0.0" },
|
||||
generatedAt: "2026-07-15T10:05:00.000Z",
|
||||
query: {
|
||||
from: "2026-07-15T10:00:00.000Z",
|
||||
to: "2026-07-15T10:05:00.000Z",
|
||||
resolutionMs: 60_000,
|
||||
sourceIds: ["unit-42"],
|
||||
order: "asc",
|
||||
},
|
||||
facts: [{ ...canonicalFact, bucketStart: "2026-07-15T10:00:00.000Z" }],
|
||||
nextCursor: "opaque_cursor",
|
||||
};
|
||||
assert.equal(validateDataProductHistory(history).ok, true);
|
||||
assert.equal(validateDataProductHistory({
|
||||
...history,
|
||||
query: { ...history.query, order: "desc" },
|
||||
}).errors.includes("query.order_must_be_asc"), true);
|
||||
assert.equal(validateDataProductHistory({
|
||||
...history,
|
||||
query: { ...history.query, from: history.query.to, to: history.query.from },
|
||||
}).errors.includes("query.range_invalid"), true);
|
||||
assert.equal(validateDataProductHistory({
|
||||
...history,
|
||||
query: { ...history.query, sourceIds: ["unit-42", "unit-42"] },
|
||||
}).errors.includes("query.sourceIds_must_be_unique_and_sorted"), true);
|
||||
assert.equal(validateDataProductHistory({
|
||||
...history,
|
||||
facts: [
|
||||
{ ...history.facts[0], sourceId: "unit-43" },
|
||||
history.facts[0],
|
||||
],
|
||||
}).errors.includes("facts[1]_not_strictly_ordered"), true);
|
||||
assert.equal(validateDataProductHistory({
|
||||
...history,
|
||||
facts: [{ ...history.facts[0], attributes: { token: "forbidden" } }],
|
||||
}).errors.includes("history_must_not_contain_secret_material"), true);
|
||||
assert.equal(validateDataProductSnapshot({
|
||||
...snapshot,
|
||||
facts: [{ ...canonicalFact, attributes: { status: `ndc_edppr_${"P".repeat(43)}` } }],
|
||||
|
||||
Reference in New Issue
Block a user