feat(platform): add managed data product history plane

This commit is contained in:
Codex
2026-07-18 14:38:06 +03:00
parent 02816c4352
commit 3c5d8f6cef
34 changed files with 2091 additions and 163 deletions
@@ -91,6 +91,8 @@ Provider-neutral runtime использует отдельные wire schemas:
- `nodedc.data-product.publish/v1` — unscoped publish request от
`NDC Data Product Publish`; содержит только batch identity и canonical facts;
- `nodedc.data-product.snapshot/v1` — согласованный current snapshot с cursor;
- `nodedc.data-product.history/v1` — bounded Timescale history window с
`from/to`, provider-neutral `sourceIds`, resolution и opaque keyset cursor;
- `nodedc.data-product.patch/v1` — committed upsert operations из durable
outbox с `previousCursor`/`cursor`.
@@ -109,9 +111,11 @@ binding; shared internal bearer и caller-provided scope headers являютс
stream с усечённой базой. `nextPageCursor` зарезервирован для будущего отдельно
версионируемого query contract и текущим bounded runtime не выдаётся.
Для большей cardinality definition заранее раскладывается по стабильным
partition Data Products с независимыми snapshot/patch cursors либо использует
будущий query contract с единым snapshot barrier и continuation semantics.
Для большей current cardinality definition заранее раскладывается по стабильным
partition Data Products с независимыми snapshot/patch cursors. History является
отдельным immutable-window query: cursor привязан digest-ом к exact
`from/to/resolution/sourceIds`, поэтому его нельзя переиспользовать с другим
запросом; offset pagination запрещена.
Offset/source-ID pagination поверх меняющегося current snapshot запрещена:
между страницами она способна потерять или задублировать изменения относительно
patch cursor.
@@ -1,9 +1,11 @@
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
export { validateIntakeBatch } from "./intake-batch.mjs";
export {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
validateDataProductHistory,
validateDataProductPatch,
validateDataProductPublish,
validateDataProductSnapshot,
@@ -1,6 +1,7 @@
export const DATA_PRODUCT_PUBLISH_SCHEMA_VERSION = "nodedc.data-product.publish/v1";
export const DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION = "nodedc.data-product.snapshot/v1";
export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1";
export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1";
import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs";
@@ -71,6 +72,67 @@ export function validateDataProductSnapshot(value) {
return result(errors);
}
export function validateDataProductHistory(value) {
const errors = envelopeErrors(value, DATA_PRODUCT_HISTORY_SCHEMA_VERSION, "history");
rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "generatedAt", "query", "facts", "nextCursor"]), "history", errors);
requiredIsoTimestamp(value?.generatedAt, "generatedAt", errors);
if (!isPlainObject(value?.query)) {
errors.push("query_must_be_object");
} else {
rejectUnknownKeys(value.query, new Set(["from", "to", "resolutionMs", "sourceIds", "order"]), "query", errors);
requiredIsoTimestamp(value.query.from, "query.from", errors);
requiredIsoTimestamp(value.query.to, "query.to", errors);
if (
!Number.isNaN(Date.parse(value.query.from))
&& !Number.isNaN(Date.parse(value.query.to))
&& Date.parse(value.query.from) >= Date.parse(value.query.to)
) errors.push("query.range_invalid");
if (!Number.isInteger(value.query.resolutionMs) || value.query.resolutionMs < 1000) {
errors.push("query.resolutionMs_invalid");
}
if (!Array.isArray(value.query.sourceIds)) {
errors.push("query.sourceIds_must_be_array");
} else {
value.query.sourceIds.forEach((sourceId, index) => requiredIdentifier(sourceId, `query.sourceIds[${index}]`, errors));
if (JSON.stringify(value.query.sourceIds) !== JSON.stringify([...new Set(value.query.sourceIds)].sort())) {
errors.push("query.sourceIds_must_be_unique_and_sorted");
}
}
if (value.query.order !== "asc") errors.push("query.order_must_be_asc");
}
if (!Array.isArray(value?.facts)) {
errors.push("facts_must_be_array");
} else {
let previousOrderKey = null;
value.facts.forEach((fact, index) => {
validateCanonicalFact(fact, `facts[${index}]`, errors, { allowBucketStart: true });
if (isPlainObject(fact)) {
requiredIsoTimestamp(fact.bucketStart, `facts[${index}].bucketStart`, errors);
const bucketTime = Date.parse(fact.bucketStart);
if (
!Number.isNaN(bucketTime)
&& isPlainObject(value?.query)
&& (
bucketTime < Date.parse(value.query.from)
|| bucketTime >= Date.parse(value.query.to)
)
) errors.push(`facts[${index}].bucketStart_outside_query_range`);
const orderKey = `${fact.bucketStart}\u0000${fact.sourceId}\u0000${fact.semanticType}`;
if (previousOrderKey !== null && orderKey <= previousOrderKey) {
errors.push(`facts[${index}]_not_strictly_ordered`);
}
previousOrderKey = orderKey;
}
});
}
if (value?.nextCursor !== undefined) {
requiredString(value.nextCursor, "nextCursor", errors);
if (!/^[A-Za-z0-9_-]{1,1024}$/.test(String(value.nextCursor || ""))) errors.push("nextCursor_invalid");
}
if (containsSecretLikeMaterial(value)) errors.push("history_must_not_contain_secret_material");
return result(errors);
}
export function validateDataProductPatch(value) {
const errors = envelopeErrors(value, DATA_PRODUCT_PATCH_SCHEMA_VERSION, "patch");
rejectUnknownKeys(value, new Set(["schemaVersion", "dataProduct", "cursor", "previousCursor", "emittedAt", "operations"]), "patch", errors);
@@ -108,13 +170,14 @@ function envelopeErrors(value, schemaVersion, label) {
return errors;
}
function validateFact(value, path, errors, { maxAttributesBytes, canonical = false }) {
function validateFact(value, path, errors, { maxAttributesBytes, canonical = false, allowBucketStart = false }) {
if (!isPlainObject(value)) {
errors.push(`${path}_must_be_object`);
return;
}
const allowedKeys = new Set(["sourceId", "semanticType", "observedAt", "attributes", "geometry"]);
if (canonical) allowedKeys.add("receivedAt");
if (allowBucketStart) allowedKeys.add("bucketStart");
rejectUnknownKeys(value, allowedKeys, path, errors);
requiredIdentifier(value.sourceId, `${path}.sourceId`, errors);
requiredIdentifier(value.semanticType, `${path}.semanticType`, errors);
@@ -129,8 +192,8 @@ function validateFact(value, path, errors, { maxAttributesBytes, canonical = fal
if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors);
}
function validateCanonicalFact(value, path, errors) {
validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true });
function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) {
validateFact(value, path, errors, { maxAttributesBytes: 64 * 1024, canonical: true, allowBucketStart });
if (!isPlainObject(value)) return;
requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors);
}
@@ -19,9 +19,11 @@ export {
} from "./provider-package.mjs";
export {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
validateDataProductHistory,
validateDataProductPatch,
validateDataProductPublish,
validateDataProductSnapshot,
@@ -1,8 +1,10 @@
import assert from "node:assert/strict";
import {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
validateDataProductHistory,
validateDataProductPatch,
validateDataProductPublish,
validateDataProductSnapshot,
@@ -89,6 +91,45 @@ assert.equal(validateDataProductSnapshot({
...snapshot,
facts: [{ ...canonicalFact, attributes: { status: "ndc_edprb_forbidden-reader-token" } }],
}).errors.includes("snapshot_must_not_contain_secret_material"), true);
const history = {
schemaVersion: DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
dataProduct: { id: "fleet.positions.current.v1", version: "1.0.0" },
generatedAt: "2026-07-15T10:05:00.000Z",
query: {
from: "2026-07-15T10:00:00.000Z",
to: "2026-07-15T10:05:00.000Z",
resolutionMs: 60_000,
sourceIds: ["unit-42"],
order: "asc",
},
facts: [{ ...canonicalFact, bucketStart: "2026-07-15T10:00:00.000Z" }],
nextCursor: "opaque_cursor",
};
assert.equal(validateDataProductHistory(history).ok, true);
assert.equal(validateDataProductHistory({
...history,
query: { ...history.query, order: "desc" },
}).errors.includes("query.order_must_be_asc"), true);
assert.equal(validateDataProductHistory({
...history,
query: { ...history.query, from: history.query.to, to: history.query.from },
}).errors.includes("query.range_invalid"), true);
assert.equal(validateDataProductHistory({
...history,
query: { ...history.query, sourceIds: ["unit-42", "unit-42"] },
}).errors.includes("query.sourceIds_must_be_unique_and_sorted"), true);
assert.equal(validateDataProductHistory({
...history,
facts: [
{ ...history.facts[0], sourceId: "unit-43" },
history.facts[0],
],
}).errors.includes("facts[1]_not_strictly_ordered"), true);
assert.equal(validateDataProductHistory({
...history,
facts: [{ ...history.facts[0], attributes: { token: "forbidden" } }],
}).errors.includes("history_must_not_contain_secret_material"), true);
assert.equal(validateDataProductSnapshot({
...snapshot,
facts: [{ ...canonicalFact, attributes: { status: `ndc_edppr_${"P".repeat(43)}` } }],