SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(platform): add managed data product history plane

Browse Source
This commit is contained in:
Codex
2026-07-18 14:38:06 +03:00
parent 02816c4352
commit 3c5d8f6cef
34 changed files with 2091 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/external-data-plane/test/api.integration.test.mjs
+51 -2
View File
@@ -5,7 +5,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises";
import net from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import { validateDataProductHistory, validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import {
MANAGED_PROVISIONER_HEADERS,
managedProvisionerSigningPayload,
@@ -77,7 +77,7 @@ try {
const managedBody = {
source: scope,
allowedDataProductIds: [productId],
expiresAt: expiry,
expiresAt: null,
generation: 1,
capabilityDigest: createHash("sha256").update(managedToken, "utf8").digest("hex"),
};
@@ -194,6 +194,48 @@ try {
});
assert.equal(rejectedManagedGeneration1.status, 401);
const managedReaderToken = "ndc_edprb_managed_api_test_0123456789abcdefghijklmnopqrstuvwxyz";
const managedReaderBody = {
source: scope,
allowedDataProductIds: [productId],
expiresAt: null,
generation: 1,
capabilityDigest: createHash("sha256").update(managedReaderToken, "utf8").digest("hex"),
};
const managedReaderPath = "/internal/data-plane/v1/reader-bindings/by-key/engine.api-connection.positions-reader";
const managedReaderResults = await Promise.all([
rawJsonRequest(managedReaderPath, { method: "PUT", managedSignature: true, body: managedReaderBody }),
rawJsonRequest(managedReaderPath, { method: "PUT", managedSignature: true, body: managedReaderBody }),
]);
assert.deepEqual(managedReaderResults.map(({ value }) => value.idempotent).sort(), [false, true]);
assert.equal(managedReaderResults[0].value.readerBinding.id, managedReaderResults[1].value.readerBinding.id);
assert.equal("token" in managedReaderResults[0].value, false);
assert.equal(JSON.stringify(managedReaderResults[0].value).includes(managedReaderBody.capabilityDigest), false);
assert.deepEqual(
(await jsonRequest("/internal/data-plane/v1/reader/data-products", { token: managedReaderToken })).dataProducts.map((value) => value.id),
[productId],
);
const legacyManagedReaderRevoke = await fetch(
`${baseUrl}/internal/data-plane/v1/reader-bindings/${managedReaderResults[0].value.readerBinding.id}/revoke`,
{ method: "POST", headers: { Authorization: `Bearer ${provisionerSecret}` } },
);
assert.equal(legacyManagedReaderRevoke.status, 404);
const managedReaderRevoke = await jsonRequest(
`${managedReaderPath}/generations/1/revoke`,
{ method: "POST", managedSignature: true },
);
assert.equal(managedReaderRevoke.idempotent, false);
assert.equal(managedReaderRevoke.readerBinding.active, false);
const managedReaderRevokeRetry = await jsonRequest(
`${managedReaderPath}/generations/1/revoke`,
{ method: "POST", managedSignature: true },
);
assert.equal(managedReaderRevokeRetry.idempotent, true);
const rejectedManagedReader = await fetch(`${baseUrl}/internal/data-plane/v1/reader/data-products`, {
headers: { Authorization: `Bearer ${managedReaderToken}` },
});
assert.equal(rejectedManagedReader.status, 401);
const manualWriterBody = { source: scope, allowedDataProductIds: [productId], expiresAt: expiry };
const signedOnlyManualProvisioning = await signedFetch("/internal/data-plane/v1/writer-bindings", {
method: "POST",
@@ -245,6 +287,13 @@ try {
const snapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
assert.equal(snapshot.facts.length, 1);
const history = await jsonRequest(
`/internal/data-plane/v1/data-products/${productId}/history?from=2026-07-15T12%3A00%3A00.000Z&to=2026-07-15T12%3A02%3A00.000Z&resolutionMs=60000&sourceIds=unit-01&limit=100`,
{ token: reader.token },
);
assert.equal(validateDataProductHistory(history).ok, true);
assert.equal(history.facts.length, 1);
assert.equal(history.facts[0].bucketStart, "2026-07-15T12:00:00.000Z");
const controller = new AbortController();
const response = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${snapshot.cursor}`, {
services/external-data-plane/test/data-product-delivery.integration.test.mjs
+43 -1
View File
@@ -1,10 +1,11 @@
import assert from "node:assert/strict";
import { Pool } from "pg";
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import { validateDataProductHistory, validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import {
loadDataProductDefinition,
persistDataProductDefinition,
persistDataProductPublish,
readDataProductHistory,
readDataProductSnapshot,
readPatchEvents,
} from "../src/data-product-delivery.mjs";
@@ -154,6 +155,47 @@ try {
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:01:30.000Z"), true);
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:02:00.000Z"), true);
const historyPageOne = await readDataProductHistory(pool, binding, storedDefinition, {
from: "2026-07-15T10:00:00.000Z",
to: "2026-07-15T10:04:00.000Z",
resolutionMs: 60_000,
sourceIds: ["unit-01"],
limit: 2,
});
assert.equal(validateDataProductHistory(historyPageOne).ok, true);
assert.equal(historyPageOne.facts.length, 2);
assert.equal(typeof historyPageOne.nextCursor, "string");
const historyPageTwo = await readDataProductHistory(pool, binding, storedDefinition, {
from: "2026-07-15T10:00:00.000Z",
to: "2026-07-15T10:04:00.000Z",
resolutionMs: 60_000,
sourceIds: ["unit-01"],
limit: 2,
cursor: historyPageOne.nextCursor,
});
assert.equal(validateDataProductHistory(historyPageTwo).ok, true);
assert.equal(historyPageTwo.facts.length, 1);
assert.equal(historyPageTwo.nextCursor, undefined);
assert.deepEqual(
[...historyPageOne.facts, ...historyPageTwo.facts].map((value) => value.bucketStart),
[
"2026-07-15T10:00:00.000Z",
"2026-07-15T10:01:00.000Z",
"2026-07-15T10:02:00.000Z",
],
);
await assert.rejects(
readDataProductHistory(pool, binding, storedDefinition, {
from: "2026-07-15T10:00:00.000Z",
to: "2026-07-15T10:04:00.000Z",
resolutionMs: 60_000,
sourceIds: ["unit-02"],
limit: 2,
cursor: historyPageOne.nextCursor,
}),
(error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid",
);
console.log("external-data-plane delivery integration: ok");
} finally {
await pool.end();
services/external-data-plane/test/reader-binding.test.mjs
+42
View File
@@ -3,7 +3,9 @@ import {
assertReaderProduct,
createReaderToken,
hashReaderToken,
normalizeManagedReaderBindingRequest,
normalizeReaderBindingRequest,
readerBindingRequestHash,
safeReaderBinding,
} from "../src/reader-binding.mjs";
@@ -22,4 +24,44 @@ const safe = safeReaderBinding({ id: "reader-01", ...policy, active: true, token
assert.equal("token" in safe, false);
assert.equal("tokenHash" in safe, false);
const managed = normalizeManagedReaderBindingRequest({
source: { tenantId: policy.tenantId, connectionId: policy.connectionId, providerId: policy.providerId },
allowedDataProductIds: ["fleet.positions.current.v1"],
expiresAt: null,
generation: 2,
capabilityDigest: hashReaderToken(token),
}, { now });
assert.equal(managed.generation, 2);
assert.match(readerBindingRequestHash(managed), /^[a-f0-9]{64}$/);
const safeManaged = safeReaderBinding({
id: "reader-managed-01",
bindingKey: "dprg-reader-managed",
generation: 2,
...managed,
active: true,
});
assert.equal(safeManaged.bindingKey, "dprg-reader-managed");
assert.equal(safeManaged.generation, 2);
assert.equal(safeManaged.expiresAt, null);
assert.equal(
assertReaderProduct(safeManaged, "fleet.positions.current.v1", new Date("2036-07-15T12:00:00.000Z")),
"fleet.positions.current.v1",
);
assert.equal(JSON.stringify(safeManaged).includes(managed.capabilityDigest), false);
assert.throws(() => normalizeManagedReaderBindingRequest({
source: { tenantId: policy.tenantId, connectionId: policy.connectionId, providerId: policy.providerId },
allowedDataProductIds: managed.allowedDataProductIds,
expiresAt: managed.expiresAt,
generation: managed.generation,
capabilityDigest: managed.capabilityDigest,
token: "forbidden",
}, { now }), /managed_reader_binding_secret_material_forbidden/);
assert.throws(() => normalizeManagedReaderBindingRequest({
source: { tenantId: policy.tenantId, connectionId: policy.connectionId, providerId: policy.providerId },
allowedDataProductIds: managed.allowedDataProductIds,
expiresAt: policy.expiresAt,
generation: managed.generation,
capabilityDigest: managed.capabilityDigest,
}), /managed_reader_binding_must_be_durable/);
console.log("external-data-plane reader bindings: ok");
services/external-data-plane/test/writer-binding.test.mjs
+39
View File
@@ -1,6 +1,7 @@
import assert from "node:assert/strict";
import { validateIntakeBatch } from "../../../packages/external-provider-contract/src/data-plane.mjs";
import {
canMigrateManagedWriterBindingToDurable,
createWriterToken,
hashWriterToken,
materializeDataProductPublish,
@@ -38,6 +39,7 @@ assert.notEqual(hashWriterToken(token), hashWriterToken(`${token}x`));
const managedPolicy = normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
allowedDataProductIds: ["fleet.positions.current.v1", "asset.status.current.v1"],
generation: 1,
capabilityDigest: hashWriterToken(token),
@@ -48,26 +50,47 @@ assert.deepEqual(managedPolicy.allowedDataProductIds, [
]);
assert.equal(managedPolicy.generation, 1);
assert.equal(managedPolicy.capabilityDigest, hashWriterToken(token));
assert.equal(managedPolicy.expiresAt, null);
assert.equal(canMigrateManagedWriterBindingToDurable({
...managedPolicy,
expiresAt: "2026-08-01T12:00:00.000Z",
active: true,
capabilityDigest: managedPolicy.capabilityDigest,
}, managedPolicy, now), true);
assert.equal(canMigrateManagedWriterBindingToDurable({
...managedPolicy,
expiresAt: "2026-07-01T12:00:00.000Z",
active: true,
capabilityDigest: managedPolicy.capabilityDigest,
}, managedPolicy, now), false);
assert.equal(writerBindingRequestHash(managedPolicy), writerBindingRequestHash({
...managedPolicy,
allowedDataProductIds: [...managedPolicy.allowedDataProductIds].reverse(),
}));
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
generation: 0,
capabilityDigest: hashWriterToken(token),
}, { now, maxTtlDays: 90 }), /managed_writer_binding_generation_invalid/);
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
generation: 1,
capabilityDigest: "not-a-digest",
}, { now, maxTtlDays: 90 }), /managed_writer_binding_capability_digest_invalid/);
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
generation: 1,
capabilityDigest: hashWriterToken(token),
token,
}, { now, maxTtlDays: 90 }), /managed_writer_binding_secret_material_forbidden/);
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
generation: 1,
capabilityDigest: hashWriterToken(token),
}), /managed_writer_binding_must_be_durable/);
const unscopedBatch = {
schemaVersion: "nodedc.external-provider-contract/v1",
@@ -148,5 +171,21 @@ const safeBinding = safeWriterBinding({
});
assert.equal("token" in safeBinding, false);
assert.equal("tokenHash" in safeBinding, false);
const safeManagedBinding = safeWriterBinding({
id: "binding-managed-01",
...managedPolicy,
active: true,
createdAt: now,
});
assert.equal(safeManagedBinding.expiresAt, null);
assert.equal(materializeDataProductPublish({
schemaVersion: "nodedc.data-product.publish/v1",
batch: { runId: "run-durable", sequence: 0, idempotencyKey: "run-durable.batch-0" },
facts: unscopedBatch.facts,
}, safeManagedBinding, {
id: "fleet.positions.current.v1",
version: "1.0.0",
ontologyRevision: "ontology.example-fleet.v1",
}, "fleet.positions.current.v1", { now: new Date("2036-07-15T12:00:00.000Z") }).batch.runId, "run-durable");
console.log("external-data-plane writer bindings: ok");