feat(data-plane): add provider contracts and ontology delivery
This commit is contained in:
Codex
@@ -0,0 +1,280 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import net from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract";
|
||||
|
||||
const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL;
|
||||
if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required");
|
||||
const port = await freePort();
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-api-test-"));
|
||||
const provisionerSecret = "nodedc_edp_integration_provisioner_secret_0123456789abcdef";
|
||||
const secretPath = join(directory, "provisioner-token");
|
||||
await writeFile(secretPath, `${provisionerSecret}\n`, { mode: 0o600 });
|
||||
|
||||
const child = spawn(process.execPath, ["src/server.mjs"], {
|
||||
cwd: new URL("..", import.meta.url),
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: String(port),
|
||||
EXTERNAL_DATA_PLANE_DATABASE_URL: databaseUrl,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath,
|
||||
EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000",
|
||||
EXTERNAL_DATA_PLANE_STREAM_POLL_MS: "250",
|
||||
EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS: "5000",
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let childOutput = "";
|
||||
child.stdout.on("data", (chunk) => { childOutput += chunk.toString(); });
|
||||
child.stderr.on("data", (chunk) => { childOutput += chunk.toString(); });
|
||||
|
||||
try {
|
||||
await waitForHealth();
|
||||
const productId = "api.test.positions.v1";
|
||||
const product = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}`, {
|
||||
method: "PUT",
|
||||
token: provisionerSecret,
|
||||
body: {
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.api.test.v1",
|
||||
deliveryMode: "snapshot+patch",
|
||||
semanticTypes: ["map.moving_object"],
|
||||
fields: ["source_id", "observed_at", "geometry", "status"],
|
||||
history: { mode: "sampled", intervalMs: 60_000, strategy: "latest-per-entity-per-bucket", retentionDays: 30 },
|
||||
},
|
||||
});
|
||||
assert.equal(product.dataProduct.id, productId);
|
||||
|
||||
const expiry = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString();
|
||||
const scope = { tenantId: "api-tenant", connectionId: "api-connection", providerId: "api-provider" };
|
||||
const writerIssuance = await rawJsonRequest("/internal/data-plane/v1/writer-bindings", {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry },
|
||||
});
|
||||
const readerIssuance = await rawJsonRequest("/internal/data-plane/v1/reader-bindings", {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry },
|
||||
});
|
||||
assertOneTimeCapabilityResponse(writerIssuance.response);
|
||||
assertOneTimeCapabilityResponse(readerIssuance.response);
|
||||
const writer = writerIssuance.value;
|
||||
const reader = readerIssuance.value;
|
||||
assert.equal(typeof writer.token, "string");
|
||||
assert.equal(typeof reader.token, "string");
|
||||
|
||||
const catalog = await jsonRequest("/internal/data-plane/v1/writer/data-products", { token: writer.token });
|
||||
assert.deepEqual(catalog.dataProducts.map((value) => value.id), [productId]);
|
||||
|
||||
const first = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61),
|
||||
});
|
||||
assert.equal(first.currentUpdatedCount, 1);
|
||||
const duplicate = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61),
|
||||
});
|
||||
assert.equal(duplicate.idempotent, true);
|
||||
const reusedKey = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${writer.token}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify(publish("api-run-01", "2026-07-15T12:00:00.000Z", 99.9)),
|
||||
});
|
||||
assert.equal(reusedKey.status, 409);
|
||||
assert.equal((await reusedKey.json()).error, "idempotency_key_reused");
|
||||
|
||||
const snapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
|
||||
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
|
||||
assert.equal(snapshot.facts.length, 1);
|
||||
|
||||
const controller = new AbortController();
|
||||
const response = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${snapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
signal: controller.signal,
|
||||
});
|
||||
assert.equal(response.status, 200);
|
||||
const patchPromise = readFirstPatch(response, controller);
|
||||
await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-02", "2026-07-15T12:00:10.000Z", 37.62),
|
||||
});
|
||||
const patch = await patchPromise;
|
||||
assert.equal(validateDataProductPatch(patch).ok, true);
|
||||
assert.equal(patch.previousCursor, snapshot.cursor);
|
||||
|
||||
const forbidden = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${reader.token}`,
|
||||
"X-NODEDC-Tenant-Id": scope.tenantId,
|
||||
"X-NODEDC-Connection-Id": scope.connectionId,
|
||||
},
|
||||
});
|
||||
assert.equal(forbidden.status, 400);
|
||||
|
||||
const revocationSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
|
||||
const revocationStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${revocationSnapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
});
|
||||
assert.equal(revocationStream.status, 200);
|
||||
const revokedEventPromise = readNamedEvent(revocationStream, "error");
|
||||
const rotatedReaderIssuance = await rawJsonRequest(`/internal/data-plane/v1/reader-bindings/${reader.readerBinding.id}/rotate`, {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
});
|
||||
assertOneTimeCapabilityResponse(rotatedReaderIssuance.response);
|
||||
const rotatedReader = rotatedReaderIssuance.value;
|
||||
const revokedEvent = await revokedEventPromise;
|
||||
assert.equal(revokedEvent.error, "reader_stream_access_revoked");
|
||||
const rejectedOldReader = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
});
|
||||
assert.equal(rejectedOldReader.status, 401);
|
||||
const rotatedSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: rotatedReader.token });
|
||||
assert.equal(rotatedSnapshot.dataProduct.id, productId);
|
||||
|
||||
const shutdownStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${rotatedSnapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${rotatedReader.token}` },
|
||||
});
|
||||
assert.equal(shutdownStream.status, 200);
|
||||
child.kill("SIGTERM");
|
||||
assert.equal(await waitForChildExit(child, 3_000), true);
|
||||
|
||||
console.log("external-data-plane API integration: ok");
|
||||
} catch (error) {
|
||||
error.message = `${error.message}\nserver_output=${childOutput.replace(/[A-Za-z0-9_-]{48,}/g, "[redacted]").slice(-4000)}`;
|
||||
throw error;
|
||||
} finally {
|
||||
child.kill("SIGTERM");
|
||||
await Promise.race([
|
||||
new Promise((resolve) => child.once("exit", resolve)),
|
||||
new Promise((resolve) => setTimeout(resolve, 3000)),
|
||||
]);
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function publish(runId, observedAt, longitude) {
|
||||
return {
|
||||
schemaVersion: "nodedc.data-product.publish/v1",
|
||||
batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0` },
|
||||
facts: [{
|
||||
sourceId: "unit-01",
|
||||
semanticType: "map.moving_object",
|
||||
observedAt,
|
||||
attributes: { status: "online" },
|
||||
geometry: { type: "Point", coordinates: [longitude, 55.75] },
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async function jsonRequest(path, { method = "GET", token, body } = {}) {
|
||||
const { response, value } = await rawJsonRequest(path, { method, token, body });
|
||||
if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
async function rawJsonRequest(path, { method = "GET", token, body } = {}) {
|
||||
const response = await fetch(`${baseUrl}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
...(body ? { "Content-Type": "application/json" } : {}),
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
const value = await response.json();
|
||||
if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`);
|
||||
return { response, value };
|
||||
}
|
||||
|
||||
function assertOneTimeCapabilityResponse(response) {
|
||||
assert.equal(response.headers.get("cache-control"), "no-store, max-age=0");
|
||||
assert.equal(response.headers.get("pragma"), "no-cache");
|
||||
assert.equal(response.headers.get("expires"), "0");
|
||||
}
|
||||
|
||||
async function waitForHealth() {
|
||||
const deadline = Date.now() + 15_000;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/healthz`);
|
||||
if (response.ok) return;
|
||||
} catch {}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
throw new Error("server_health_timeout");
|
||||
}
|
||||
|
||||
async function readFirstPatch(response, controller) {
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const timeout = setTimeout(() => controller.abort(), 10_000);
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) throw new Error("stream_closed_before_patch");
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
for (const frame of buffer.split("\n\n")) {
|
||||
const data = frame.split("\n").find((line) => line.startsWith("data: "));
|
||||
if (frame.includes("event: nodedc.data-product.patch.v1") && data) {
|
||||
controller.abort();
|
||||
return JSON.parse(data.slice(6));
|
||||
}
|
||||
}
|
||||
const boundary = buffer.lastIndexOf("\n\n");
|
||||
if (boundary >= 0) buffer = buffer.slice(boundary + 2);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function readNamedEvent(response, eventName) {
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline) {
|
||||
const next = await Promise.race([
|
||||
reader.read(),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("stream_event_timeout")), 10_000)),
|
||||
]);
|
||||
if (next.done) throw new Error(`stream_closed_before_${eventName}`);
|
||||
buffer += decoder.decode(next.value, { stream: true }).replace(/\r\n/g, "\n");
|
||||
let boundary;
|
||||
while ((boundary = buffer.indexOf("\n\n")) !== -1) {
|
||||
const frame = buffer.slice(0, boundary);
|
||||
buffer = buffer.slice(boundary + 2);
|
||||
if (!frame.includes(`event: ${eventName}`)) continue;
|
||||
const data = frame.split("\n").find((line) => line.startsWith("data: "));
|
||||
if (data) return JSON.parse(data.slice(6));
|
||||
}
|
||||
}
|
||||
throw new Error(`stream_event_timeout:${eventName}`);
|
||||
}
|
||||
|
||||
async function waitForChildExit(process, timeoutMs) {
|
||||
if (process.exitCode !== null) return true;
|
||||
return Promise.race([
|
||||
new Promise((resolve) => process.once("exit", () => resolve(true))),
|
||||
new Promise((resolve) => setTimeout(() => resolve(false), timeoutMs)),
|
||||
]);
|
||||
}
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject));
|
||||
const { port } = server.address();
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
return port;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { readConfig } from "../src/config.mjs";
|
||||
|
||||
const base = {
|
||||
EXTERNAL_DATA_PLANE_DATABASE_URL: "postgresql://user:pass@example.invalid/data_plane",
|
||||
NODEDC_INTERNAL_ACCESS_TOKEN: "legacy-internal-token",
|
||||
};
|
||||
|
||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-config-"));
|
||||
const secretPath = join(directory, "provisioner-token");
|
||||
const secret = "provisioner_secret_is_separate_from_legacy_internal_token_123456";
|
||||
try {
|
||||
await writeFile(secretPath, `${secret}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
const config = readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath,
|
||||
EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS: "120",
|
||||
EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000",
|
||||
});
|
||||
assert.equal(config.provisionerAccessToken, secret);
|
||||
assert.equal(config.maxFutureSkewSeconds, 120);
|
||||
assert.equal(config.retentionSweepMs, 60000);
|
||||
assert.equal(config.maxPatchBytes, 262144);
|
||||
assert.equal(config.receiptRetentionMs, 604800000);
|
||||
assert.equal(config.legacyIntakeEnabled, false);
|
||||
assert.equal(readConfig({ ...base, EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED: "true" }).legacyIntakeEnabled, true);
|
||||
assert.throws(() => readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath,
|
||||
NODEDC_INTERNAL_ACCESS_TOKEN: secret,
|
||||
}), /provisioner_token_must_differ_from_internal_token/);
|
||||
assert.throws(() => readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"),
|
||||
}), /external_data_plane_provisioner_token_file_unreadable/);
|
||||
assert.equal(readConfig({
|
||||
...base,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "false",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"),
|
||||
}).provisionerAccessToken, "");
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
console.log("external-data-plane config: ok");
|
||||
@@ -0,0 +1,184 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { Pool } from "pg";
|
||||
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract";
|
||||
import {
|
||||
loadDataProductDefinition,
|
||||
persistDataProductDefinition,
|
||||
persistDataProductPublish,
|
||||
readDataProductSnapshot,
|
||||
readPatchEvents,
|
||||
} from "../src/data-product-delivery.mjs";
|
||||
import { normalizeDataProductDefinition } from "../src/data-product-policy.mjs";
|
||||
import { migrate } from "../src/schema.mjs";
|
||||
|
||||
const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL;
|
||||
if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required");
|
||||
const pool = new Pool({ connectionString: databaseUrl, max: 4 });
|
||||
|
||||
try {
|
||||
await migrate(pool);
|
||||
await pool.query(`truncate table
|
||||
external_data_plane_patch_outbox,
|
||||
external_data_plane_history,
|
||||
external_data_plane_current,
|
||||
external_data_plane_facts,
|
||||
external_data_plane_raw_envelopes,
|
||||
external_data_plane_batches,
|
||||
external_data_plane_delivery_state,
|
||||
external_data_plane_reader_bindings,
|
||||
external_data_plane_writer_bindings,
|
||||
external_data_plane_products
|
||||
restart identity cascade`);
|
||||
|
||||
const definition = normalizeDataProductDefinition({
|
||||
id: "test.positions.current.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.test.positions.v1",
|
||||
deliveryMode: "snapshot+patch",
|
||||
semanticTypes: ["map.moving_object"],
|
||||
fields: ["source_id", "observed_at", "geometry", "status"],
|
||||
history: {
|
||||
mode: "sampled",
|
||||
intervalMs: 60_000,
|
||||
strategy: "latest-per-entity-per-bucket",
|
||||
retentionDays: 30,
|
||||
},
|
||||
});
|
||||
await persistDataProductDefinition(pool, definition);
|
||||
const storedDefinition = await loadDataProductDefinition(pool, definition.id);
|
||||
const binding = {
|
||||
id: "reader-binding-test",
|
||||
tenantId: "tenant-test",
|
||||
connectionId: "connection-test",
|
||||
providerId: "provider-test",
|
||||
allowedDataProductIds: [definition.id],
|
||||
active: true,
|
||||
expiresAt: "2027-01-01T00:00:00.000Z",
|
||||
};
|
||||
|
||||
const firstBatch = batch("run-01", "2026-07-15T10:00:05.000Z", [
|
||||
fact("unit-01", 37.61, 55.75, "online"),
|
||||
fact("unit-02", 37.62, 55.76, "online"),
|
||||
]);
|
||||
const first = await persistDataProductPublish(pool, firstBatch, storedDefinition);
|
||||
assert.equal(first.idempotent, false);
|
||||
assert.equal(first.currentUpdatedCount, 2);
|
||||
assert.equal(first.historyInsertedCount, 2);
|
||||
assert.equal(first.cursor, "1");
|
||||
|
||||
const duplicate = await persistDataProductPublish(pool, firstBatch, storedDefinition);
|
||||
assert.equal(duplicate.idempotent, true);
|
||||
assert.equal(duplicate.cursor, "1");
|
||||
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, {
|
||||
...firstBatch,
|
||||
facts: [fact("unit-01", 99.9, 55.75, "changed")],
|
||||
}, storedDefinition),
|
||||
(error) => error?.status === 409 && error?.code === "idempotency_key_reused",
|
||||
);
|
||||
|
||||
const concurrentBatch = batch("run-concurrent", "2026-07-15T10:00:08.000Z", [
|
||||
fact("unit-03", 37.63, 55.77, "online", "2026-07-15T10:00:08.000Z"),
|
||||
]);
|
||||
const concurrent = await Promise.all([
|
||||
persistDataProductPublish(pool, concurrentBatch, storedDefinition),
|
||||
persistDataProductPublish(pool, concurrentBatch, storedDefinition),
|
||||
]);
|
||||
assert.deepEqual(concurrent.map((value) => value.idempotent).sort(), [false, true]);
|
||||
|
||||
const second = await persistDataProductPublish(pool, batch(
|
||||
"run-02",
|
||||
"2026-07-15T10:00:15.000Z",
|
||||
[fact("unit-01", 37.611, 55.751, "moving", "2026-07-15T10:00:15.000Z")],
|
||||
), storedDefinition);
|
||||
assert.equal(second.currentUpdatedCount, 1);
|
||||
assert.equal(second.cursor, "3");
|
||||
|
||||
const latest = await persistDataProductPublish(pool, batch(
|
||||
"run-latest",
|
||||
"2026-07-15T10:02:05.000Z",
|
||||
[fact("unit-01", 37.7, 55.8, "latest", "2026-07-15T10:02:00.000Z")],
|
||||
), storedDefinition);
|
||||
assert.equal(latest.currentUpdatedCount, 1);
|
||||
const late = await persistDataProductPublish(pool, batch(
|
||||
"run-late",
|
||||
"2026-07-15T10:03:00.000Z",
|
||||
[fact("unit-01", 37.65, 55.78, "late", "2026-07-15T10:01:30.000Z")],
|
||||
), storedDefinition);
|
||||
assert.equal(late.currentUpdatedCount, 0);
|
||||
assert.equal(late.historyInsertedCount, 1);
|
||||
assert.equal(late.patchOperationCount, 0);
|
||||
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, batch("run-bad-type", "2026-07-15T10:03:10.000Z", [{
|
||||
...fact("unit-04", 37.6, 55.7, "online"),
|
||||
semanticType: "map.forbidden",
|
||||
}]), storedDefinition),
|
||||
(error) => error?.status === 422 && error?.code === "data_product_semantic_type_forbidden",
|
||||
);
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, batch("run-bad-field", "2026-07-15T10:03:11.000Z", [{
|
||||
...fact("unit-04", 37.6, 55.7, "online"),
|
||||
attributes: { undeclared: true },
|
||||
}]), storedDefinition),
|
||||
(error) => error?.status === 422 && error?.code === "data_product_field_forbidden",
|
||||
);
|
||||
await assert.rejects(
|
||||
persistDataProductPublish(pool, batch("run-duplicate-fact", "2026-07-15T10:03:12.000Z", [
|
||||
fact("unit-04", 37.6, 55.7, "online"),
|
||||
fact("unit-04", 37.61, 55.71, "moving"),
|
||||
]), storedDefinition),
|
||||
(error) => error?.status === 422 && error?.code === "data_product_duplicate_entity_key",
|
||||
);
|
||||
|
||||
const snapshot = await readDataProductSnapshot(pool, binding, storedDefinition);
|
||||
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
|
||||
assert.equal(snapshot.cursor, "4");
|
||||
assert.equal(snapshot.facts.length, 3);
|
||||
assert.equal(snapshot.facts.find((value) => value.sourceId === "unit-01").attributes.status, "latest");
|
||||
|
||||
const patches = await readPatchEvents(pool, binding, storedDefinition, 0n);
|
||||
assert.equal(patches.length, 4);
|
||||
assert.equal(patches.every((patch) => validateDataProductPatch(patch).ok), true);
|
||||
assert.deepEqual(patches.map((patch) => patch.cursor), ["1", "2", "3", "4"]);
|
||||
|
||||
await assert.rejects(
|
||||
readPatchEvents(pool, binding, storedDefinition, 999n),
|
||||
(error) => error?.status === 409 && error?.code === "resync_required",
|
||||
);
|
||||
|
||||
const history = await pool.query("select source_id, observed_at from external_data_plane_history order by source_id");
|
||||
assert.equal(history.rowCount, 5);
|
||||
const unitOneHistory = history.rows.filter((row) => row.source_id === "unit-01");
|
||||
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:01:30.000Z"), true);
|
||||
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:02:00.000Z"), true);
|
||||
|
||||
console.log("external-data-plane delivery integration: ok");
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
function batch(runId, receivedAt, facts) {
|
||||
return {
|
||||
schemaVersion: "nodedc.external-provider-contract/v1",
|
||||
source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" },
|
||||
contract: {
|
||||
dataProductId: "test.positions.current.v1",
|
||||
ontologyRevision: "ontology.test.positions.v1",
|
||||
version: "1.0.0",
|
||||
},
|
||||
batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0`, receivedAt },
|
||||
facts,
|
||||
};
|
||||
}
|
||||
|
||||
function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10:00:00.000Z") {
|
||||
return {
|
||||
sourceId,
|
||||
semanticType: "map.moving_object",
|
||||
observedAt,
|
||||
attributes: { status },
|
||||
geometry: { type: "Point", coordinates: [longitude, latitude] },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { normalizeDataProductDefinition } from "../src/data-product-policy.mjs";
|
||||
|
||||
const input = {
|
||||
id: "fleet.positions.current.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.map.v1",
|
||||
deliveryMode: "snapshot+patch",
|
||||
semanticTypes: ["vehicle.trike", "map.moving_object"],
|
||||
fields: ["status", "source_id", "observed_at", "geometry"],
|
||||
history: {
|
||||
mode: "sampled",
|
||||
intervalMs: 60_000,
|
||||
strategy: "latest-per-entity-per-bucket",
|
||||
retentionDays: 365,
|
||||
},
|
||||
};
|
||||
const definition = normalizeDataProductDefinition(input);
|
||||
assert.equal(definition.history.mode, "sampled");
|
||||
assert.equal(definition.history.intervalMs, 60_000);
|
||||
assert.deepEqual(definition.semanticTypes, ["map.moving_object", "vehicle.trike"]);
|
||||
assert.deepEqual(definition.fields, ["geometry", "observed_at", "source_id", "status"]);
|
||||
assert.deepEqual(input.semanticTypes, ["vehicle.trike", "map.moving_object"]);
|
||||
assert.deepEqual(input.fields, ["status", "source_id", "observed_at", "geometry"]);
|
||||
assert.equal(Object.isFrozen(definition.semanticTypes), true);
|
||||
assert.equal(Object.isFrozen(definition.fields), true);
|
||||
|
||||
const reordered = normalizeDataProductDefinition({
|
||||
...input,
|
||||
semanticTypes: [...input.semanticTypes].reverse(),
|
||||
fields: [...input.fields].reverse(),
|
||||
});
|
||||
assert.deepEqual(reordered.semanticTypes, definition.semanticTypes);
|
||||
assert.deepEqual(reordered.fields, definition.fields);
|
||||
|
||||
assert.throws(() => normalizeDataProductDefinition({ ...definition, providerId: "gelios" }), /data_product_definition_invalid/);
|
||||
assert.throws(() => normalizeDataProductDefinition({ ...definition, history: { mode: "none", intervalMs: 1000 } }), /history_policy_none_has_sampling_fields/);
|
||||
|
||||
for (const semanticTypes of [
|
||||
undefined,
|
||||
"map.moving_object",
|
||||
[],
|
||||
["map.moving_object", null],
|
||||
["map.moving_object", "Map.invalid"],
|
||||
]) {
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, semanticTypes }),
|
||||
/data_product_definition_(semantic_types_invalid|shape_invalid)/,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, semanticTypes: ["map.moving_object", "map.moving_object"] }),
|
||||
/data_product_definition_semantic_types_duplicate/,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, semanticTypes: ["map.moving_object", " map.moving_object "] }),
|
||||
/data_product_definition_semantic_types_duplicate/,
|
||||
);
|
||||
|
||||
for (const fields of [
|
||||
undefined,
|
||||
"source_id",
|
||||
[],
|
||||
["source_id", null],
|
||||
["source_id", "Invalid"],
|
||||
]) {
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, fields }),
|
||||
/data_product_definition_(fields_invalid|shape_invalid)/,
|
||||
);
|
||||
}
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, fields: ["source_id", "source_id"] }),
|
||||
/data_product_definition_fields_duplicate/,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeDataProductDefinition({ ...input, fields: ["source_id", " source_id "] }),
|
||||
/data_product_definition_fields_duplicate/,
|
||||
);
|
||||
|
||||
console.log("external-data-plane data product policy: ok");
|
||||
@@ -0,0 +1,32 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { loadDataProductDefinitions } from "../src/definitions.mjs";
|
||||
|
||||
const bundled = await loadDataProductDefinitions();
|
||||
assert.deepEqual(bundled.map((definition) => definition.id), ["fleet.positions.current.v1"]);
|
||||
assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]);
|
||||
assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1");
|
||||
assert.equal(bundled[0].history.mode, "sampled");
|
||||
assert.equal(bundled[0].history.intervalMs, 60_000);
|
||||
assert.equal(bundled[0].history.retentionDays, 90);
|
||||
assert.equal(bundled[0].fields.includes("geometry"), true);
|
||||
assert.equal(bundled[0].fields.includes("providerUnitId"), false);
|
||||
assert.equal(bundled[0].fields.includes("provider_unit_id"), false);
|
||||
|
||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-"));
|
||||
try {
|
||||
await writeFile(join(directory, "wrong-name.json"), JSON.stringify({
|
||||
...bundled[0],
|
||||
id: "another.product.v1",
|
||||
}));
|
||||
await assert.rejects(loadDataProductDefinitions(directory), /data_product_definition_filename_mismatch/);
|
||||
await mkdir(join(directory, "ignored.json"));
|
||||
await assert.rejects(loadDataProductDefinitions(directory), /data_product_definition_file_invalid/);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
console.log("external-data-plane definitions: ok");
|
||||
@@ -0,0 +1,26 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { assertBatchTimeBounds, rawRetentionExpiry } from "../src/intake-policy.mjs";
|
||||
|
||||
const now = new Date("2026-07-15T12:00:00.000Z");
|
||||
const batch = {
|
||||
batch: { receivedAt: "2026-07-15T12:04:59.000Z" },
|
||||
facts: [{ observedAt: "2026-07-15T12:05:00.000Z" }],
|
||||
};
|
||||
|
||||
assert.doesNotThrow(() => assertBatchTimeBounds(batch, { now, maxFutureSkewSeconds: 300 }));
|
||||
assert.throws(() => assertBatchTimeBounds({
|
||||
...batch,
|
||||
batch: { receivedAt: "2026-07-15T12:05:01.000Z" },
|
||||
}, { now, maxFutureSkewSeconds: 300 }), /batch_received_at_too_far_in_future/);
|
||||
assert.throws(() => assertBatchTimeBounds({
|
||||
...batch,
|
||||
facts: [{ observedAt: "2026-07-15T12:05:01.000Z" }],
|
||||
}, { now, maxFutureSkewSeconds: 300 }), /fact_observed_at_too_far_in_future/);
|
||||
|
||||
assert.equal(
|
||||
rawRetentionExpiry({ now, rawRetentionDays: 14 }).toISOString(),
|
||||
"2026-07-29T12:00:00.000Z",
|
||||
);
|
||||
assert.throws(() => rawRetentionExpiry({ now, rawRetentionDays: 0 }), /raw_retention_days_invalid/);
|
||||
|
||||
console.log("external-data-plane intake policy: ok");
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
assertReaderProduct,
|
||||
createReaderToken,
|
||||
hashReaderToken,
|
||||
normalizeReaderBindingRequest,
|
||||
safeReaderBinding,
|
||||
} from "../src/reader-binding.mjs";
|
||||
|
||||
const now = new Date("2026-07-15T12:00:00.000Z");
|
||||
const policy = normalizeReaderBindingRequest({
|
||||
source: { tenantId: "tenant-01", connectionId: "connection-01", providerId: "example-provider" },
|
||||
allowedDataProductIds: ["fleet.positions.current.v1"],
|
||||
expiresAt: "2026-08-01T12:00:00.000Z",
|
||||
}, { now });
|
||||
const token = createReaderToken();
|
||||
assert.match(token, /^ndc_edprb_[A-Za-z0-9_-]{40,}$/);
|
||||
assert.equal(hashReaderToken(token), hashReaderToken(token));
|
||||
assert.equal(assertReaderProduct({ ...policy, active: true }, "fleet.positions.current.v1", now), "fleet.positions.current.v1");
|
||||
assert.throws(() => assertReaderProduct({ ...policy, active: true }, "other.product.v1", now), /reader_binding_data_product_forbidden/);
|
||||
const safe = safeReaderBinding({ id: "reader-01", ...policy, active: true, token, tokenHash: hashReaderToken(token) });
|
||||
assert.equal("token" in safe, false);
|
||||
assert.equal("tokenHash" in safe, false);
|
||||
|
||||
console.log("external-data-plane reader bindings: ok");
|
||||
@@ -0,0 +1,117 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { validateIntakeBatch } from "../../../packages/external-provider-contract/src/index.mjs";
|
||||
import {
|
||||
createWriterToken,
|
||||
hashWriterToken,
|
||||
materializeDataProductPublish,
|
||||
materializeWriterBoundBatch,
|
||||
normalizeWriterBindingRequest,
|
||||
safeWriterBinding,
|
||||
} from "../src/writer-binding.mjs";
|
||||
|
||||
const now = new Date("2026-07-15T12:00:00.000Z");
|
||||
const request = {
|
||||
source: {
|
||||
tenantId: "tenant-01",
|
||||
connectionId: "connection-01",
|
||||
providerId: "example-provider",
|
||||
},
|
||||
allowedDataProductIds: ["fleet.positions.current.v1"],
|
||||
expiresAt: "2026-08-01T12:00:00.000Z",
|
||||
};
|
||||
|
||||
const bindingPolicy = normalizeWriterBindingRequest(request, { now, maxTtlDays: 90 });
|
||||
assert.deepEqual(bindingPolicy, {
|
||||
tenantId: "tenant-01",
|
||||
connectionId: "connection-01",
|
||||
providerId: "example-provider",
|
||||
allowedDataProductIds: ["fleet.positions.current.v1"],
|
||||
expiresAt: "2026-08-01T12:00:00.000Z",
|
||||
});
|
||||
|
||||
const token = createWriterToken();
|
||||
assert.match(token, /^ndc_edpwb_[A-Za-z0-9_-]{40,}$/);
|
||||
assert.equal(hashWriterToken(token), hashWriterToken(token));
|
||||
assert.notEqual(hashWriterToken(token), hashWriterToken(`${token}x`));
|
||||
|
||||
const unscopedBatch = {
|
||||
schemaVersion: "nodedc.external-provider-contract/v1",
|
||||
source: { providerId: "example-provider" },
|
||||
contract: { dataProductId: "fleet.positions.current.v1", ontologyRevision: "example.v1", version: "1.0.0" },
|
||||
batch: { runId: "run-01", sequence: 0, idempotencyKey: "run-01.batch-0", receivedAt: "2026-07-15T12:00:00.000Z" },
|
||||
facts: [{ sourceId: "unit-01", semanticType: "map.moving_object", observedAt: "2026-07-15T12:00:00.000Z" }],
|
||||
};
|
||||
const bound = materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: true }, { now });
|
||||
assert.deepEqual(bound.source, { providerId: "example-provider", tenantId: "tenant-01", connectionId: "connection-01" });
|
||||
assert.equal(validateIntakeBatch(bound).ok, true);
|
||||
assert.throws(() => materializeWriterBoundBatch({
|
||||
...unscopedBatch,
|
||||
source: { ...unscopedBatch.source, tenantId: "forged-tenant" },
|
||||
}, { ...bindingPolicy, active: true }, { now }), /writer_bound_scope_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: true }, {
|
||||
now,
|
||||
hasScopeHeaders: true,
|
||||
}), /writer_bound_scope_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch(unscopedBatch, {
|
||||
...bindingPolicy,
|
||||
providerId: "other-provider",
|
||||
active: true,
|
||||
}, { now }), /writer_binding_provider_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch({
|
||||
...unscopedBatch,
|
||||
contract: { ...unscopedBatch.contract, dataProductId: "other.product.v1" },
|
||||
}, { ...bindingPolicy, active: true }, { now }), /writer_binding_data_product_forbidden/);
|
||||
assert.throws(() => materializeWriterBoundBatch(unscopedBatch, { ...bindingPolicy, active: false }, { now }), /writer_binding_inactive/);
|
||||
|
||||
const materializedPublish = materializeDataProductPublish({
|
||||
schemaVersion: "nodedc.data-product.publish/v1",
|
||||
batch: { runId: "run-02", sequence: 0, idempotencyKey: "run-02.batch-0" },
|
||||
facts: unscopedBatch.facts,
|
||||
}, { ...bindingPolicy, active: true }, {
|
||||
id: "fleet.positions.current.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.example-fleet.v1",
|
||||
}, "fleet.positions.current.v1", { now });
|
||||
assert.deepEqual(materializedPublish.source, {
|
||||
tenantId: "tenant-01",
|
||||
connectionId: "connection-01",
|
||||
providerId: "example-provider",
|
||||
});
|
||||
assert.equal(materializedPublish.batch.receivedAt, now.toISOString());
|
||||
assert.equal(materializedPublish.contract.version, "1.0.0");
|
||||
assert.equal(validateIntakeBatch(materializedPublish).ok, true);
|
||||
assert.throws(() => materializeDataProductPublish({
|
||||
schemaVersion: "nodedc.data-product.publish/v1",
|
||||
batch: { runId: "run-02", sequence: 0, idempotencyKey: "run-02.batch-0" },
|
||||
facts: unscopedBatch.facts,
|
||||
}, { ...bindingPolicy, active: true }, {
|
||||
id: "other.product.v1",
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.example-fleet.v1",
|
||||
}, "other.product.v1", { now }), /writer_binding_data_product_forbidden/);
|
||||
|
||||
assert.throws(() => normalizeWriterBindingRequest({
|
||||
...request,
|
||||
expiresAt: "2027-01-01T00:00:00.000Z",
|
||||
}, { now, maxTtlDays: 90 }), /writer_binding_expiry_invalid/);
|
||||
assert.throws(() => normalizeWriterBindingRequest({
|
||||
...request,
|
||||
apiToken: "must-never-be-here",
|
||||
}, { now, maxTtlDays: 90 }), /writer_binding_request_secret_material_forbidden/);
|
||||
assert.throws(() => normalizeWriterBindingRequest({
|
||||
...request,
|
||||
arbitraryRuntimeSetting: "must-not-be-accepted",
|
||||
}, { now, maxTtlDays: 90 }), /writer_binding_request_fields_invalid/);
|
||||
|
||||
const safeBinding = safeWriterBinding({
|
||||
id: "binding-01",
|
||||
...bindingPolicy,
|
||||
active: true,
|
||||
createdAt: now,
|
||||
token: "must-not-leak",
|
||||
tokenHash: "must-not-leak",
|
||||
});
|
||||
assert.equal("token" in safeBinding, false);
|
||||
assert.equal("tokenHash" in safeBinding, false);
|
||||
|
||||
console.log("external-data-plane writer bindings: ok");
|
||||
Reference in New Issue
Block a user