319 lines
13 KiB
JavaScript
319 lines
13 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { Pool } from "pg";
|
|
import { validateDataProductHistory, validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
|
|
import {
|
|
loadDataProductDefinition,
|
|
persistDataProductDefinition,
|
|
persistDataProductPublish,
|
|
readDataProductHistory,
|
|
readDataProductSnapshot,
|
|
readPatchEvents,
|
|
} from "../src/data-product-delivery.mjs";
|
|
import { normalizeDataProductDefinition } from "../src/data-product-policy.mjs";
|
|
import { migrate } from "../src/schema.mjs";
|
|
|
|
const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL;
|
|
if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required");
|
|
const pool = new Pool({ connectionString: databaseUrl, max: 4 });
|
|
|
|
try {
|
|
await migrate(pool);
|
|
await pool.query(`truncate table
|
|
external_data_plane_patch_outbox,
|
|
external_data_plane_history,
|
|
external_data_plane_current,
|
|
external_data_plane_facts,
|
|
external_data_plane_raw_envelopes,
|
|
external_data_plane_batches,
|
|
external_data_plane_delivery_state,
|
|
external_data_plane_reader_bindings,
|
|
external_data_plane_writer_bindings,
|
|
external_data_plane_products
|
|
restart identity cascade`);
|
|
|
|
const definition = normalizeDataProductDefinition({
|
|
id: "test.positions.current.v1",
|
|
version: "1.0.0",
|
|
ontologyRevision: "ontology.test.positions.v1",
|
|
deliveryMode: "snapshot+patch",
|
|
semanticTypes: ["map.moving_object"],
|
|
fields: ["source_id", "observed_at", "geometry", "status"],
|
|
fieldContracts: {
|
|
source_id: { type: "string", required: true },
|
|
observed_at: { type: "string", required: true },
|
|
geometry: { type: "point", required: true },
|
|
status: { type: "string", required: true },
|
|
},
|
|
history: {
|
|
mode: "sampled",
|
|
intervalMs: 60_000,
|
|
strategy: "latest-per-entity-per-bucket",
|
|
retentionDays: 30,
|
|
},
|
|
});
|
|
await persistDataProductDefinition(pool, definition);
|
|
const storedDefinition = await loadDataProductDefinition(pool, definition.id);
|
|
assert.deepEqual(storedDefinition.fieldContracts, definition.fieldContracts);
|
|
const binding = {
|
|
id: "reader-binding-test",
|
|
bindingKey: "managed-reader-binding-test",
|
|
tenantId: "tenant-test",
|
|
connectionId: "consumer-connection-test",
|
|
sourceConnectionId: "connection-test",
|
|
providerId: "provider-test",
|
|
allowedDataProductIds: [definition.id],
|
|
active: true,
|
|
expiresAt: "2027-01-01T00:00:00.000Z",
|
|
};
|
|
|
|
const firstBatch = batch("run-01", "2026-07-15T10:00:05.000Z", [
|
|
fact("unit-01", 37.61, 55.75, "online"),
|
|
fact("unit-02", 37.62, 55.76, "online"),
|
|
]);
|
|
const first = await persistDataProductPublish(pool, firstBatch, storedDefinition);
|
|
assert.equal(first.idempotent, false);
|
|
assert.equal(first.currentUpdatedCount, 2);
|
|
assert.equal(first.historyInsertedCount, 2);
|
|
assert.equal(first.cursor, "1");
|
|
|
|
const duplicate = await persistDataProductPublish(pool, firstBatch, storedDefinition);
|
|
assert.equal(duplicate.idempotent, true);
|
|
assert.equal(duplicate.cursor, "1");
|
|
|
|
await assert.rejects(
|
|
persistDataProductPublish(pool, {
|
|
...firstBatch,
|
|
facts: [fact("unit-01", 99.9, 55.75, "changed")],
|
|
}, storedDefinition),
|
|
(error) => error?.status === 409 && error?.code === "idempotency_key_reused",
|
|
);
|
|
|
|
const concurrentBatch = batch("run-concurrent", "2026-07-15T10:00:08.000Z", [
|
|
fact("unit-03", 37.63, 55.77, "online", "2026-07-15T10:00:08.000Z"),
|
|
]);
|
|
const concurrent = await Promise.all([
|
|
persistDataProductPublish(pool, concurrentBatch, storedDefinition),
|
|
persistDataProductPublish(pool, concurrentBatch, storedDefinition),
|
|
]);
|
|
assert.deepEqual(concurrent.map((value) => value.idempotent).sort(), [false, true]);
|
|
|
|
const second = await persistDataProductPublish(pool, batch(
|
|
"run-02",
|
|
"2026-07-15T10:00:15.000Z",
|
|
[fact("unit-01", 37.611, 55.751, "moving", "2026-07-15T10:00:15.000Z")],
|
|
), storedDefinition);
|
|
assert.equal(second.currentUpdatedCount, 1);
|
|
assert.equal(second.cursor, "3");
|
|
|
|
const latest = await persistDataProductPublish(pool, batch(
|
|
"run-latest",
|
|
"2026-07-15T10:02:05.000Z",
|
|
[fact("unit-01", 37.7, 55.8, "latest", "2026-07-15T10:02:00.000Z")],
|
|
), storedDefinition);
|
|
assert.equal(latest.currentUpdatedCount, 1);
|
|
const late = await persistDataProductPublish(pool, batch(
|
|
"run-late",
|
|
"2026-07-15T10:03:00.000Z",
|
|
[fact("unit-01", 37.65, 55.78, "late", "2026-07-15T10:01:30.000Z")],
|
|
), storedDefinition);
|
|
assert.equal(late.currentUpdatedCount, 0);
|
|
assert.equal(late.historyInsertedCount, 1);
|
|
assert.equal(late.patchOperationCount, 0);
|
|
|
|
await assert.rejects(
|
|
persistDataProductPublish(pool, batch("run-bad-type", "2026-07-15T10:03:10.000Z", [{
|
|
...fact("unit-04", 37.6, 55.7, "online"),
|
|
semanticType: "map.forbidden",
|
|
}]), storedDefinition),
|
|
(error) => error?.status === 422 && error?.code === "data_product_semantic_type_forbidden",
|
|
);
|
|
await assert.rejects(
|
|
persistDataProductPublish(pool, batch("run-bad-field", "2026-07-15T10:03:11.000Z", [{
|
|
...fact("unit-04", 37.6, 55.7, "online"),
|
|
attributes: { undeclared: true },
|
|
}]), storedDefinition),
|
|
(error) => error?.status === 422 && error?.code === "data_product_field_forbidden",
|
|
);
|
|
await assert.rejects(
|
|
persistDataProductPublish(pool, batch("run-duplicate-fact", "2026-07-15T10:03:12.000Z", [
|
|
fact("unit-04", 37.6, 55.7, "online"),
|
|
fact("unit-04", 37.61, 55.71, "moving"),
|
|
]), storedDefinition),
|
|
(error) => error?.status === 422 && error?.code === "data_product_duplicate_entity_key",
|
|
);
|
|
|
|
const snapshot = await readDataProductSnapshot(pool, binding, storedDefinition);
|
|
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
|
|
assert.equal(snapshot.cursor, "4");
|
|
assert.equal(snapshot.facts.length, 3);
|
|
assert.equal(snapshot.facts.find((value) => value.sourceId === "unit-01").attributes.status, "latest");
|
|
|
|
const patches = await readPatchEvents(pool, binding, storedDefinition, 0n);
|
|
assert.equal(patches.length, 4);
|
|
assert.equal(patches.every((patch) => validateDataProductPatch(patch).ok), true);
|
|
assert.deepEqual(patches.map((patch) => patch.cursor), ["1", "2", "3", "4"]);
|
|
|
|
await assert.rejects(
|
|
readPatchEvents(pool, binding, storedDefinition, 999n),
|
|
(error) => error?.status === 409 && error?.code === "resync_required",
|
|
);
|
|
|
|
const history = await pool.query("select source_id, observed_at from external_data_plane_history order by source_id");
|
|
assert.equal(history.rowCount, 5);
|
|
const unitOneHistory = history.rows.filter((row) => row.source_id === "unit-01");
|
|
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:01:30.000Z"), true);
|
|
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:02:00.000Z"), true);
|
|
|
|
const historyPageOne = await readDataProductHistory(pool, binding, storedDefinition, {
|
|
from: "2026-07-15T10:00:00.000Z",
|
|
to: "2026-07-15T10:04:00.000Z",
|
|
resolutionMs: 60_000,
|
|
sourceIds: ["unit-01"],
|
|
limit: 2,
|
|
});
|
|
assert.equal(validateDataProductHistory(historyPageOne).ok, true);
|
|
assert.equal(historyPageOne.facts.length, 2);
|
|
assert.equal(typeof historyPageOne.nextCursor, "string");
|
|
const historyPageTwo = await readDataProductHistory(pool, binding, storedDefinition, {
|
|
from: "2026-07-15T10:00:00.000Z",
|
|
to: "2026-07-15T10:04:00.000Z",
|
|
resolutionMs: 60_000,
|
|
sourceIds: ["unit-01"],
|
|
limit: 2,
|
|
cursor: historyPageOne.nextCursor,
|
|
});
|
|
assert.equal(validateDataProductHistory(historyPageTwo).ok, true);
|
|
assert.equal(historyPageTwo.facts.length, 1);
|
|
assert.equal(historyPageTwo.nextCursor, undefined);
|
|
assert.deepEqual(
|
|
[...historyPageOne.facts, ...historyPageTwo.facts].map((value) => value.bucketStart),
|
|
[
|
|
"2026-07-15T10:00:00.000Z",
|
|
"2026-07-15T10:01:00.000Z",
|
|
"2026-07-15T10:02:00.000Z",
|
|
],
|
|
);
|
|
await assert.rejects(
|
|
readDataProductHistory(pool, binding, storedDefinition, {
|
|
from: "2026-07-15T10:00:00.000Z",
|
|
to: "2026-07-15T10:04:00.000Z",
|
|
resolutionMs: 60_000,
|
|
sourceIds: ["unit-02"],
|
|
limit: 2,
|
|
cursor: historyPageOne.nextCursor,
|
|
}),
|
|
(error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid",
|
|
);
|
|
|
|
const zoneDefinition = normalizeDataProductDefinition({
|
|
id: "test.zones.current.v1",
|
|
version: "1.0.0",
|
|
ontologyRevision: "ontology.map.zone.v1",
|
|
deliveryMode: "snapshot+patch",
|
|
semanticTypes: ["map.zone"],
|
|
fields: ["display_name", "geometry", "geometry_kind", "source_kind", "source_revision"],
|
|
fieldContracts: {
|
|
display_name: { type: "string", required: true },
|
|
geometry: { type: "geometry", required: true },
|
|
geometry_kind: { type: "string", required: true, enum: ["polygon"] },
|
|
source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] },
|
|
source_revision: { type: "string", required: true },
|
|
},
|
|
history: { mode: "none", retentionDays: 1 },
|
|
});
|
|
await persistDataProductDefinition(pool, zoneDefinition);
|
|
const storedZoneDefinition = await loadDataProductDefinition(pool, zoneDefinition.id);
|
|
const zoneBinding = { ...binding, allowedDataProductIds: [zoneDefinition.id] };
|
|
const firstZoneGenerationAt = "2026-07-15T10:10:00.000Z";
|
|
const firstZoneGeneration = await persistDataProductPublish(
|
|
pool,
|
|
zoneBatch("zones-01", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt), zoneFact("zone-02", firstZoneGenerationAt)]),
|
|
storedZoneDefinition,
|
|
);
|
|
assert.equal(firstZoneGeneration.currentUpdatedCount, 2);
|
|
assert.equal(firstZoneGeneration.currentRemovedCount, 0);
|
|
const secondZoneGenerationAt = "2026-07-15T10:11:00.000Z";
|
|
const secondZoneGeneration = await persistDataProductPublish(
|
|
pool,
|
|
zoneBatch("zones-02", secondZoneGenerationAt, [zoneFact("zone-01", secondZoneGenerationAt)]),
|
|
storedZoneDefinition,
|
|
);
|
|
assert.equal(secondZoneGeneration.currentUpdatedCount, 1);
|
|
assert.equal(secondZoneGeneration.currentRemovedCount, 1);
|
|
assert.equal(secondZoneGeneration.patchOperationCount, 2);
|
|
const zoneSnapshot = await readDataProductSnapshot(pool, zoneBinding, storedZoneDefinition);
|
|
assert.equal(validateDataProductSnapshot(zoneSnapshot).ok, true);
|
|
assert.deepEqual(zoneSnapshot.facts.map((value) => value.sourceId), ["zone-01"]);
|
|
assert.equal(zoneSnapshot.facts[0].geometry.type, "Polygon");
|
|
const zonePatches = await readPatchEvents(pool, zoneBinding, storedZoneDefinition, 0n);
|
|
assert.equal(zonePatches.every((value) => validateDataProductPatch(value).ok), true);
|
|
assert.equal(zonePatches.flatMap((value) => value.operations).some((value) => value.op === "remove" && value.sourceId === "zone-02"), true);
|
|
await assert.rejects(
|
|
persistDataProductPublish(pool, zoneBatch("zones-stale", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt)]), storedZoneDefinition),
|
|
(error) => error?.status === 409 && error?.code === "data_product_generation_not_newer",
|
|
);
|
|
|
|
console.log("external-data-plane delivery integration: ok");
|
|
} finally {
|
|
await pool.end();
|
|
}
|
|
|
|
function batch(runId, receivedAt, facts) {
|
|
return {
|
|
schemaVersion: "nodedc.external-provider-contract/v1",
|
|
source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" },
|
|
contract: {
|
|
dataProductId: "test.positions.current.v1",
|
|
ontologyRevision: "ontology.test.positions.v1",
|
|
version: "1.0.0",
|
|
},
|
|
batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0`, receivedAt },
|
|
facts,
|
|
};
|
|
}
|
|
|
|
function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10:00:00.000Z") {
|
|
return {
|
|
sourceId,
|
|
semanticType: "map.moving_object",
|
|
observedAt,
|
|
attributes: { status },
|
|
geometry: { type: "Point", coordinates: [longitude, latitude] },
|
|
};
|
|
}
|
|
|
|
function zoneBatch(runId, generationAt, facts) {
|
|
return {
|
|
schemaVersion: "nodedc.external-provider-contract/v1",
|
|
source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" },
|
|
contract: { dataProductId: "test.zones.current.v1", ontologyRevision: "ontology.map.zone.v1", version: "1.0.0" },
|
|
batch: {
|
|
runId,
|
|
sequence: 0,
|
|
idempotencyKey: `${runId}.chunk-0`,
|
|
receivedAt: generationAt,
|
|
mode: "replace",
|
|
generationAt,
|
|
},
|
|
facts,
|
|
};
|
|
}
|
|
|
|
function zoneFact(sourceId, observedAt) {
|
|
return {
|
|
sourceId,
|
|
semanticType: "map.zone",
|
|
observedAt,
|
|
attributes: {
|
|
display_name: sourceId,
|
|
geometry_kind: "polygon",
|
|
source_kind: "live_api",
|
|
source_revision: "gelios-rest-v1",
|
|
},
|
|
geometry: {
|
|
type: "Polygon",
|
|
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
|
|
},
|
|
};
|
|
}
|