feat(data-plane): add provider contracts and ontology delivery
This commit is contained in:
Codex
@@ -0,0 +1,280 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import net from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract";
|
||||
|
||||
const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL;
|
||||
if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required");
|
||||
const port = await freePort();
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-api-test-"));
|
||||
const provisionerSecret = "nodedc_edp_integration_provisioner_secret_0123456789abcdef";
|
||||
const secretPath = join(directory, "provisioner-token");
|
||||
await writeFile(secretPath, `${provisionerSecret}\n`, { mode: 0o600 });
|
||||
|
||||
const child = spawn(process.execPath, ["src/server.mjs"], {
|
||||
cwd: new URL("..", import.meta.url),
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: String(port),
|
||||
EXTERNAL_DATA_PLANE_DATABASE_URL: databaseUrl,
|
||||
EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true",
|
||||
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath,
|
||||
EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000",
|
||||
EXTERNAL_DATA_PLANE_STREAM_POLL_MS: "250",
|
||||
EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS: "5000",
|
||||
},
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let childOutput = "";
|
||||
child.stdout.on("data", (chunk) => { childOutput += chunk.toString(); });
|
||||
child.stderr.on("data", (chunk) => { childOutput += chunk.toString(); });
|
||||
|
||||
try {
|
||||
await waitForHealth();
|
||||
const productId = "api.test.positions.v1";
|
||||
const product = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}`, {
|
||||
method: "PUT",
|
||||
token: provisionerSecret,
|
||||
body: {
|
||||
version: "1.0.0",
|
||||
ontologyRevision: "ontology.api.test.v1",
|
||||
deliveryMode: "snapshot+patch",
|
||||
semanticTypes: ["map.moving_object"],
|
||||
fields: ["source_id", "observed_at", "geometry", "status"],
|
||||
history: { mode: "sampled", intervalMs: 60_000, strategy: "latest-per-entity-per-bucket", retentionDays: 30 },
|
||||
},
|
||||
});
|
||||
assert.equal(product.dataProduct.id, productId);
|
||||
|
||||
const expiry = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString();
|
||||
const scope = { tenantId: "api-tenant", connectionId: "api-connection", providerId: "api-provider" };
|
||||
const writerIssuance = await rawJsonRequest("/internal/data-plane/v1/writer-bindings", {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry },
|
||||
});
|
||||
const readerIssuance = await rawJsonRequest("/internal/data-plane/v1/reader-bindings", {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry },
|
||||
});
|
||||
assertOneTimeCapabilityResponse(writerIssuance.response);
|
||||
assertOneTimeCapabilityResponse(readerIssuance.response);
|
||||
const writer = writerIssuance.value;
|
||||
const reader = readerIssuance.value;
|
||||
assert.equal(typeof writer.token, "string");
|
||||
assert.equal(typeof reader.token, "string");
|
||||
|
||||
const catalog = await jsonRequest("/internal/data-plane/v1/writer/data-products", { token: writer.token });
|
||||
assert.deepEqual(catalog.dataProducts.map((value) => value.id), [productId]);
|
||||
|
||||
const first = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61),
|
||||
});
|
||||
assert.equal(first.currentUpdatedCount, 1);
|
||||
const duplicate = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-01", "2026-07-15T12:00:00.000Z", 37.61),
|
||||
});
|
||||
assert.equal(duplicate.idempotent, true);
|
||||
const reusedKey = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${writer.token}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify(publish("api-run-01", "2026-07-15T12:00:00.000Z", 99.9)),
|
||||
});
|
||||
assert.equal(reusedKey.status, 409);
|
||||
assert.equal((await reusedKey.json()).error, "idempotency_key_reused");
|
||||
|
||||
const snapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
|
||||
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
|
||||
assert.equal(snapshot.facts.length, 1);
|
||||
|
||||
const controller = new AbortController();
|
||||
const response = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${snapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
signal: controller.signal,
|
||||
});
|
||||
assert.equal(response.status, 200);
|
||||
const patchPromise = readFirstPatch(response, controller);
|
||||
await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/publish`, {
|
||||
method: "POST",
|
||||
token: writer.token,
|
||||
body: publish("api-run-02", "2026-07-15T12:00:10.000Z", 37.62),
|
||||
});
|
||||
const patch = await patchPromise;
|
||||
assert.equal(validateDataProductPatch(patch).ok, true);
|
||||
assert.equal(patch.previousCursor, snapshot.cursor);
|
||||
|
||||
const forbidden = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${reader.token}`,
|
||||
"X-NODEDC-Tenant-Id": scope.tenantId,
|
||||
"X-NODEDC-Connection-Id": scope.connectionId,
|
||||
},
|
||||
});
|
||||
assert.equal(forbidden.status, 400);
|
||||
|
||||
const revocationSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
|
||||
const revocationStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${revocationSnapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
});
|
||||
assert.equal(revocationStream.status, 200);
|
||||
const revokedEventPromise = readNamedEvent(revocationStream, "error");
|
||||
const rotatedReaderIssuance = await rawJsonRequest(`/internal/data-plane/v1/reader-bindings/${reader.readerBinding.id}/rotate`, {
|
||||
method: "POST",
|
||||
token: provisionerSecret,
|
||||
});
|
||||
assertOneTimeCapabilityResponse(rotatedReaderIssuance.response);
|
||||
const rotatedReader = rotatedReaderIssuance.value;
|
||||
const revokedEvent = await revokedEventPromise;
|
||||
assert.equal(revokedEvent.error, "reader_stream_access_revoked");
|
||||
const rejectedOldReader = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/snapshot`, {
|
||||
headers: { Authorization: `Bearer ${reader.token}` },
|
||||
});
|
||||
assert.equal(rejectedOldReader.status, 401);
|
||||
const rotatedSnapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: rotatedReader.token });
|
||||
assert.equal(rotatedSnapshot.dataProduct.id, productId);
|
||||
|
||||
const shutdownStream = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${rotatedSnapshot.cursor}`, {
|
||||
headers: { Authorization: `Bearer ${rotatedReader.token}` },
|
||||
});
|
||||
assert.equal(shutdownStream.status, 200);
|
||||
child.kill("SIGTERM");
|
||||
assert.equal(await waitForChildExit(child, 3_000), true);
|
||||
|
||||
console.log("external-data-plane API integration: ok");
|
||||
} catch (error) {
|
||||
error.message = `${error.message}\nserver_output=${childOutput.replace(/[A-Za-z0-9_-]{48,}/g, "[redacted]").slice(-4000)}`;
|
||||
throw error;
|
||||
} finally {
|
||||
child.kill("SIGTERM");
|
||||
await Promise.race([
|
||||
new Promise((resolve) => child.once("exit", resolve)),
|
||||
new Promise((resolve) => setTimeout(resolve, 3000)),
|
||||
]);
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function publish(runId, observedAt, longitude) {
|
||||
return {
|
||||
schemaVersion: "nodedc.data-product.publish/v1",
|
||||
batch: { runId, sequence: 0, idempotencyKey: `${runId}.chunk-0` },
|
||||
facts: [{
|
||||
sourceId: "unit-01",
|
||||
semanticType: "map.moving_object",
|
||||
observedAt,
|
||||
attributes: { status: "online" },
|
||||
geometry: { type: "Point", coordinates: [longitude, 55.75] },
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async function jsonRequest(path, { method = "GET", token, body } = {}) {
|
||||
const { response, value } = await rawJsonRequest(path, { method, token, body });
|
||||
if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
async function rawJsonRequest(path, { method = "GET", token, body } = {}) {
|
||||
const response = await fetch(`${baseUrl}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
...(body ? { "Content-Type": "application/json" } : {}),
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
const value = await response.json();
|
||||
if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`);
|
||||
return { response, value };
|
||||
}
|
||||
|
||||
function assertOneTimeCapabilityResponse(response) {
|
||||
assert.equal(response.headers.get("cache-control"), "no-store, max-age=0");
|
||||
assert.equal(response.headers.get("pragma"), "no-cache");
|
||||
assert.equal(response.headers.get("expires"), "0");
|
||||
}
|
||||
|
||||
async function waitForHealth() {
|
||||
const deadline = Date.now() + 15_000;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/healthz`);
|
||||
if (response.ok) return;
|
||||
} catch {}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
throw new Error("server_health_timeout");
|
||||
}
|
||||
|
||||
async function readFirstPatch(response, controller) {
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const timeout = setTimeout(() => controller.abort(), 10_000);
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) throw new Error("stream_closed_before_patch");
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
for (const frame of buffer.split("\n\n")) {
|
||||
const data = frame.split("\n").find((line) => line.startsWith("data: "));
|
||||
if (frame.includes("event: nodedc.data-product.patch.v1") && data) {
|
||||
controller.abort();
|
||||
return JSON.parse(data.slice(6));
|
||||
}
|
||||
}
|
||||
const boundary = buffer.lastIndexOf("\n\n");
|
||||
if (boundary >= 0) buffer = buffer.slice(boundary + 2);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function readNamedEvent(response, eventName) {
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline) {
|
||||
const next = await Promise.race([
|
||||
reader.read(),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("stream_event_timeout")), 10_000)),
|
||||
]);
|
||||
if (next.done) throw new Error(`stream_closed_before_${eventName}`);
|
||||
buffer += decoder.decode(next.value, { stream: true }).replace(/\r\n/g, "\n");
|
||||
let boundary;
|
||||
while ((boundary = buffer.indexOf("\n\n")) !== -1) {
|
||||
const frame = buffer.slice(0, boundary);
|
||||
buffer = buffer.slice(boundary + 2);
|
||||
if (!frame.includes(`event: ${eventName}`)) continue;
|
||||
const data = frame.split("\n").find((line) => line.startsWith("data: "));
|
||||
if (data) return JSON.parse(data.slice(6));
|
||||
}
|
||||
}
|
||||
throw new Error(`stream_event_timeout:${eventName}`);
|
||||
}
|
||||
|
||||
async function waitForChildExit(process, timeoutMs) {
|
||||
if (process.exitCode !== null) return true;
|
||||
return Promise.race([
|
||||
new Promise((resolve) => process.once("exit", () => resolve(true))),
|
||||
new Promise((resolve) => setTimeout(() => resolve(false), timeoutMs)),
|
||||
]);
|
||||
}
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
await new Promise((resolve, reject) => server.listen(0, "127.0.0.1", resolve).once("error", reject));
|
||||
const { port } = server.address();
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
return port;
|
||||
}
|
||||
Reference in New Issue
Block a user