feat(data-plane): add provider contracts and ontology delivery
This commit is contained in:
Codex
@@ -0,0 +1,406 @@
|
||||
import express from "express";
|
||||
import { randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { createServer } from "node:http";
|
||||
import { Pool } from "pg";
|
||||
import { GELIOS_CAPABILITIES } from "./capabilities.mjs";
|
||||
import { readConfig, unitIsInScope } from "./config.mjs";
|
||||
import { FLEET_POSITIONS_CURRENT, toFleetPositionPatch, toFleetPositionsSnapshot } from "./data-products.mjs";
|
||||
import { extractUnitItems, hash, normalizeUnit } from "./normalize.mjs";
|
||||
import { migrate } from "./schema.mjs";
|
||||
|
||||
const config = readConfig();
|
||||
const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize });
|
||||
const app = express();
|
||||
const httpServer = createServer(app);
|
||||
const streamClients = new Set();
|
||||
let intakeInFlight = null;
|
||||
|
||||
app.disable("x-powered-by");
|
||||
app.use(express.json({ limit: "1mb" }));
|
||||
|
||||
app.get("/healthz", asyncRoute(async (_req, res) => {
|
||||
await pool.query("select 1");
|
||||
res.json({
|
||||
ok: true,
|
||||
service: "nodedc-gelios-gateway",
|
||||
database: "ready",
|
||||
internalApiConfigured: Boolean(config.internalAccessToken),
|
||||
intakeEnabled: config.intakeEnabled,
|
||||
unitScope: config.unitScope,
|
||||
credentialOwner: "engine",
|
||||
providerTransport: "absent",
|
||||
commandTransport: "absent",
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/status", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const latestRun = await pool.query(
|
||||
`select id, capability_id, status, unit_count, inserted_snapshots, error_code, started_at, completed_at
|
||||
from gelios_collection_runs
|
||||
where tenant_id = $1 and connection_id = $2
|
||||
order by started_at desc
|
||||
limit 1`,
|
||||
[scope.tenantId, scope.connectionId],
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
provider: "gelios",
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
unitScope: config.unitScope,
|
||||
approvedUnitCount: config.unitScope === "all" ? null : config.allowedUnitIds.length,
|
||||
intakeEnabled: config.intakeEnabled,
|
||||
credentialOwner: "engine",
|
||||
providerTransport: "absent",
|
||||
commandTransport: "absent",
|
||||
lastRun: latestRun.rows[0] || null,
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/capabilities", requireInternalApi, (_req, res) => {
|
||||
res.json({ ok: true, provider: "gelios", capabilities: GELIOS_CAPABILITIES, commandTransport: "absent" });
|
||||
});
|
||||
|
||||
app.post("/internal/gelios/v1/intake/units", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const result = await runIntake({
|
||||
scope,
|
||||
payload: req.body,
|
||||
receivedAt: parseReceivedAt(req.body?.receivedAt),
|
||||
});
|
||||
res.status(result.status === "completed" ? 200 : 409).json(result);
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/units/current", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const limit = boundedLimit(req.query.limit, 200, 1, 1000);
|
||||
const rows = await listCurrentRows(scope, limit);
|
||||
res.json({
|
||||
ok: true,
|
||||
contract: "gelios.current-position.v1",
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
units: rows,
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/data-products/fleet.positions.current.v1/snapshot", requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireConnectionScope(req);
|
||||
const limit = boundedLimit(req.query.limit, 200, 1, 1000);
|
||||
const rows = await listCurrentRows(scope, limit);
|
||||
res.json({
|
||||
ok: true,
|
||||
...toFleetPositionsSnapshot(rows, { ...scope, staleAfterMs: config.positionStaleAfterMs }),
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/gelios/v1/data-products/fleet.positions.current.v1/stream", requireInternalApi, (req, res) => {
|
||||
let scope;
|
||||
try {
|
||||
scope = requireConnectionScope(req);
|
||||
} catch (error) {
|
||||
res.status(Number(error.status || 400)).json({ ok: false, error: error.message });
|
||||
return;
|
||||
}
|
||||
const client = openSseClient(req, res, scope, "data-product");
|
||||
writeSse(res, "nodedc.data-product.ready.v1", {
|
||||
dataProduct: FLEET_POSITIONS_CURRENT,
|
||||
scope,
|
||||
bootstrap: "fetch-snapshot",
|
||||
});
|
||||
return client;
|
||||
});
|
||||
|
||||
app.get("/internal/gelios/v1/stream", requireInternalApi, (req, res) => {
|
||||
let scope;
|
||||
try {
|
||||
scope = requireConnectionScope(req);
|
||||
} catch (error) {
|
||||
res.status(Number(error.status || 400)).json({ ok: false, error: error.message });
|
||||
return;
|
||||
}
|
||||
const client = openSseClient(req, res, scope, "legacy");
|
||||
writeSse(res, "gelios.ready", {
|
||||
ok: true,
|
||||
contract: "gelios.current-position.v1",
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
commandTransport: "absent",
|
||||
});
|
||||
return client;
|
||||
});
|
||||
|
||||
async function listCurrentRows(scope, limit) {
|
||||
const rows = await pool.query(
|
||||
`select
|
||||
stable_subject_id as "subjectId", source_unit_id as "sourceUnitId", name,
|
||||
observed_at as "observedAt", received_at as "receivedAt", latitude, longitude,
|
||||
speed, course, gps_valid as "gpsValid", satellites, hdop,
|
||||
accuracy_m as "accuracyM", telemetry
|
||||
from gelios_unit_current
|
||||
where tenant_id = $1 and connection_id = $2
|
||||
order by observed_at desc, source_unit_id asc
|
||||
limit $3`,
|
||||
[scope.tenantId, scope.connectionId, limit],
|
||||
);
|
||||
return rows.rows;
|
||||
}
|
||||
|
||||
app.use((error, _req, res, _next) => {
|
||||
const status = Number(error?.status || 500);
|
||||
const publicStatus = status >= 400 && status < 600 ? status : 500;
|
||||
console.error(JSON.stringify({ event: "gelios_gateway_error", error: safeErrorCode(error), status: publicStatus }));
|
||||
res.status(publicStatus).json({ ok: false, error: publicStatus >= 500 ? "internal_error" : safeErrorCode(error) });
|
||||
});
|
||||
|
||||
await migrate(pool);
|
||||
await upsertConnection();
|
||||
|
||||
httpServer.listen(config.port, "0.0.0.0", () => {
|
||||
console.log(`NODE.DC Gelios Gateway listening on http://0.0.0.0:${config.port}`);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
async function upsertConnection() {
|
||||
await pool.query(
|
||||
`insert into gelios_connections (
|
||||
tenant_id, connection_id, unit_scope, allowed_unit_ids, collection_enabled, updated_at
|
||||
) values ($1, $2, $3, $4::jsonb, $5, now())
|
||||
on conflict (tenant_id, connection_id) do update
|
||||
set unit_scope = excluded.unit_scope,
|
||||
allowed_unit_ids = excluded.allowed_unit_ids,
|
||||
collection_enabled = excluded.collection_enabled,
|
||||
updated_at = now()`,
|
||||
[config.tenantId, config.connectionId, config.unitScope, JSON.stringify(config.allowedUnitIds), config.intakeEnabled],
|
||||
);
|
||||
}
|
||||
|
||||
async function runIntake({ scope, payload, receivedAt }) {
|
||||
if (!config.intakeEnabled) return { ok: false, status: "disabled" };
|
||||
if (config.unitScope === "allowlist" && !config.allowedUnitIds.length) {
|
||||
return { ok: false, status: "scope_missing" };
|
||||
}
|
||||
if (intakeInFlight) return { ok: false, status: "busy" };
|
||||
intakeInFlight = ingestCurrentUnits({ scope, payload, receivedAt }).finally(() => {
|
||||
intakeInFlight = null;
|
||||
});
|
||||
return intakeInFlight;
|
||||
}
|
||||
|
||||
async function ingestCurrentUnits({ scope, payload, receivedAt }) {
|
||||
const runId = randomUUID();
|
||||
await pool.query(
|
||||
`insert into gelios_collection_runs (id, tenant_id, connection_id, capability_id, status)
|
||||
values ($1, $2, $3, 'gelios.units.current_intake', 'running')`,
|
||||
[runId, scope.tenantId, scope.connectionId],
|
||||
);
|
||||
|
||||
try {
|
||||
const sourcePayload = payload?.units ? { units: payload.units } : payload;
|
||||
const items = extractUnitItems(sourcePayload);
|
||||
if (!items.length) throw Object.assign(new Error("intake_units_required"), { status: 422 });
|
||||
await storeRawEnvelope({ runId, scope, payload: sourcePayload, receivedAt });
|
||||
const normalized = items
|
||||
.map((item) => normalizeUnit(item, { ...scope, receivedAt }))
|
||||
.filter(Boolean)
|
||||
.filter((item) => unitIsInScope(config, item.sourceUnitId));
|
||||
|
||||
let insertedSnapshots = 0;
|
||||
for (const unit of normalized) {
|
||||
const result = await persistCurrentUnit(unit);
|
||||
insertedSnapshots += result.insertedSnapshot ? 1 : 0;
|
||||
if (result.current) broadcastCurrentUnit(result.current, scope.tenantId, scope.connectionId);
|
||||
}
|
||||
await pruneExpiredRawEnvelopes();
|
||||
await pool.query(
|
||||
`update gelios_collection_runs
|
||||
set status = 'completed', unit_count = $2, inserted_snapshots = $3, completed_at = now()
|
||||
where id = $1`,
|
||||
[runId, normalized.length, insertedSnapshots],
|
||||
);
|
||||
return { ok: true, status: "completed", runId, unitCount: normalized.length, insertedSnapshots };
|
||||
} catch (error) {
|
||||
await pool.query(
|
||||
`update gelios_collection_runs
|
||||
set status = 'failed', error_code = $2, completed_at = now()
|
||||
where id = $1`,
|
||||
[runId, safeErrorCode(error)],
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function storeRawEnvelope({ runId, scope, payload, receivedAt }) {
|
||||
const serialized = JSON.stringify(payload);
|
||||
await pool.query(
|
||||
`insert into gelios_raw_envelopes (
|
||||
id, tenant_id, connection_id, collection_run_id, capability_id,
|
||||
payload_hash, payload, payload_bytes, received_at, expires_at
|
||||
) values ($1, $2, $3, $4, 'gelios.units.current_intake', $5, $6::jsonb, $7, $8, $9)
|
||||
on conflict (tenant_id, connection_id, capability_id, payload_hash) do nothing`,
|
||||
[
|
||||
randomUUID(), scope.tenantId, scope.connectionId, runId, hash(payload), serialized,
|
||||
Buffer.byteLength(serialized), receivedAt,
|
||||
new Date(receivedAt.getTime() + config.rawRetentionDays * 24 * 60 * 60 * 1000),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
async function persistCurrentUnit(unit) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
await client.query(
|
||||
`insert into gelios_units (
|
||||
tenant_id, connection_id, source_unit_id, stable_subject_id, name, last_seen_at
|
||||
) values ($1, $2, $3, $4, $5, $6)
|
||||
on conflict (tenant_id, connection_id, source_unit_id) do update
|
||||
set stable_subject_id = excluded.stable_subject_id,
|
||||
name = excluded.name,
|
||||
last_seen_at = excluded.last_seen_at`,
|
||||
[unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.stableSubjectId, unit.name, unit.receivedAt],
|
||||
);
|
||||
const current = await client.query(
|
||||
`insert into gelios_unit_current (
|
||||
tenant_id, connection_id, source_unit_id, stable_subject_id, name,
|
||||
observed_at, received_at, latitude, longitude, position, speed, course,
|
||||
gps_valid, satellites, hdop, accuracy_m, telemetry, fingerprint
|
||||
) values (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9,
|
||||
case when $8::double precision is not null and $9::double precision is not null
|
||||
then ST_SetSRID(ST_MakePoint($9, $8), 4326)::geography else null end,
|
||||
$10, $11, $12, $13, $14, $15, $16::jsonb, $17
|
||||
)
|
||||
on conflict (tenant_id, connection_id, source_unit_id) do update
|
||||
set stable_subject_id = excluded.stable_subject_id,
|
||||
name = excluded.name,
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
latitude = excluded.latitude,
|
||||
longitude = excluded.longitude,
|
||||
position = excluded.position,
|
||||
speed = excluded.speed,
|
||||
course = excluded.course,
|
||||
gps_valid = excluded.gps_valid,
|
||||
satellites = excluded.satellites,
|
||||
hdop = excluded.hdop,
|
||||
accuracy_m = excluded.accuracy_m,
|
||||
telemetry = excluded.telemetry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at >= gelios_unit_current.observed_at
|
||||
returning stable_subject_id as "subjectId", source_unit_id as "sourceUnitId", name,
|
||||
observed_at as "observedAt", received_at as "receivedAt", latitude, longitude,
|
||||
speed, course, gps_valid as "gpsValid", satellites, hdop, accuracy_m as "accuracyM", telemetry`,
|
||||
[
|
||||
unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.stableSubjectId, unit.name,
|
||||
unit.observedAt, unit.receivedAt, unit.latitude, unit.longitude, unit.speed, unit.course,
|
||||
unit.gpsValid, unit.satellites, unit.hdop, unit.accuracyM, JSON.stringify(unit.telemetry), unit.fingerprint,
|
||||
],
|
||||
);
|
||||
const snapshot = await client.query(
|
||||
`insert into gelios_telemetry_snapshots (
|
||||
tenant_id, connection_id, source_unit_id, observed_at, received_at, fingerprint, telemetry
|
||||
) values ($1, $2, $3, $4, $5, $6, $7::jsonb)
|
||||
on conflict do nothing`,
|
||||
[unit.tenantId, unit.connectionId, unit.sourceUnitId, unit.observedAt, unit.receivedAt, unit.fingerprint, JSON.stringify(unit.telemetry)],
|
||||
);
|
||||
await client.query("commit");
|
||||
return { current: current.rows[0] || null, insertedSnapshot: snapshot.rowCount > 0 };
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function pruneExpiredRawEnvelopes() {
|
||||
await pool.query("delete from gelios_raw_envelopes where expires_at < now()");
|
||||
}
|
||||
|
||||
function requireInternalApi(req, res, next) {
|
||||
const expected = Buffer.from(config.internalAccessToken || "");
|
||||
const received = Buffer.from(String(req.get("authorization") || "").replace(/^Bearer\s+/i, "").trim());
|
||||
if (!expected.length) {
|
||||
res.status(503).json({ ok: false, error: "internal_api_not_configured" });
|
||||
return;
|
||||
}
|
||||
if (expected.length !== received.length || !timingSafeEqual(expected, received)) {
|
||||
res.status(401).json({ ok: false, error: "internal_authorization_required" });
|
||||
return;
|
||||
}
|
||||
next();
|
||||
}
|
||||
|
||||
function requireConnectionScope(req) {
|
||||
const tenantId = String(req.get("x-nodedc-tenant-id") || "").trim();
|
||||
const connectionId = String(req.query.connectionId || req.get("x-nodedc-connection-id") || config.connectionId).trim();
|
||||
if (tenantId !== config.tenantId) throw Object.assign(new Error("tenant_scope_required"), { status: 403 });
|
||||
if (connectionId !== config.connectionId) throw Object.assign(new Error("connection_scope_denied"), { status: 403 });
|
||||
return { tenantId, connectionId };
|
||||
}
|
||||
|
||||
function broadcastCurrentUnit(unit, tenantId, connectionId) {
|
||||
for (const client of streamClients) {
|
||||
if (client.tenantId === tenantId && client.connectionId === connectionId) {
|
||||
if (client.kind === "data-product") {
|
||||
writeSse(client.res, "nodedc.data-product.patch.v1", toFleetPositionPatch(unit, {
|
||||
staleAfterMs: config.positionStaleAfterMs,
|
||||
}));
|
||||
} else {
|
||||
writeSse(client.res, "gelios.current.updated", unit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function openSseClient(req, res, scope, kind) {
|
||||
const client = { id: randomUUID(), res, kind, ...scope };
|
||||
res.setHeader("Content-Type", "text/event-stream");
|
||||
res.setHeader("Cache-Control", "no-cache, no-transform");
|
||||
res.setHeader("Connection", "keep-alive");
|
||||
res.setHeader("X-Accel-Buffering", "no");
|
||||
res.flushHeaders?.();
|
||||
const keepAlive = setInterval(() => res.write(": keep-alive\n\n"), 30000);
|
||||
streamClients.add(client);
|
||||
req.on("close", () => {
|
||||
clearInterval(keepAlive);
|
||||
streamClients.delete(client);
|
||||
});
|
||||
return client;
|
||||
}
|
||||
|
||||
function writeSse(res, event, payload) {
|
||||
res.write(`event: ${event}\ndata: ${JSON.stringify(payload)}\n\n`);
|
||||
}
|
||||
|
||||
function parseReceivedAt(value) {
|
||||
if (value === undefined || value === null || value === "") return new Date();
|
||||
const parsed = new Date(value);
|
||||
if (Number.isNaN(parsed.getTime())) throw Object.assign(new Error("intake_received_at_invalid"), { status: 422 });
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function boundedLimit(value, fallback, min, max) {
|
||||
const parsed = Number.parseInt(String(value || ""), 10);
|
||||
if (!Number.isInteger(parsed)) return fallback;
|
||||
return Math.max(min, Math.min(max, parsed));
|
||||
}
|
||||
|
||||
function safeErrorCode(error) {
|
||||
return String(error?.message || "internal_error").replace(/[^a-zA-Z0-9_:.-]+/g, "_").slice(0, 160);
|
||||
}
|
||||
|
||||
function asyncRoute(handler) {
|
||||
return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next);
|
||||
}
|
||||
|
||||
async function shutdown() {
|
||||
await new Promise((resolve) => httpServer.close(resolve));
|
||||
await pool.end();
|
||||
process.exit(0);
|
||||
}
|
||||
Reference in New Issue
Block a user