feat(foundry): close the operational map data loop
This commit is contained in:
Codex
@@ -43,6 +43,19 @@ function safeErrorCode(error) {
|
||||
}
|
||||
|
||||
function safeStatusFromFact(fact, policy, nowMs) {
|
||||
const statusContract = policy.statusContract || null;
|
||||
if (statusContract) {
|
||||
const sourceStatus = fact?.attributes?.[statusContract.attribute];
|
||||
if (typeof sourceStatus !== "string" || !statusContract.allowedValues.includes(sourceStatus)) {
|
||||
throw consumerError("data_product_consumer_fact_status_invalid", 502);
|
||||
}
|
||||
if (statusContract.freshness === "none" || policy.terminalStatuses.includes(sourceStatus)) {
|
||||
return sourceStatus;
|
||||
}
|
||||
const observedAt = Date.parse(String(fact?.observedAt || ""));
|
||||
if (Number.isFinite(observedAt) && nowMs - observedAt > policy.staleAfterMs) return "stale";
|
||||
return sourceStatus;
|
||||
}
|
||||
const sourceStatus = [fact?.attributes?.operational_status, fact?.attributes?.status]
|
||||
.find((value) => typeof value === "string" && value.trim());
|
||||
const normalized = String(sourceStatus || "active").trim().toLowerCase().replace(/[^a-z0-9_-]+/g, "-").slice(0, 64) || "active";
|
||||
@@ -112,20 +125,39 @@ function safeStateSummary(record) {
|
||||
timestamps: state.timestamps,
|
||||
lastError: state.lastError || null,
|
||||
readerGrant: "target-scoped-server-only",
|
||||
readerGrantGeneration: Number.isSafeInteger(state.readerGrantGeneration) ? state.readerGrantGeneration : 1,
|
||||
};
|
||||
}
|
||||
|
||||
function activeReaderGrantGeneration(state) {
|
||||
const generation = Number(state?.readerGrantGeneration ?? 1);
|
||||
if (!Number.isSafeInteger(generation) || generation < 1 || generation > 2_147_483_647) {
|
||||
throw consumerError("data_product_reader_grant_generation_invalid", 500);
|
||||
}
|
||||
return generation;
|
||||
}
|
||||
|
||||
function validatePolicy(policy, product) {
|
||||
if (!policy || policy.dataProductId !== product.id || policy.productVersion !== product.version) {
|
||||
throw consumerError("data_product_consumer_policy_not_found", 409);
|
||||
}
|
||||
if (!Number.isInteger(policy.staleAfterMs) || policy.staleAfterMs < 1_000 || policy.staleAfterMs > 7 * 24 * 60 * 60 * 1000) {
|
||||
const statusContract = policy.statusContract === undefined
|
||||
? null
|
||||
: validateStatusContract(policy.statusContract);
|
||||
const staleDisabled = statusContract?.freshness === "none";
|
||||
if (
|
||||
(staleDisabled && policy.staleAfterMs !== null)
|
||||
|| (!staleDisabled && (!Number.isInteger(policy.staleAfterMs) || policy.staleAfterMs < 1_000 || policy.staleAfterMs > 7 * 24 * 60 * 60 * 1000))
|
||||
) {
|
||||
throw consumerError("data_product_consumer_policy_invalid", 500);
|
||||
}
|
||||
const terminalStatuses = Array.isArray(policy.terminalStatuses) ? policy.terminalStatuses : [];
|
||||
if (terminalStatuses.some((value) => typeof value !== "string" || !/^[a-z0-9_-]{1,64}$/.test(value))) {
|
||||
throw consumerError("data_product_consumer_policy_invalid", 500);
|
||||
}
|
||||
if (statusContract && terminalStatuses.some((value) => !statusContract.allowedValues.includes(value))) {
|
||||
throw consumerError("data_product_consumer_policy_invalid", 500);
|
||||
}
|
||||
if (policy.removeMode !== "canonical-tombstone-or-snapshot-rebase") {
|
||||
throw consumerError("data_product_consumer_policy_invalid", 500);
|
||||
}
|
||||
@@ -136,10 +168,39 @@ function validatePolicy(policy, product) {
|
||||
productVersion: product.version,
|
||||
staleAfterMs: policy.staleAfterMs,
|
||||
terminalStatuses: [...terminalStatuses],
|
||||
...(statusContract ? { statusContract } : {}),
|
||||
removeMode: policy.removeMode,
|
||||
};
|
||||
}
|
||||
|
||||
function validateStatusContract(value) {
|
||||
const keys = value && typeof value === "object" && !Array.isArray(value)
|
||||
? Object.keys(value).sort()
|
||||
: [];
|
||||
if (JSON.stringify(keys) !== JSON.stringify(["allowedValues", "attribute", "freshness", "missing"])) {
|
||||
throw consumerError("data_product_consumer_policy_invalid", 500);
|
||||
}
|
||||
const attribute = String(value.attribute || "");
|
||||
const allowedValues = Array.isArray(value.allowedValues) ? value.allowedValues : [];
|
||||
if (
|
||||
!/^[A-Za-z][A-Za-z0-9_.-]{0,127}$/.test(attribute)
|
||||
|| allowedValues.length === 0
|
||||
|| allowedValues.length > 32
|
||||
|| new Set(allowedValues).size !== allowedValues.length
|
||||
|| allowedValues.some((item) => typeof item !== "string" || !/^[a-z0-9_-]{1,64}$/.test(item))
|
||||
|| value.missing !== "reject"
|
||||
|| !["none", "observed-at"].includes(value.freshness)
|
||||
) {
|
||||
throw consumerError("data_product_consumer_policy_invalid", 500);
|
||||
}
|
||||
return {
|
||||
attribute,
|
||||
allowedValues: [...allowedValues],
|
||||
missing: "reject",
|
||||
freshness: value.freshness,
|
||||
};
|
||||
}
|
||||
|
||||
function parseSseBlock(block) {
|
||||
const fields = { event: "message", id: "", data: [] };
|
||||
for (const line of block.split("\n")) {
|
||||
@@ -174,6 +235,7 @@ export function createFoundryDataProductConsumerManager({
|
||||
readReaderToken,
|
||||
inspectReaderGrant,
|
||||
ensureReaderGrant,
|
||||
revokeReaderGrant,
|
||||
resolvePolicy,
|
||||
sanitizeSnapshot,
|
||||
sanitizePatch,
|
||||
@@ -240,16 +302,18 @@ export function createFoundryDataProductConsumerManager({
|
||||
}
|
||||
}
|
||||
|
||||
async function catalog(target) {
|
||||
async function catalog(target, { generation = 1, ensureGeneration = generation } = {}) {
|
||||
if (!dataPlaneUrl) throw consumerError("data_product_runtime_not_configured", 503);
|
||||
let product = null;
|
||||
let readerGrantAction = "reuse";
|
||||
let readerGrantGeneration = generation;
|
||||
if (inspectReaderGrant) {
|
||||
const inspection = await inspectReaderGrant(target);
|
||||
const inspection = await inspectReaderGrant(target, { generation });
|
||||
product = inspection?.product || null;
|
||||
readerGrantAction = inspection?.readerGrantAction === "ensure" ? "ensure" : "reuse";
|
||||
readerGrantGeneration = readerGrantAction === "ensure" ? ensureGeneration : generation;
|
||||
} else {
|
||||
const token = await readReaderToken(target.application.id, target.page.id, target.binding.id);
|
||||
const token = await readReaderToken(target.application.id, target.page.id, target.binding.id, generation);
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(new URL("/internal/data-plane/v1/reader/data-products", `${dataPlaneUrl}/`), {
|
||||
@@ -273,13 +337,32 @@ export function createFoundryDataProductConsumerManager({
|
||||
if (target.binding.semanticTypes.some((semanticType) => !product.semanticTypes?.includes(semanticType))) {
|
||||
throw consumerError("data_product_consumer_semantic_scope_mismatch", 409);
|
||||
}
|
||||
return { product, policy: validatePolicy(resolvePolicy(product), product), readerGrantAction };
|
||||
const policy = validatePolicy(resolvePolicy(product), product);
|
||||
if (policy.statusContract && !target.binding.fieldProjection.includes(policy.statusContract.attribute)) {
|
||||
throw consumerError("data_product_consumer_status_field_not_projected", 409);
|
||||
}
|
||||
return {
|
||||
product,
|
||||
policy,
|
||||
readerGrantAction,
|
||||
readerGrantGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
async function plan(input) {
|
||||
const target = await resolveTarget(input.applicationId, input.pageId, input.bindingId);
|
||||
const { product, policy, readerGrantAction } = await catalog(target);
|
||||
const record = await recordFor(target);
|
||||
const currentReaderGrantGeneration = activeReaderGrantGeneration(record.state);
|
||||
const ensureReaderGrantGeneration = record.state
|
||||
? Math.min(2_147_483_647, currentReaderGrantGeneration + 1)
|
||||
: currentReaderGrantGeneration;
|
||||
const { product, policy, readerGrantAction, readerGrantGeneration } = await catalog(target, {
|
||||
generation: currentReaderGrantGeneration,
|
||||
ensureGeneration: ensureReaderGrantGeneration,
|
||||
});
|
||||
if (readerGrantAction === "ensure" && record.state && currentReaderGrantGeneration === 2_147_483_647) {
|
||||
throw consumerError("data_product_reader_grant_generation_exhausted", 409);
|
||||
}
|
||||
const safe = safeTarget(target);
|
||||
const configuration = {
|
||||
target: safe,
|
||||
@@ -287,6 +370,7 @@ export function createFoundryDataProductConsumerManager({
|
||||
policy,
|
||||
readerGrant: "target-scoped-server-only",
|
||||
readerGrantAction,
|
||||
readerGrantGeneration,
|
||||
};
|
||||
const existingMatches = record.state
|
||||
&& JSON.stringify(record.state.target) === JSON.stringify(safe)
|
||||
@@ -294,7 +378,16 @@ export function createFoundryDataProductConsumerManager({
|
||||
&& record.state.product?.version === product.version
|
||||
&& record.state.policy?.id === policy.id
|
||||
&& record.state.policy?.version === policy.version;
|
||||
const action = !record.state ? "create" : existingMatches ? (record.state.enabled ? "refresh" : "resume") : "replace";
|
||||
const predecessorReaderGrantGeneration = Number.isSafeInteger(record.state?.predecessorReaderGrantGeneration)
|
||||
? record.state.predecessorReaderGrantGeneration
|
||||
: null;
|
||||
const action = !record.state
|
||||
? "create"
|
||||
: existingMatches && predecessorReaderGrantGeneration
|
||||
? "finalize-reader-grant-rotation"
|
||||
: existingMatches
|
||||
? (record.state.enabled ? "refresh" : "resume")
|
||||
: "replace";
|
||||
return {
|
||||
schemaVersion: PLAN_SCHEMA_VERSION,
|
||||
planId: planHash({ configuration, action }),
|
||||
@@ -302,16 +395,19 @@ export function createFoundryDataProductConsumerManager({
|
||||
configuration,
|
||||
current: record.state ? { enabled: record.state.enabled === true, cursor: record.state.cursor, product: record.state.product } : null,
|
||||
effects: [
|
||||
...(readerGrantAction === "ensure" ? ["ensure-target-scoped-reader-grant"] : []),
|
||||
...(readerGrantAction === "ensure"
|
||||
? [record.state ? "ensure-successor-target-scoped-reader-grant" : "ensure-target-scoped-reader-grant"]
|
||||
: []),
|
||||
"persist-consumer-state",
|
||||
"bootstrap-scoped-snapshot",
|
||||
...(action === "finalize-reader-grant-rotation" ? [] : ["bootstrap-scoped-snapshot"]),
|
||||
...(predecessorReaderGrantGeneration ? ["revoke-predecessor-reader-grant"] : []),
|
||||
"share-one-upstream-stream-per-active-binding",
|
||||
],
|
||||
destructive: false,
|
||||
};
|
||||
}
|
||||
|
||||
function initialState(target, product, policy) {
|
||||
function initialState(target, product, policy, readerGrantGeneration) {
|
||||
const timestamp = new Date(now()).toISOString();
|
||||
return {
|
||||
schemaVersion: STATE_SCHEMA_VERSION,
|
||||
@@ -321,6 +417,8 @@ export function createFoundryDataProductConsumerManager({
|
||||
runtimeState: "bootstrapping",
|
||||
product: { id: product.id, version: product.version },
|
||||
policy,
|
||||
readerGrantGeneration,
|
||||
predecessorReaderGrantGeneration: null,
|
||||
cursor: "0",
|
||||
snapshotGeneration: 0,
|
||||
subjects: {},
|
||||
@@ -337,7 +435,12 @@ export function createFoundryDataProductConsumerManager({
|
||||
}
|
||||
|
||||
async function fetchSnapshot(record) {
|
||||
const token = await readReaderToken(record.state.target.applicationId, record.state.target.pageId, record.state.target.bindingId);
|
||||
const token = await readReaderToken(
|
||||
record.state.target.applicationId,
|
||||
record.state.target.pageId,
|
||||
record.state.target.bindingId,
|
||||
activeReaderGrantGeneration(record.state),
|
||||
);
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(upstreamUrl(record, "snapshot"), {
|
||||
@@ -410,26 +513,77 @@ export function createFoundryDataProductConsumerManager({
|
||||
const planned = await plan(input);
|
||||
if (input.planId !== planned.planId) throw consumerError("data_product_consumer_plan_mismatch", 409);
|
||||
const target = await resolveTarget(input.applicationId, input.pageId, input.bindingId);
|
||||
const record = await recordFor(target);
|
||||
const plannedGeneration = planned.configuration.readerGrantGeneration;
|
||||
|
||||
if (planned.action === "finalize-reader-grant-rotation") {
|
||||
const predecessorGeneration = record.state?.predecessorReaderGrantGeneration;
|
||||
if (!revokeReaderGrant || !Number.isSafeInteger(predecessorGeneration)) {
|
||||
throw consumerError("data_product_reader_grant_rotation_invalid", 500);
|
||||
}
|
||||
await revokeReaderGrant(record.target, { generation: predecessorGeneration });
|
||||
record.state.predecessorReaderGrantGeneration = null;
|
||||
record.state.timestamps.updatedAt = new Date(now()).toISOString();
|
||||
record.state.lastError = null;
|
||||
await writeState(record);
|
||||
return { plan: planned, consumer: safeStateSummary(record) };
|
||||
}
|
||||
|
||||
if (planned.configuration.readerGrantAction === "ensure") {
|
||||
if (!ensureReaderGrant) throw consumerError("data_product_reader_grant_not_found", 403);
|
||||
await ensureReaderGrant(target);
|
||||
await ensureReaderGrant(target, { generation: plannedGeneration });
|
||||
}
|
||||
const record = await recordFor(target);
|
||||
const { product, policy, readerGrantAction } = await catalog(target);
|
||||
const { product, policy, readerGrantAction, readerGrantGeneration } = await catalog(target, {
|
||||
generation: plannedGeneration,
|
||||
ensureGeneration: plannedGeneration,
|
||||
});
|
||||
if (readerGrantAction !== "reuse") throw consumerError("data_product_reader_grant_not_ready", 409);
|
||||
if (record.stream) stopStream(record);
|
||||
if (!record.state || planned.action === "replace") record.state = initialState(target, product, policy);
|
||||
if (readerGrantGeneration !== plannedGeneration) throw consumerError("data_product_reader_grant_generation_mismatch", 409);
|
||||
|
||||
const previousTarget = record.target;
|
||||
const previousState = record.state ? structuredClone(record.state) : null;
|
||||
const previousGeneration = previousState ? activeReaderGrantGeneration(previousState) : null;
|
||||
if (record.stream) await stopStreamAndWait(record);
|
||||
if (!record.state || planned.action === "replace") {
|
||||
record.target = target;
|
||||
record.state = initialState(target, product, policy, plannedGeneration);
|
||||
if (previousGeneration && previousGeneration !== plannedGeneration) {
|
||||
record.state.predecessorReaderGrantGeneration = previousGeneration;
|
||||
}
|
||||
}
|
||||
else {
|
||||
record.target = target;
|
||||
record.state.target = safeTarget(target);
|
||||
record.state.product = { id: product.id, version: product.version };
|
||||
record.state.policy = policy;
|
||||
record.state.readerGrantGeneration = plannedGeneration;
|
||||
record.state.enabled = true;
|
||||
record.state.runtimeState = "bootstrapping";
|
||||
record.state.lastError = null;
|
||||
}
|
||||
await writeState(record);
|
||||
await bootstrap(record, { notify: false });
|
||||
try {
|
||||
await writeState(record);
|
||||
await bootstrap(record, { notify: false });
|
||||
} catch (error) {
|
||||
if (previousState) {
|
||||
record.target = previousTarget;
|
||||
record.state = previousState;
|
||||
record.state.runtimeState = record.listeners.size ? "connecting" : "idle";
|
||||
record.state.timestamps.updatedAt = new Date(now()).toISOString();
|
||||
await writeState(record);
|
||||
if (record.listeners.size) startStream(record);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const predecessorGeneration = record.state.predecessorReaderGrantGeneration;
|
||||
if (Number.isSafeInteger(predecessorGeneration)) {
|
||||
if (!revokeReaderGrant) throw consumerError("data_product_reader_grant_rotation_not_configured", 503);
|
||||
await revokeReaderGrant(previousTarget, { generation: predecessorGeneration });
|
||||
record.state.predecessorReaderGrantGeneration = null;
|
||||
record.state.timestamps.updatedAt = new Date(now()).toISOString();
|
||||
await writeState(record);
|
||||
}
|
||||
if (record.listeners.size) startStream(record);
|
||||
return { plan: planned, consumer: safeStateSummary(record) };
|
||||
}
|
||||
@@ -526,7 +680,12 @@ export function createFoundryDataProductConsumerManager({
|
||||
}
|
||||
|
||||
async function consumeOnce(record, signal) {
|
||||
const token = await readReaderToken(record.state.target.applicationId, record.state.target.pageId, record.state.target.bindingId);
|
||||
const token = await readReaderToken(
|
||||
record.state.target.applicationId,
|
||||
record.state.target.pageId,
|
||||
record.state.target.bindingId,
|
||||
activeReaderGrantGeneration(record.state),
|
||||
);
|
||||
const url = upstreamUrl(record, "stream", record.state.cursor);
|
||||
const response = openStream
|
||||
? await openStream({ url, token, signal })
|
||||
@@ -625,12 +784,27 @@ export function createFoundryDataProductConsumerManager({
|
||||
record.stream?.controller.abort();
|
||||
}
|
||||
|
||||
async function stopStreamAndWait(record) {
|
||||
const pending = record.stream?.promise;
|
||||
stopStream(record);
|
||||
if (!pending) return;
|
||||
await Promise.race([
|
||||
pending.catch(() => undefined),
|
||||
new Promise((resolve) => setTimeout(resolve, 750)),
|
||||
]);
|
||||
}
|
||||
|
||||
async function snapshot(input) {
|
||||
const target = input?.binding ? input : await resolveTarget(input.applicationId, input.pageId, input.bindingId);
|
||||
const record = await ensureProvisioned(target);
|
||||
// A removed target grant revokes browser reads immediately even though the
|
||||
// last safe snapshot remains persisted for rollback/diagnostics.
|
||||
await readReaderToken(record.state.target.applicationId, record.state.target.pageId, record.state.target.bindingId);
|
||||
await readReaderToken(
|
||||
record.state.target.applicationId,
|
||||
record.state.target.pageId,
|
||||
record.state.target.bindingId,
|
||||
activeReaderGrantGeneration(record.state),
|
||||
);
|
||||
await refreshPresentation(record);
|
||||
return {
|
||||
schemaVersion: "nodedc.data-product.snapshot/v1",
|
||||
|
||||
Reference in New Issue
Block a user