Initial import NDC_1C
This commit is contained in:
@@ -0,0 +1,479 @@
|
||||
import type {
|
||||
RetrievalConfidence,
|
||||
RetrievalResultStatus,
|
||||
RetrievalResultType,
|
||||
UnifiedRetrievalResult
|
||||
} from "../types/assistant";
|
||||
import { FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1 } from "../config";
|
||||
import { EVIDENCE_SOURCE_REF_SCHEMA_VERSION } from "../types/stage1Contracts";
|
||||
import type {
|
||||
EvidenceConfidence,
|
||||
EvidenceItem,
|
||||
EvidenceLimitationReasonCode,
|
||||
EvidenceKind,
|
||||
EvidencePointer,
|
||||
EvidenceSourceRef
|
||||
} from "../types/stage1Contracts";
|
||||
|
||||
interface RawRetrievalResult {
|
||||
status?: string;
|
||||
result_type?: string;
|
||||
items?: unknown;
|
||||
summary?: unknown;
|
||||
evidence?: unknown;
|
||||
why_included?: unknown;
|
||||
selection_reason?: unknown;
|
||||
risk_factors?: unknown;
|
||||
business_interpretation?: unknown;
|
||||
confidence?: unknown;
|
||||
limitations?: unknown;
|
||||
errors?: unknown;
|
||||
}
|
||||
|
||||
function toObject(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function toStringOrNull(value: unknown): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function toNumberOrNull(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeStatus(value: string | undefined): RetrievalResultStatus {
|
||||
if (value === "ok" || value === "empty" || value === "partial" || value === "error") {
|
||||
return value;
|
||||
}
|
||||
return "error";
|
||||
}
|
||||
|
||||
function normalizeResultType(value: string | undefined): RetrievalResultType {
|
||||
if (value === "list" || value === "summary" || value === "object" || value === "chain" || value === "ranking") {
|
||||
return value;
|
||||
}
|
||||
return "summary";
|
||||
}
|
||||
|
||||
function normalizeObjectArray(value: unknown): Array<Record<string, unknown>> {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value
|
||||
.map((item) => (item && typeof item === "object" ? (item as Record<string, unknown>) : null))
|
||||
.filter((item): item is Record<string, unknown> => item !== null);
|
||||
}
|
||||
|
||||
function normalizeSummary(value: unknown): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return {};
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function normalizeErrors(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.map((item) => String(item));
|
||||
}
|
||||
|
||||
function normalizeStringArray(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.map((item) => String(item));
|
||||
}
|
||||
|
||||
function normalizeConfidence(value: unknown): RetrievalConfidence {
|
||||
if (value === "high" || value === "medium" || value === "low") {
|
||||
return value;
|
||||
}
|
||||
return "medium";
|
||||
}
|
||||
|
||||
function parseEvidenceConfidence(value: unknown): EvidenceConfidence | null {
|
||||
if (value === "high" || value === "medium" || value === "low") {
|
||||
return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeEvidenceNamespace(value: unknown): EvidencePointer["source"]["namespace"] {
|
||||
const normalized = toStringOrNull(value)?.toLowerCase();
|
||||
if (!normalized) return "unknown";
|
||||
if (normalized === "snapshot_2020" || normalized === "snapshot") return "snapshot_2020";
|
||||
if (normalized === "assistant_derived" || normalized === "derived") return "assistant_derived";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function inferEvidenceKind(item: Record<string, unknown>): EvidenceKind {
|
||||
if (item.mechanism_of_failure !== undefined || item.failed_expected_edge !== undefined || item.expected_next_step !== undefined) {
|
||||
return "mechanism_link";
|
||||
}
|
||||
if (item.risk_score !== undefined || item.zero_guid_values !== undefined || item.unknown_link_count !== undefined) {
|
||||
return "anomaly_signal";
|
||||
}
|
||||
if (item.records_count !== undefined || item.operations_count !== undefined || item.document_refs_count !== undefined) {
|
||||
return "aggregation";
|
||||
}
|
||||
if (item.limitation !== undefined || item.is_snapshot_limited !== undefined) {
|
||||
return "limitation_note";
|
||||
}
|
||||
return "factual_anchor";
|
||||
}
|
||||
|
||||
function inferMechanismNoteLegacy(kind: EvidenceKind, item: Record<string, unknown>): string {
|
||||
const explicit = toStringOrNull(item.mechanism_note);
|
||||
if (explicit) {
|
||||
return explicit;
|
||||
}
|
||||
if (kind === "mechanism_link") {
|
||||
const failure = toStringOrNull(item.mechanism_of_failure);
|
||||
if (failure) return failure;
|
||||
return "Mechanism link inferred from retrieval evidence.";
|
||||
}
|
||||
if (kind === "anomaly_signal") {
|
||||
return "Anomaly signal inferred from risk-oriented fields.";
|
||||
}
|
||||
if (kind === "aggregation") {
|
||||
return "Aggregated evidence item.";
|
||||
}
|
||||
if (kind === "limitation_note") {
|
||||
return "Evidence includes explicit limitation hints.";
|
||||
}
|
||||
return "Factual evidence anchor.";
|
||||
}
|
||||
|
||||
interface MechanismNoteResolution {
|
||||
note: string | null;
|
||||
reliable: boolean;
|
||||
}
|
||||
|
||||
function resolveMechanismNote(kind: EvidenceKind, item: Record<string, unknown>): MechanismNoteResolution {
|
||||
const explicit = toStringOrNull(item.mechanism_note);
|
||||
if (explicit) {
|
||||
return {
|
||||
note: explicit,
|
||||
reliable: true
|
||||
};
|
||||
}
|
||||
|
||||
if (kind === "mechanism_link") {
|
||||
const failure = toStringOrNull(item.mechanism_of_failure);
|
||||
if (failure) {
|
||||
return {
|
||||
note: failure,
|
||||
reliable: true
|
||||
};
|
||||
}
|
||||
const failedEdge = toStringOrNull(item.failed_expected_edge);
|
||||
const expectedNext = toStringOrNull(item.expected_next_step);
|
||||
const composed = [failedEdge ? `failed_edge=${failedEdge}` : null, expectedNext ? `expected_next_step=${expectedNext}` : null]
|
||||
.filter((part): part is string => Boolean(part))
|
||||
.join("; ");
|
||||
if (composed) {
|
||||
return {
|
||||
note: composed,
|
||||
reliable: true
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1) {
|
||||
return {
|
||||
note: inferMechanismNoteLegacy(kind, item),
|
||||
reliable: false
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
note: null,
|
||||
reliable: false
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeEvidenceSourceType(value: unknown, record: Record<string, unknown>): EvidenceItem["source_type"] {
|
||||
const normalized = toStringOrNull(value);
|
||||
if (normalized === "retrieval_item" || normalized === "retrieval_summary" || normalized === "derived") {
|
||||
return normalized;
|
||||
}
|
||||
if (record.records_count !== undefined || record.operations_count !== undefined || record.document_refs_count !== undefined) {
|
||||
return "retrieval_summary";
|
||||
}
|
||||
return "retrieval_item";
|
||||
}
|
||||
|
||||
function readPointer(record: Record<string, unknown>): Record<string, unknown> {
|
||||
const pointer = toObject(record.pointer);
|
||||
return pointer ?? {};
|
||||
}
|
||||
|
||||
interface NormalizedPointerResult {
|
||||
pointer: EvidencePointer;
|
||||
fallback_source_namespace: boolean;
|
||||
fallback_source_entity: boolean;
|
||||
fallback_source_id: boolean;
|
||||
}
|
||||
|
||||
function normalizeEvidencePointer(
|
||||
fragmentId: string,
|
||||
route: string,
|
||||
record: Record<string, unknown>,
|
||||
index: number
|
||||
): NormalizedPointerResult {
|
||||
const pointer = readPointer(record);
|
||||
const source = toObject(pointer.source);
|
||||
const locator = toObject(pointer.locator);
|
||||
|
||||
const sourceEntityCandidate = toStringOrNull(source?.entity) ?? toStringOrNull(record.source_entity);
|
||||
const sourceEntity = sourceEntityCandidate ?? "unknown_entity";
|
||||
const sourceIdCandidate = toStringOrNull(source?.id) ?? toStringOrNull(record.source_id);
|
||||
const sourceId = sourceIdCandidate ?? `${route}:${fragmentId}:${index + 1}`;
|
||||
const period = toStringOrNull(source?.period) ?? toStringOrNull(record.period);
|
||||
const namespace = normalizeEvidenceNamespace(source?.namespace ?? record.source_namespace);
|
||||
|
||||
return {
|
||||
pointer: {
|
||||
fragment_id: toStringOrNull(pointer.fragment_id) ?? fragmentId,
|
||||
route: toStringOrNull(pointer.route) ?? route,
|
||||
source: {
|
||||
namespace,
|
||||
entity: sourceEntity,
|
||||
id: sourceId,
|
||||
period
|
||||
},
|
||||
locator: {
|
||||
field_path: toStringOrNull(locator?.field_path) ?? toStringOrNull(record.field_path),
|
||||
item_index: toNumberOrNull(locator?.item_index) ?? index
|
||||
}
|
||||
},
|
||||
fallback_source_namespace: namespace === "unknown",
|
||||
fallback_source_entity: sourceEntityCandidate === null,
|
||||
fallback_source_id: sourceIdCandidate === null
|
||||
};
|
||||
}
|
||||
|
||||
function canonicalizeSourceRefPart(value: string | null): string {
|
||||
return encodeURIComponent((value ?? "none").trim().toLowerCase());
|
||||
}
|
||||
|
||||
function buildSourceRef(pointer: EvidencePointer): EvidenceSourceRef {
|
||||
return {
|
||||
schema_version: EVIDENCE_SOURCE_REF_SCHEMA_VERSION,
|
||||
namespace: pointer.source.namespace,
|
||||
entity: pointer.source.entity,
|
||||
id: pointer.source.id,
|
||||
period: pointer.source.period,
|
||||
canonical_ref: [
|
||||
EVIDENCE_SOURCE_REF_SCHEMA_VERSION,
|
||||
canonicalizeSourceRefPart(pointer.source.namespace),
|
||||
canonicalizeSourceRefPart(pointer.source.entity),
|
||||
canonicalizeSourceRefPart(pointer.source.id),
|
||||
canonicalizeSourceRefPart(pointer.source.period)
|
||||
].join("|")
|
||||
};
|
||||
}
|
||||
|
||||
function toBoolean(value: unknown): boolean {
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "number") return value !== 0;
|
||||
if (typeof value === "string") {
|
||||
const lowered = value.trim().toLowerCase();
|
||||
return lowered === "true" || lowered === "1" || lowered === "yes";
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function limitationCodeFromText(text: string): EvidenceLimitationReasonCode {
|
||||
const lower = text.toLowerCase();
|
||||
if (/(snapshot|read-only|read only)/i.test(lower)) {
|
||||
return "snapshot_only";
|
||||
}
|
||||
if (/heuristic/i.test(lower)) {
|
||||
return "heuristic_inference";
|
||||
}
|
||||
if (/mechanism/i.test(lower)) {
|
||||
return "missing_mechanism";
|
||||
}
|
||||
if (/(guid|detail|specific)/i.test(lower)) {
|
||||
return "insufficient_detail";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
interface LimitationResolutionInput {
|
||||
record: Record<string, unknown>;
|
||||
sourceType: EvidenceItem["source_type"];
|
||||
evidenceKind: EvidenceKind;
|
||||
mechanismReliable: boolean;
|
||||
mechanismExpected: boolean;
|
||||
pointerWeak: boolean;
|
||||
}
|
||||
|
||||
function resolveEvidenceLimitation(input: LimitationResolutionInput): EvidenceItem["limitation"] {
|
||||
const explicitLimitation = toStringOrNull(input.record.limitation);
|
||||
if (explicitLimitation) {
|
||||
return {
|
||||
reason_code: limitationCodeFromText(explicitLimitation),
|
||||
note: explicitLimitation
|
||||
};
|
||||
}
|
||||
if (toBoolean(input.record.is_snapshot_limited)) {
|
||||
return {
|
||||
reason_code: "snapshot_only",
|
||||
note: null
|
||||
};
|
||||
}
|
||||
if (!FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1) {
|
||||
return null;
|
||||
}
|
||||
if (input.mechanismExpected && !input.mechanismReliable) {
|
||||
return {
|
||||
reason_code: "missing_mechanism",
|
||||
note: null
|
||||
};
|
||||
}
|
||||
if (input.pointerWeak) {
|
||||
return {
|
||||
reason_code: "weak_source_mapping",
|
||||
note: null
|
||||
};
|
||||
}
|
||||
if (input.sourceType === "derived") {
|
||||
return {
|
||||
reason_code: "heuristic_inference",
|
||||
note: null
|
||||
};
|
||||
}
|
||||
if (input.evidenceKind === "limitation_note") {
|
||||
return {
|
||||
reason_code: "unknown",
|
||||
note: null
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function downgradeConfidence(value: EvidenceConfidence): EvidenceConfidence {
|
||||
if (value === "high") return "medium";
|
||||
if (value === "medium") return "low";
|
||||
return "low";
|
||||
}
|
||||
|
||||
interface ConfidenceResolutionInput {
|
||||
explicitConfidence: EvidenceConfidence | null;
|
||||
sourceType: EvidenceItem["source_type"];
|
||||
mechanismReliable: boolean;
|
||||
mechanismExpected: boolean;
|
||||
limitation: EvidenceItem["limitation"];
|
||||
pointerWeak: boolean;
|
||||
}
|
||||
|
||||
function resolveEvidenceConfidence(input: ConfidenceResolutionInput): EvidenceConfidence {
|
||||
if (!FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1) {
|
||||
return input.explicitConfidence ?? "medium";
|
||||
}
|
||||
|
||||
let confidence: EvidenceConfidence = input.explicitConfidence ?? (input.sourceType === "retrieval_item" ? "medium" : "low");
|
||||
|
||||
if (input.limitation?.reason_code === "missing_mechanism" || input.limitation?.reason_code === "weak_source_mapping") {
|
||||
confidence = downgradeConfidence(confidence);
|
||||
}
|
||||
if (input.sourceType === "derived" && !input.explicitConfidence) {
|
||||
confidence = "low";
|
||||
}
|
||||
if (input.mechanismExpected && !input.mechanismReliable) {
|
||||
confidence = "low";
|
||||
}
|
||||
if (input.pointerWeak) {
|
||||
confidence = "low";
|
||||
}
|
||||
|
||||
return confidence;
|
||||
}
|
||||
|
||||
function normalizeEvidenceItems(
|
||||
fragmentId: string,
|
||||
requirementIds: string[],
|
||||
route: string,
|
||||
value: unknown
|
||||
): EvidenceItem[] {
|
||||
const records = normalizeObjectArray(value);
|
||||
return records.map((record, index) => {
|
||||
const evidenceId = toStringOrNull(record.evidence_id) ?? `ev-${fragmentId}-${index + 1}`;
|
||||
const claimRef =
|
||||
toStringOrNull(record.claim_ref) ??
|
||||
(requirementIds[0] ? `requirement:${requirementIds[0]}` : `fragment:${fragmentId}`);
|
||||
const evidenceKind = inferEvidenceKind(record);
|
||||
const sourceType = normalizeEvidenceSourceType(record.source_type, record);
|
||||
const pointerResult = normalizeEvidencePointer(fragmentId, route, record, index);
|
||||
const mechanism = resolveMechanismNote(evidenceKind, record);
|
||||
const mechanismExpected = evidenceKind === "mechanism_link" || evidenceKind === "anomaly_signal" || evidenceKind === "aggregation";
|
||||
const pointerWeak =
|
||||
pointerResult.fallback_source_namespace || pointerResult.fallback_source_entity || pointerResult.fallback_source_id;
|
||||
const limitation = resolveEvidenceLimitation({
|
||||
record,
|
||||
sourceType,
|
||||
evidenceKind,
|
||||
mechanismReliable: mechanism.reliable,
|
||||
mechanismExpected,
|
||||
pointerWeak
|
||||
});
|
||||
const confidence = resolveEvidenceConfidence({
|
||||
explicitConfidence: parseEvidenceConfidence(record.confidence),
|
||||
sourceType,
|
||||
mechanismReliable: mechanism.reliable,
|
||||
mechanismExpected,
|
||||
limitation,
|
||||
pointerWeak
|
||||
});
|
||||
|
||||
return {
|
||||
evidence_id: evidenceId,
|
||||
claim_ref: claimRef,
|
||||
source_type: sourceType,
|
||||
source_ref: buildSourceRef(pointerResult.pointer),
|
||||
pointer: pointerResult.pointer,
|
||||
evidence_kind: evidenceKind,
|
||||
mechanism_note: mechanism.note,
|
||||
confidence,
|
||||
limitation,
|
||||
payload: record
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeRetrievalResult(
|
||||
fragmentId: string,
|
||||
requirementIds: string[],
|
||||
route: string,
|
||||
raw: RawRetrievalResult
|
||||
): UnifiedRetrievalResult {
|
||||
return {
|
||||
fragment_id: fragmentId,
|
||||
requirement_ids: requirementIds,
|
||||
route,
|
||||
status: normalizeStatus(raw.status),
|
||||
result_type: normalizeResultType(raw.result_type),
|
||||
items: normalizeObjectArray(raw.items),
|
||||
summary: normalizeSummary(raw.summary),
|
||||
evidence: normalizeEvidenceItems(fragmentId, requirementIds, route, raw.evidence),
|
||||
why_included: normalizeStringArray(raw.why_included),
|
||||
selection_reason: normalizeStringArray(raw.selection_reason),
|
||||
risk_factors: normalizeStringArray(raw.risk_factors),
|
||||
business_interpretation: normalizeStringArray(raw.business_interpretation),
|
||||
confidence: normalizeConfidence(raw.confidence),
|
||||
limitations: normalizeStringArray(raw.limitations),
|
||||
errors: normalizeErrors(raw.errors)
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user