Stage 2 завершён: problem-first ответы и follow-up continuity - ассистент переведён от entity-heavy логики к problem-first ответам с problem-unit слоем, удержанием контекста в follow-up и очисткой пользовательского ответа от сырых технических ссылок.

2026-03-26 14:53:52 +03:00
parent ece1abed76
commit 96353cfd48
2474 changed files with 21678 additions and 3292445 deletions
@@ -0,0 +1,592 @@
+import type { EvidenceItem } from "../types/stage1Contracts";
+import type {
+  CandidateEvidenceItem,
+  ProblemConfidence,
+  ProblemScore,
+  ProblemUnit,
+  ProblemUnitEntityBacklink,
+  ProblemUnitSummary,
+  ProblemUnitType
+} from "../types/stage2ProblemUnits";
+import {
+  CANDIDATE_EVIDENCE_SCHEMA_VERSION,
+  PROBLEM_UNIT_SCHEMA_VERSION,
+  PROBLEM_UNIT_SUMMARY_SCHEMA_VERSION
+} from "../types/stage2ProblemUnits";
+
+type RetrievalResultType = "list" | "summary" | "object" | "chain" | "ranking";
+
+interface AssembleProblemUnitsInput {
+  route: string;
+  result_type?: RetrievalResultType;
+  evidence: EvidenceItem[];
+  raw_entities?: Array<Record<string, unknown>>;
+  summary?: Record<string, unknown>;
+  risk_factors?: string[];
+  selection_reason?: string[];
+  business_interpretation?: string[];
+}
+
+interface CandidateCluster {
+  cluster_id: string;
+  candidates: CandidateEvidenceItem[];
+}
+
+interface SeverityResult {
+  severity: ProblemScore;
+  confidence: ProblemConfidence;
+}
+
+interface CandidateBuildContext {
+  route: string;
+  result_type?: RetrievalResultType;
+  raw_entities: Array<Record<string, unknown>>;
+  summary_relation_patterns: string[];
+  summary_anomaly_patterns: string[];
+  risk_factors: string[];
+}
+
+function toObject(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return null;
+  }
+  return value as Record<string, unknown>;
+}
+
+function uniqueStrings(values: string[]): string[] {
+  return Array.from(new Set(values.map((item) => item.trim()).filter(Boolean)));
+}
+
+function clampUnitScore(value: number): number {
+  if (!Number.isFinite(value)) {
+    return 0;
+  }
+  if (value <= 0) return 0;
+  if (value >= 1) return 1;
+  return Number(value.toFixed(2));
+}
+
+function gradeForScore(score: number): "low" | "medium" | "high" {
+  if (score >= 0.7) return "high";
+  if (score >= 0.4) return "medium";
+  return "low";
+}
+
+function confidenceGradeForScore(score: number): "low" | "medium" | "high" {
+  if (score >= 0.75) return "high";
+  if (score >= 0.45) return "medium";
+  return "low";
+}
+
+function confidenceToScore(value: CandidateEvidenceItem["confidence_hint"]): number {
+  if (value === "high") return 1;
+  if (value === "medium") return 0.6;
+  return 0.3;
+}
+
+function valueFromPayload(item: EvidenceItem, key: string): string | null {
+  const value = item.payload[key];
+  if (typeof value !== "string") {
+    return null;
+  }
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function stringArrayFromUnknown(value: unknown): string[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return uniqueStrings(value.map((entry) => String(entry)));
+}
+
+function stringArrayFromPayload(item: EvidenceItem, key: string): string[] {
+  return stringArrayFromUnknown(item.payload[key]);
+}
+
+function extractSemanticProfile(summary: Record<string, unknown>): {
+  relation_patterns: string[];
+  anomaly_patterns: string[];
+} {
+  const semanticProfile = toObject(summary.semantic_profile);
+  return {
+    relation_patterns: stringArrayFromUnknown(semanticProfile?.relation_patterns),
+    anomaly_patterns: stringArrayFromUnknown(semanticProfile?.anomaly_patterns)
+  };
+}
+
+function resolveEntityOverlay(item: EvidenceItem, rawEntities: Array<Record<string, unknown>>): Record<string, unknown> | null {
+  const sourceId = String(item.pointer.source.id ?? "").toLowerCase();
+  if (!sourceId) {
+    return null;
+  }
+  for (const entity of rawEntities) {
+    const candidates = [
+      String(entity.source_id ?? ""),
+      String(entity.entity_id ?? ""),
+      String(entity.id ?? "")
+    ]
+      .map((entry) => entry.toLowerCase())
+      .filter(Boolean);
+    if (candidates.includes(sourceId)) {
+      return entity;
+    }
+  }
+  return null;
+}
+
+function detectRelationPatternHits(
+  item: EvidenceItem,
+  overlay: Record<string, unknown> | null,
+  context: CandidateBuildContext
+): string[] {
+  const hits: string[] = [];
+  const failedEdge = valueFromPayload(item, "failed_expected_edge");
+  if (failedEdge) {
+    hits.push(`failed_edge:${failedEdge}`);
+  }
+  const expectedStep = valueFromPayload(item, "expected_next_step");
+  if (expectedStep) {
+    hits.push(`expected_step:${expectedStep}`);
+  }
+  const relationPattern = valueFromPayload(item, "relation_pattern");
+  if (relationPattern) {
+    hits.push(relationPattern);
+  }
+
+  hits.push(...stringArrayFromPayload(item, "relation_patterns"));
+  hits.push(...stringArrayFromPayload(item, "relation_pattern_hits"));
+
+  if (overlay) {
+    hits.push(...stringArrayFromUnknown(overlay.relation_pattern_hits));
+    hits.push(...stringArrayFromUnknown(overlay.relation_types));
+  }
+
+  hits.push(...context.summary_relation_patterns);
+  if (context.route === "hybrid_store_plus_live" && hits.length === 0) {
+    hits.push("chain_scope");
+  }
+
+  return uniqueStrings(hits);
+}
+
+function detectAnomalyPatterns(
+  item: EvidenceItem,
+  overlay: Record<string, unknown> | null,
+  context: CandidateBuildContext
+): string[] {
+  const patterns: string[] = [];
+  patterns.push(...stringArrayFromPayload(item, "anomaly_patterns"));
+  patterns.push(...stringArrayFromPayload(item, "risk_factors"));
+  patterns.push(...stringArrayFromPayload(item, "lifecycle_gaps"));
+  patterns.push(...stringArrayFromPayload(item, "lifecycle_markers"));
+
+  const explicit = valueFromPayload(item, "anomaly_pattern");
+  if (explicit) {
+    patterns.push(explicit);
+  }
+
+  if (overlay) {
+    patterns.push(...stringArrayFromUnknown(overlay.risk_factors));
+    patterns.push(...stringArrayFromUnknown(overlay.lifecycle_gaps));
+    patterns.push(...stringArrayFromUnknown(overlay.anomaly_patterns));
+  }
+
+  patterns.push(...context.summary_anomaly_patterns);
+  patterns.push(...context.risk_factors);
+
+  if (item.evidence_kind === "anomaly_signal") {
+    patterns.push("anomaly_signal");
+  }
+  if (item.limitation?.reason_code === "missing_mechanism") {
+    patterns.push("missing_mechanism");
+  }
+  if (item.limitation?.reason_code === "insufficient_detail") {
+    patterns.push("insufficient_detail");
+  }
+
+  const defectClass = valueFromPayload(item, "business_defect_class");
+  if (defectClass) {
+    patterns.push(defectClass);
+  }
+
+  if (context.route === "store_feature_risk") {
+    patterns.push("risk_route");
+  }
+
+  return uniqueStrings(patterns);
+}
+
+function buildEntityBacklinks(item: EvidenceItem, overlay: Record<string, unknown> | null): ProblemUnitEntityBacklink[] {
+  const backlinks: ProblemUnitEntityBacklink[] = [];
+  const sourceEntity = String(item.pointer.source.entity ?? "").trim();
+  const sourceId = String(item.pointer.source.id ?? "").trim();
+  if (sourceEntity && sourceId) {
+    backlinks.push({
+      entity: sourceEntity,
+      id: sourceId
+    });
+  }
+
+  const payloadEntity = valueFromPayload(item, "source_entity");
+  const payloadId = valueFromPayload(item, "source_id");
+  if (payloadEntity && payloadId) {
+    backlinks.push({
+      entity: payloadEntity,
+      id: payloadId
+    });
+  }
+
+  const overlayEntity = overlay ? String(overlay.source_entity ?? "").trim() : "";
+  const overlayId = overlay ? String(overlay.source_id ?? "").trim() : "";
+  if (overlayEntity && overlayId) {
+    backlinks.push({
+      entity: overlayEntity,
+      id: overlayId
+    });
+  }
+
+  return Array.from(
+    new Map(backlinks.map((entry) => [`${entry.entity.toLowerCase()}|${entry.id.toLowerCase()}`, entry])).values()
+  );
+}
+
+function inferExpectedState(item: EvidenceItem): string | undefined {
+  return valueFromPayload(item, "expected_state") ?? valueFromPayload(item, "expected_next_step") ?? undefined;
+}
+
+function inferActualState(item: EvidenceItem): string | undefined {
+  return valueFromPayload(item, "actual_state") ?? valueFromPayload(item, "mechanism_of_failure") ?? undefined;
+}
+
+export function buildCandidateEvidence(
+  items: EvidenceItem[],
+  route: string,
+  contextInput?: Partial<CandidateBuildContext>
+): CandidateEvidenceItem[] {
+  const context: CandidateBuildContext = {
+    route,
+    result_type: contextInput?.result_type,
+    raw_entities: contextInput?.raw_entities ?? [],
+    summary_relation_patterns: contextInput?.summary_relation_patterns ?? [],
+    summary_anomaly_patterns: contextInput?.summary_anomaly_patterns ?? [],
+    risk_factors: contextInput?.risk_factors ?? []
+  };
+
+  return items.map((item, index) => {
+    const overlay = resolveEntityOverlay(item, context.raw_entities);
+    return {
+      schema_version: CANDIDATE_EVIDENCE_SCHEMA_VERSION,
+      candidate_id: `cand-${item.evidence_id || `${route}-${index + 1}`}`,
+      route,
+      source_ref: item.source_ref,
+      expected_state: inferExpectedState(item),
+      actual_state: inferActualState(item),
+      relation_pattern_hits: detectRelationPatternHits(item, overlay, context),
+      anomaly_patterns: detectAnomalyPatterns(item, overlay, context),
+      entity_backlinks: buildEntityBacklinks(item, overlay),
+      confidence_hint: item.confidence
+    };
+  });
+}
+
+function clusterSignature(candidate: CandidateEvidenceItem): string {
+  const relation = candidate.relation_pattern_hits[0] ?? "none";
+  const anomaly = candidate.anomaly_patterns[0] ?? "none";
+  return [candidate.route, candidate.source_ref.canonical_ref, relation, anomaly].join("|");
+}
+
+export function clusterCandidateEvidence(candidates: CandidateEvidenceItem[]): CandidateCluster[] {
+  const byCluster = new Map<string, CandidateEvidenceItem[]>();
+  for (const candidate of candidates) {
+    const signature = clusterSignature(candidate);
+    const current = byCluster.get(signature) ?? [];
+    current.push(candidate);
+    byCluster.set(signature, current);
+  }
+  return Array.from(byCluster.entries()).map(([cluster_id, clusterCandidates]) => ({
+    cluster_id,
+    candidates: clusterCandidates
+  }));
+}
+
+function hasAny(value: string, pattern: RegExp): boolean {
+  return pattern.test(value);
+}
+
+export function detectProblemUnitType(cluster: CandidateCluster): ProblemUnitType {
+  const relationText = cluster.candidates.flatMap((item) => item.relation_pattern_hits).join(" ").toLowerCase();
+  const anomalyText = cluster.candidates.flatMap((item) => item.anomaly_patterns).join(" ").toLowerCase();
+  const sourceText = cluster.candidates
+    .map((item) => `${item.source_ref.entity} ${item.source_ref.id}`)
+    .join(" ")
+    .toLowerCase();
+  const routeText = cluster.candidates.map((item) => item.route).join(" ").toLowerCase();
+
+  if (hasAny(`${anomalyText} ${sourceText}`, /cross[_\s-]?branch|vat|nds|tax|ндс/)) {
+    return "cross_branch_inconsistency_cluster";
+  }
+  if (hasAny(`${anomalyText} ${relationText}`, /period|close[_\s-]?risk|reporting|закрыт|period_close/)) {
+    return "period_risk_cluster";
+  }
+  if (hasAny(anomalyText, /settlement|tail|unresolved|хвост|незакрыт/)) {
+    return "unresolved_settlement_cluster";
+  }
+  if (hasAny(anomalyText, /lifecycle|deferred|broken_lifecycle|списани|амортиз|рбп/)) {
+    return "lifecycle_anomaly_node";
+  }
+  if (
+    hasAny(relationText, /failed_edge|statement_to_document|payment_to_settlement|chain|broken|цепоч|разрыв/)
+    || (routeText.includes("hybrid_store_plus_live") && relationText.length > 0)
+  ) {
+    return "broken_chain_segment";
+  }
+  return "document_conflict";
+}
+
+export function scoreProblemSeverity(cluster: CandidateCluster): SeverityResult {
+  const candidates = cluster.candidates;
+  const averageConfidence =
+    candidates.length > 0
+      ? candidates.reduce((acc, item) => acc + confidenceToScore(item.confidence_hint), 0) / candidates.length
+      : 0;
+  const hasEdgeBreak = candidates.some((item) =>
+    item.relation_pattern_hits.some((pattern) => /failed_edge|chain|statement_to_document|payment_to_settlement/i.test(pattern))
+  );
+  const hasAnomaly = candidates.some((item) => item.anomaly_patterns.length > 0);
+  const hasPeriodRisk = candidates.some((item) => item.anomaly_patterns.some((pattern) => /period|close|reporting|закрыт/i.test(pattern)));
+  const candidateBoost = Math.min(candidates.length, 5) * 0.08;
+
+  let severityScore = 0.25 + candidateBoost;
+  if (hasEdgeBreak) severityScore += 0.2;
+  if (hasAnomaly) severityScore += 0.15;
+  if (hasPeriodRisk) severityScore += 0.1;
+  const normalizedSeverity = clampUnitScore(severityScore);
+
+  let confidenceScore = averageConfidence;
+  if (hasEdgeBreak) confidenceScore += 0.05;
+  const normalizedConfidence = clampUnitScore(confidenceScore);
+
+  return {
+    severity: {
+      score: normalizedSeverity,
+      grade: gradeForScore(normalizedSeverity)
+    },
+    confidence: {
+      score: normalizedConfidence,
+      grade: confidenceGradeForScore(normalizedConfidence)
+    }
+  };
+}
+
+function unitTitle(type: ProblemUnitType): string {
+  if (type === "document_conflict") return "Document conflict detected";
+  if (type === "broken_chain_segment") return "Broken chain segment detected";
+  if (type === "lifecycle_anomaly_node") return "Lifecycle anomaly node detected";
+  if (type === "unresolved_settlement_cluster") return "Unresolved settlement cluster detected";
+  if (type === "period_risk_cluster") return "Period risk cluster detected";
+  return "Cross-branch inconsistency cluster detected";
+}
+
+function mechanismSummary(cluster: CandidateCluster, type: ProblemUnitType): string {
+  const relationHints = uniqueStrings(cluster.candidates.flatMap((item) => item.relation_pattern_hits));
+  const anomalyHints = uniqueStrings(cluster.candidates.flatMap((item) => item.anomaly_patterns));
+  const primaryRelation = relationHints[0];
+  const primaryAnomaly = anomalyHints[0];
+
+  if (primaryRelation) {
+    return `Mechanism candidate: ${primaryRelation}.`;
+  }
+  if (primaryAnomaly) {
+    return `Mechanism inferred from anomaly pattern: ${primaryAnomaly}.`;
+  }
+  return `Mechanism is currently inferred at baseline level for ${type}.`;
+}
+
+function businessDefectClass(cluster: CandidateCluster, type: ProblemUnitType): string {
+  const patterns = uniqueStrings([
+    ...cluster.candidates.flatMap((item) => item.relation_pattern_hits),
+    ...cluster.candidates.flatMap((item) => item.anomaly_patterns)
+  ]);
+  return patterns[0] ?? type;
+}
+
+function collectAffectedByEntity(backlinks: ProblemUnitEntityBacklink[], pattern: RegExp): string[] {
+  return uniqueStrings(
+    backlinks.filter((entry) => pattern.test(entry.entity)).map((entry) => `${entry.entity}:${entry.id}`)
+  );
+}
+
+function parseFailedExpectedEdge(cluster: CandidateCluster): string | undefined {
+  const withEdge = cluster.candidates.flatMap((item) => item.relation_pattern_hits).find((pattern) => pattern.startsWith("failed_edge:"));
+  if (!withEdge) {
+    return undefined;
+  }
+  return withEdge.replace(/^failed_edge:/, "").trim() || undefined;
+}
+
+function mergeBacklinks(candidates: CandidateEvidenceItem[]): ProblemUnitEntityBacklink[] {
+  return Array.from(
+    new Map(
+      candidates
+        .flatMap((item) => item.entity_backlinks)
+        .map((entry) => [`${entry.entity.toLowerCase()}|${entry.id.toLowerCase()}`, entry] as const)
+    ).values()
+  );
+}
+
+export function buildProblemUnit(cluster: CandidateCluster, index: number): ProblemUnit {
+  const type = detectProblemUnitType(cluster);
+  const scored = scoreProblemSeverity(cluster);
+  const backlinks = mergeBacklinks(cluster.candidates);
+  const expectedState = cluster.candidates.find((item) => typeof item.expected_state === "string")?.expected_state;
+  const actualState = cluster.candidates.find((item) => typeof item.actual_state === "string")?.actual_state;
+  const failedExpectedEdge = parseFailedExpectedEdge(cluster);
+  const periodSensitive = cluster.candidates.some((item) =>
+    item.anomaly_patterns.some((pattern) => /period|close|reporting|закрыт/i.test(pattern))
+  );
+  const hasLowConfidence = cluster.candidates.some((item) => item.confidence_hint === "low");
+
+  return {
+    schema_version: PROBLEM_UNIT_SCHEMA_VERSION,
+    problem_unit_id: `pu-${type}-${index + 1}`,
+    problem_unit_type: type,
+    title: unitTitle(type),
+    mechanism_summary: mechanismSummary(cluster, type),
+    business_defect_class: businessDefectClass(cluster, type),
+    severity: scored.severity,
+    confidence: scored.confidence,
+    affected_entities: uniqueStrings(backlinks.map((entry) => `${entry.entity}:${entry.id}`)),
+    affected_documents: collectAffectedByEntity(backlinks, /doc|document|invoice|плат|реал|поступ/i),
+    affected_postings: collectAffectedByEntity(backlinks, /posting|journal|провод/i),
+    affected_accounts: collectAffectedByEntity(backlinks, /account|счет|сч/i),
+    affected_counterparties: collectAffectedByEntity(backlinks, /counterparty|supplier|buyer|контраг|постав|покуп/i),
+    affected_contracts: collectAffectedByEntity(backlinks, /contract|договор/i),
+    ...(expectedState ? { expected_state: expectedState } : {}),
+    ...(actualState ? { actual_state: actualState } : {}),
+    ...(failedExpectedEdge ? { failed_expected_edge: failedExpectedEdge } : {}),
+    ...(periodSensitive
+      ? {
+          period_impact: {
+            is_period_sensitive: true,
+            impact_class: "close_risk" as const
+          }
+        }
+      : {}),
+    evidence_pack: uniqueStrings(cluster.candidates.map((item) => item.candidate_id)),
+    entity_backlinks: backlinks,
+    snapshot_limitations: uniqueStrings(
+      hasLowConfidence ? ["low_confidence_candidates_present"] : []
+    )
+  };
+}
+
+function collapseSignature(unit: ProblemUnit): string {
+  const backlink = unit.entity_backlinks[0] ? `${unit.entity_backlinks[0].entity}|${unit.entity_backlinks[0].id}` : "none";
+  return [unit.problem_unit_type, unit.business_defect_class, unit.failed_expected_edge ?? "none", backlink].join("|");
+}
+
+export function collapseDuplicates(units: ProblemUnit[]): {
+  problem_units: ProblemUnit[];
+  duplicate_collapses: number;
+} {
+  const bySignature = new Map<string, ProblemUnit>();
+  let duplicateCollapses = 0;
+
+  for (const unit of units) {
+    const signature = collapseSignature(unit);
+    const existing = bySignature.get(signature);
+    if (!existing) {
+      bySignature.set(signature, unit);
+      continue;
+    }
+
+    duplicateCollapses += 1;
+    bySignature.set(signature, {
+      ...existing,
+      evidence_pack: uniqueStrings([...existing.evidence_pack, ...unit.evidence_pack]),
+      entity_backlinks: Array.from(
+        new Map(
+          [...existing.entity_backlinks, ...unit.entity_backlinks].map((entry) => [
+            `${entry.entity.toLowerCase()}|${entry.id.toLowerCase()}`,
+            entry
+          ])
+        ).values()
+      ),
+      affected_entities: uniqueStrings([...existing.affected_entities, ...unit.affected_entities]),
+      affected_documents: uniqueStrings([...existing.affected_documents, ...unit.affected_documents]),
+      affected_postings: uniqueStrings([...existing.affected_postings, ...unit.affected_postings]),
+      affected_accounts: uniqueStrings([...existing.affected_accounts, ...unit.affected_accounts]),
+      affected_counterparties: uniqueStrings([...existing.affected_counterparties, ...unit.affected_counterparties]),
+      affected_contracts: uniqueStrings([...existing.affected_contracts, ...unit.affected_contracts]),
+      snapshot_limitations: uniqueStrings([...existing.snapshot_limitations, ...unit.snapshot_limitations]),
+      severity: unit.severity.score > existing.severity.score ? unit.severity : existing.severity,
+      confidence: unit.confidence.score > existing.confidence.score ? unit.confidence : existing.confidence
+    });
+  }
+
+  return {
+    problem_units: Array.from(bySignature.values()),
+    duplicate_collapses: duplicateCollapses
+  };
+}
+
+function buildSummary(units: ProblemUnit[], duplicateCollapses: number): ProblemUnitSummary {
+  const unitTypes = uniqueStrings(units.map((item) => item.problem_unit_type)) as ProblemUnitType[];
+  const typeDistribution: Partial<Record<ProblemUnitType, number>> = {};
+  const severityDistribution: Record<"low" | "medium" | "high", number> = {
+    low: 0,
+    medium: 0,
+    high: 0
+  };
+  const confidenceDistribution: Record<"low" | "medium" | "high", number> = {
+    low: 0,
+    medium: 0,
+    high: 0
+  };
+
+  for (const unit of units) {
+    typeDistribution[unit.problem_unit_type] = (typeDistribution[unit.problem_unit_type] ?? 0) + 1;
+    severityDistribution[unit.severity.grade] += 1;
+    confidenceDistribution[unit.confidence.grade] += 1;
+  }
+
+  return {
+    schema_version: PROBLEM_UNIT_SUMMARY_SCHEMA_VERSION,
+    units_total: units.length,
+    duplicate_collapses: duplicateCollapses,
+    unit_types: unitTypes,
+    type_distribution: typeDistribution,
+    severity_distribution: severityDistribution,
+    confidence_distribution: confidenceDistribution,
+    primary_unit_type: units[0]?.problem_unit_type ?? null
+  };
+}
+
+export function assembleProblemUnits(input: AssembleProblemUnitsInput): {
+  candidate_evidence: CandidateEvidenceItem[];
+  problem_units: ProblemUnit[];
+  problem_unit_summary: ProblemUnitSummary;
+} {
+  const summary = input.summary ?? {};
+  const semanticProfile = extractSemanticProfile(summary);
+
+  const candidates = buildCandidateEvidence(input.evidence, input.route, {
+    route: input.route,
+    result_type: input.result_type,
+    raw_entities: input.raw_entities ?? [],
+    summary_relation_patterns: semanticProfile.relation_patterns,
+    summary_anomaly_patterns: semanticProfile.anomaly_patterns,
+    risk_factors: uniqueStrings(input.risk_factors ?? [])
+  });
+  const clusters = clusterCandidateEvidence(candidates);
+  const units = clusters.map((cluster, index) => buildProblemUnit(cluster, index));
+  const collapsed = collapseDuplicates(units);
+
+  return {
+    candidate_evidence: candidates,
+    problem_units: collapsed.problem_units,
+    problem_unit_summary: buildSummary(collapsed.problem_units, collapsed.duplicate_collapses)
+  };
+}
+