Stage 2 завершён: problem-first ответы и follow-up continuity - ассистент переведён от entity-heavy логики к problem-first ответам с problem-unit слоем, удержанием контекста в follow-up и очисткой пользовательского ответа от сырых технических ссылок.
This commit is contained in:
+448
-28
@@ -10,21 +10,103 @@ function fallbackFromSummary(routeSummary) {
|
||||
function uniqueStrings(values, limit = 6) {
|
||||
return Array.from(new Set(values.map((item) => item.trim()).filter(Boolean))).slice(0, limit);
|
||||
}
|
||||
const UUID_PATTERN = /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi;
|
||||
const LONG_HEX_PATTERN = /\b[0-9a-f]{24,}\b/gi;
|
||||
const RAW_REF_BLOB_PATTERN = /\bevidence_source_ref_v1\|[^\s,;]+/gi;
|
||||
const RAW_REF_TOKEN_PATTERN = /\b(?:source_ref|canonical_ref|entity_id|fragment_id|guid|uuid)\b/gi;
|
||||
function looksLikeMojibake(value) {
|
||||
const text = String(value ?? "");
|
||||
if (!text.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/(?:Р.|С.){5,}/u.test(text)) {
|
||||
return true;
|
||||
}
|
||||
if (/[ЃѓЂђЌќЎў]/u.test(text)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function looksLikeTechnicalIdentifier(value) {
|
||||
const text = String(value ?? "").trim();
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
if (UUID_PATTERN.test(text)) {
|
||||
UUID_PATTERN.lastIndex = 0;
|
||||
return true;
|
||||
}
|
||||
UUID_PATTERN.lastIndex = 0;
|
||||
if (LONG_HEX_PATTERN.test(text)) {
|
||||
LONG_HEX_PATTERN.lastIndex = 0;
|
||||
return true;
|
||||
}
|
||||
LONG_HEX_PATTERN.lastIndex = 0;
|
||||
return /(?:evidence_source_ref_v1\||cmp%3a|batch_refresh_then_store:|^cmp:)/i.test(text);
|
||||
}
|
||||
function scrubRawTechnicalRefs(value) {
|
||||
const raw = String(value ?? "").trim();
|
||||
if (!raw) {
|
||||
return "";
|
||||
}
|
||||
return raw
|
||||
.replace(RAW_REF_BLOB_PATTERN, "linked source")
|
||||
.replace(UUID_PATTERN, "[id]")
|
||||
.replace(LONG_HEX_PATTERN, "[id]")
|
||||
.replace(RAW_REF_TOKEN_PATTERN, "reference")
|
||||
.replace(/\(\s*\[id\]\s*\)/g, "")
|
||||
.replace(/\[\s*id\s*\](?:\s*,\s*\[\s*id\s*\])+/g, "[id]")
|
||||
.replace(/\s{2,}/g, " ")
|
||||
.trim();
|
||||
}
|
||||
function sanitizeUserFacingReply(value) {
|
||||
return scrubRawTechnicalRefs(value)
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
function sanitizeUserText(value) {
|
||||
const normalized = scrubRawTechnicalRefs(String(value ?? "").replace(/\s+/g, " ").trim());
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
if (looksLikeMojibake(normalized)) {
|
||||
return null;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
function sanitizeUserLines(values, limit = 6) {
|
||||
const cleaned = values
|
||||
.map((item) => sanitizeUserText(item))
|
||||
.filter((item) => Boolean(item));
|
||||
return uniqueStrings(cleaned, limit);
|
||||
}
|
||||
function formatList(items) {
|
||||
if (items.length === 0) {
|
||||
return "";
|
||||
}
|
||||
return items.map((item) => `- ${item}`).join("\n");
|
||||
}
|
||||
function formatSafeItemLine(entity, sourceId, riskScore) {
|
||||
const entityLabel = sanitizeUserText(String(entity ?? "")) ?? "Record";
|
||||
const idRaw = String(sourceId ?? "").trim();
|
||||
const exposeId = idRaw.length > 0 && !looksLikeTechnicalIdentifier(idRaw);
|
||||
const subject = exposeId ? `${entityLabel} (${idRaw})` : entityLabel;
|
||||
if (riskScore !== undefined) {
|
||||
return `${subject} - risk ${String(riskScore)}.`;
|
||||
}
|
||||
return `${subject}.`;
|
||||
}
|
||||
function extractTopFacts(results) {
|
||||
const lines = [];
|
||||
for (const result of results.filter((item) => item.status === "ok").slice(0, 3)) {
|
||||
if (result.result_type === "chain") {
|
||||
const top = result.items.slice(0, 3).map((item) => {
|
||||
const counterparty = String(item.counterparty_id ?? "не указан");
|
||||
const counterparty = String(item.counterparty_id ?? "").trim();
|
||||
const operations = String(item.operations_count ?? "0");
|
||||
const docs = String(item.document_refs_count ?? "0");
|
||||
return `Контрагент ${counterparty}: операций ${operations}, документов в связке ${docs}.`;
|
||||
const counterpartyLabel = counterparty.length > 0 && !looksLikeTechnicalIdentifier(counterparty) ? `Counterparty ${counterparty}` : "Counterparty";
|
||||
return `${counterpartyLabel}: operations ${operations}, linked docs ${docs}.`;
|
||||
});
|
||||
lines.push(...top);
|
||||
continue;
|
||||
@@ -32,41 +114,41 @@ function extractTopFacts(results) {
|
||||
if (result.result_type === "ranking") {
|
||||
const top = result.items
|
||||
.slice(0, 5)
|
||||
.map((item) => `${item.rank ?? "•"}. ${String(item.entity ?? "Сущность")} — ${String(item.records_count ?? 0)}.`);
|
||||
.map((item) => `${item.rank ?? "*"}. ${String(item.entity ?? "Entity")} - ${String(item.records_count ?? 0)}.`);
|
||||
lines.push(...top);
|
||||
continue;
|
||||
}
|
||||
if (result.result_type === "list") {
|
||||
const top = result.items.slice(0, 5).map((item) => {
|
||||
if (item.risk_score !== undefined) {
|
||||
return `${String(item.source_entity ?? "Запись")} (${String(item.source_id ?? "")}) — риск ${String(item.risk_score)}.`;
|
||||
return formatSafeItemLine(item.source_entity ?? "Record", item.source_id ?? "", item.risk_score);
|
||||
}
|
||||
return `${String(item.source_entity ?? "Запись")} (${String(item.source_id ?? "")}).`;
|
||||
return formatSafeItemLine(item.source_entity ?? "Record", item.source_id ?? "");
|
||||
});
|
||||
lines.push(...top);
|
||||
continue;
|
||||
}
|
||||
const top = result.items
|
||||
.slice(0, 3)
|
||||
.map((item) => `${String(item.source_entity ?? "Запись")} (${String(item.source_id ?? "")}).`);
|
||||
.map((item) => formatSafeItemLine(item.source_entity ?? "Record", item.source_id ?? ""));
|
||||
lines.push(...top);
|
||||
}
|
||||
return lines;
|
||||
return sanitizeUserLines(lines, 8);
|
||||
}
|
||||
function extractWhyIncluded(results) {
|
||||
return uniqueStrings(results.flatMap((item) => item.why_included));
|
||||
return sanitizeUserLines(results.flatMap((item) => item.why_included));
|
||||
}
|
||||
function extractSelectionReasons(results) {
|
||||
return uniqueStrings(results.flatMap((item) => item.selection_reason));
|
||||
return sanitizeUserLines(results.flatMap((item) => item.selection_reason));
|
||||
}
|
||||
function extractRiskFactors(results) {
|
||||
return uniqueStrings(results.flatMap((item) => item.risk_factors));
|
||||
return sanitizeUserLines(results.flatMap((item) => item.risk_factors));
|
||||
}
|
||||
function extractBusinessInterpretation(results) {
|
||||
return uniqueStrings(results.flatMap((item) => item.business_interpretation));
|
||||
return sanitizeUserLines(results.flatMap((item) => item.business_interpretation));
|
||||
}
|
||||
function extractLimitations(results) {
|
||||
return uniqueStrings(results.flatMap((item) => item.limitations));
|
||||
return sanitizeUserLines(results.flatMap((item) => item.limitations), 10);
|
||||
}
|
||||
function summaryValue(result, key) {
|
||||
const summary = result.summary ?? {};
|
||||
@@ -79,6 +161,55 @@ function summaryString(result, key) {
|
||||
const value = summaryValue(result, key);
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
function summaryNumber(result, key) {
|
||||
const value = summaryValue(result, key);
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
function summaryStringArray(result, key) {
|
||||
const value = summaryValue(result, key);
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return sanitizeUserLines(value.map((item) => String(item)), 6);
|
||||
}
|
||||
function buildFallbackWhyIncluded(results) {
|
||||
const lines = [];
|
||||
for (const result of results.slice(0, 2)) {
|
||||
const routeFocus = summaryString(result, "route_focus");
|
||||
const sourceRecords = summaryNumber(result, "source_records");
|
||||
const filteredRecords = summaryNumber(result, "filtered_records_after_narrowing");
|
||||
const checkedRecords = summaryNumber(result, "checked_records");
|
||||
if (routeFocus) {
|
||||
lines.push(`Проверка выполнена по профилю ${routeFocus}.`);
|
||||
}
|
||||
if (sourceRecords !== null && filteredRecords !== null && filteredRecords < sourceRecords) {
|
||||
lines.push(`Применено сужение выборки: ${filteredRecords} из ${sourceRecords} записей.`);
|
||||
}
|
||||
if (checkedRecords !== null) {
|
||||
lines.push(`Проверено записей в текущем проходе: ${checkedRecords}.`);
|
||||
}
|
||||
}
|
||||
return sanitizeUserLines(lines, 4);
|
||||
}
|
||||
function buildFallbackSelectionReasons(results) {
|
||||
const lines = [];
|
||||
for (const result of results.slice(0, 2)) {
|
||||
if (summaryBoolean(result, "semantic_narrowing_applied")) {
|
||||
lines.push("Отбор выполнен по семантическому сужению предметной области.");
|
||||
}
|
||||
const rankingBasis = summaryStringArray(result, "ranking_basis");
|
||||
if (rankingBasis.length > 0) {
|
||||
lines.push(`Ранжирование основано на: ${rankingBasis.join(", ")}.`);
|
||||
}
|
||||
if (summaryBoolean(result, "broad_guard_applied")) {
|
||||
lines.push("Применен broad-query guard для контроля ложной точности.");
|
||||
}
|
||||
}
|
||||
if (lines.length === 0) {
|
||||
lines.push("Отбор выполнен по совпадению предметных сигналов и доступной evidence-опоры.");
|
||||
}
|
||||
return sanitizeUserLines(lines, 4);
|
||||
}
|
||||
function suggestNextStep(requirements, coverage) {
|
||||
const next = [];
|
||||
if (coverage.clarification_needed_for.length > 0) {
|
||||
@@ -95,9 +226,131 @@ function suggestNextStep(requirements, coverage) {
|
||||
}
|
||||
return next;
|
||||
}
|
||||
const PROBLEM_HEAVY_TYPES = new Set([
|
||||
"document_conflict",
|
||||
"broken_chain_segment",
|
||||
"lifecycle_anomaly_node",
|
||||
"unresolved_settlement_cluster",
|
||||
"period_risk_cluster",
|
||||
"cross_branch_inconsistency_cluster"
|
||||
]);
|
||||
function flattenEvidence(results) {
|
||||
return results.flatMap((item) => item.evidence);
|
||||
}
|
||||
function flattenProblemUnits(results) {
|
||||
const units = [];
|
||||
for (const result of results) {
|
||||
if (!Array.isArray(result.problem_units)) {
|
||||
continue;
|
||||
}
|
||||
units.push(...result.problem_units);
|
||||
}
|
||||
const byId = new Map();
|
||||
for (const unit of units) {
|
||||
byId.set(unit.problem_unit_id, unit);
|
||||
}
|
||||
return Array.from(byId.values());
|
||||
}
|
||||
function selectProblemUnitSummary(results) {
|
||||
let selected = null;
|
||||
for (const result of results) {
|
||||
if (!result.problem_unit_summary) {
|
||||
continue;
|
||||
}
|
||||
if (!selected || result.problem_unit_summary.units_total > selected.units_total) {
|
||||
selected = result.problem_unit_summary;
|
||||
}
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
function formatAffectedScope(unit) {
|
||||
const scopeParts = [];
|
||||
if (unit.affected_accounts.length > 0) {
|
||||
scopeParts.push(`счета: ${unit.affected_accounts.slice(0, 2).join(", ")}`);
|
||||
}
|
||||
if (unit.affected_counterparties.length > 0) {
|
||||
scopeParts.push(`контрагенты: ${unit.affected_counterparties.slice(0, 2).join(", ")}`);
|
||||
}
|
||||
if (unit.affected_documents.length > 0) {
|
||||
scopeParts.push(`документы: ${unit.affected_documents.slice(0, 2).join(", ")}`);
|
||||
}
|
||||
if (scopeParts.length === 0 && unit.affected_entities.length > 0) {
|
||||
scopeParts.push(`объекты: ${unit.affected_entities.slice(0, 2).join(", ")}`);
|
||||
}
|
||||
if (scopeParts.length === 0) {
|
||||
return "затронутый контур требует уточнения";
|
||||
}
|
||||
return scopeParts.join("; ");
|
||||
}
|
||||
function buildProblemCentricActions(input) {
|
||||
const actions = [];
|
||||
const unitTypes = new Set(input.units.map((item) => item.problem_unit_type));
|
||||
if (unitTypes.has("broken_chain_segment")) {
|
||||
actions.push("Проверьте связку выписка -> документ -> проводка по проблемным участкам цепочки.");
|
||||
}
|
||||
if (unitTypes.has("unresolved_settlement_cluster")) {
|
||||
actions.push("Сверьте хвосты по расчетам: закрылся ли документ оплаты корректным закрывающим документом.");
|
||||
}
|
||||
if (unitTypes.has("period_risk_cluster")) {
|
||||
actions.push("Оцените влияние дефекта на закрытие периода и корректность регламентных операций.");
|
||||
}
|
||||
if (unitTypes.has("cross_branch_inconsistency_cluster")) {
|
||||
actions.push("Сверьте противоречия между документами, проводками и регистрами по НДС/межконтурным связям.");
|
||||
}
|
||||
if (unitTypes.has("lifecycle_anomaly_node")) {
|
||||
actions.push("Проверьте lifecycle объекта: ожидаемый этап не должен оставаться в partially_linked состоянии.");
|
||||
}
|
||||
if (input.mode === "clarification_required") {
|
||||
if (input.missingAnchors.period) {
|
||||
actions.push("Уточните период проверки, чтобы зафиксировать границы проблемного контура.");
|
||||
}
|
||||
if (input.missingAnchors.account) {
|
||||
actions.push("Уточните счет или группу счетов для предметной локализации дефекта.");
|
||||
}
|
||||
if (input.missingAnchors.documentOrObject) {
|
||||
actions.push("Укажите конкретный документ или объект трассировки для проверки механизма отклонения.");
|
||||
}
|
||||
if (input.missingAnchors.counterparty) {
|
||||
actions.push("Укажите контрагента/договор, чтобы проверить хвосты и разрывы на конкретной связке.");
|
||||
}
|
||||
}
|
||||
if (input.coverageReport.requirements_uncovered.length > 0) {
|
||||
actions.push(`Закройте непокрытые требования: ${input.coverageReport.requirements_uncovered.join(", ")}.`);
|
||||
}
|
||||
return uniqueStrings(actions, 6);
|
||||
}
|
||||
function buildProblemCentricClarifications(input) {
|
||||
if (input.mode !== "clarification_required") {
|
||||
return [];
|
||||
}
|
||||
const questions = [];
|
||||
const unitTypes = new Set(input.units.map((item) => item.problem_unit_type));
|
||||
if (input.missingAnchors.period) {
|
||||
questions.push("Уточните период (например, 2020-06), в котором нужно проверить проблемный кластер.");
|
||||
}
|
||||
if (input.missingAnchors.account) {
|
||||
questions.push("Уточните счет или связку счетов (например, 51/60), где вы ожидаете дефект.");
|
||||
}
|
||||
if (input.missingAnchors.documentOrObject) {
|
||||
questions.push("Укажите документ/объект, от которого нужно строить проверку цепочки.");
|
||||
}
|
||||
if (input.missingAnchors.counterparty) {
|
||||
questions.push("Укажите контрагента или договор, по которому проверить незакрытую экспозицию.");
|
||||
}
|
||||
if (unitTypes.has("broken_chain_segment")) {
|
||||
questions.push("Уточните участок цепочки: выписка, платежный документ или проводка.");
|
||||
}
|
||||
if (unitTypes.has("period_risk_cluster")) {
|
||||
questions.push("Уточните, какой этап закрытия периода критичен: начисление, закрытие счетов или НДС-блок.");
|
||||
}
|
||||
if (unitTypes.has("unresolved_settlement_cluster")) {
|
||||
questions.push("Уточните, интересуют хвосты поставщиков, покупателей или оба направления.");
|
||||
}
|
||||
if (input.coverageReport.clarification_needed_for.length > 0) {
|
||||
questions.push(`Закройте уточнения для требований: ${input.coverageReport.clarification_needed_for.join(", ")}.`);
|
||||
}
|
||||
return uniqueStrings(questions, 6);
|
||||
}
|
||||
function buildClaimEvidenceLinks(results) {
|
||||
const byClaim = new Map();
|
||||
for (const evidence of flattenEvidence(results)) {
|
||||
@@ -246,7 +499,7 @@ function buildClarificationQuestions(input) {
|
||||
function buildRecommendedActions(input) {
|
||||
const actions = [];
|
||||
if (input.mode === "focused_grounded") {
|
||||
actions.push("Проверьте 1-2 ключевые записи по source_ref и зафиксируйте итог в рабочем файле проверки.");
|
||||
actions.push("Проверьте 1-2 ключевые записи в учетной базе и зафиксируйте итог в рабочем файле проверки.");
|
||||
}
|
||||
if (input.mode === "broad_partial") {
|
||||
actions.push("Сузьте запрос до периода + счета или периода + документа и повторите проверку.");
|
||||
@@ -270,7 +523,7 @@ function buildRecommendedActions(input) {
|
||||
actions.push("Проверьте source mapping для связей document/register по указанным ref.");
|
||||
}
|
||||
if (input.sourceRefs.length > 0) {
|
||||
actions.push(`Начните проверку с source_ref: ${input.sourceRefs.slice(0, 2).join(", ")}.`);
|
||||
actions.push(`Начните проверку с ${input.sourceRefs.length} подтвержденных записей и сверьте их с первичными документами.`);
|
||||
}
|
||||
return uniqueStrings(actions, 6);
|
||||
}
|
||||
@@ -403,6 +656,122 @@ function buildDirectAnswer(input) {
|
||||
}
|
||||
return "Не удалось сформировать обоснованный ответ; нужно уточнение запроса.";
|
||||
}
|
||||
function buildProblemCentricAnswerSummary(input) {
|
||||
if (input.mode === "clarification_required") {
|
||||
return "Выявлены проблемные кластеры, но для надежного вывода требуется предметное уточнение фокуса.";
|
||||
}
|
||||
if (input.weakUnits) {
|
||||
return "Сформирован problem-centric срез с ограниченной опорой; вывод предварительный и требует до-проверки.";
|
||||
}
|
||||
if (input.summary?.units_total && input.summary.units_total > 1) {
|
||||
return `Сформирован problem-centric срез: выделено ${input.summary.units_total} проблемных кластера с приоритетами.`;
|
||||
}
|
||||
return "Сформирован problem-centric срез: выделен ключевой проблемный кластер и затронутый контур.";
|
||||
}
|
||||
function buildProblemCentricDirectAnswer(input) {
|
||||
const lead = input.mode === "clarification_required"
|
||||
? "Обнаружены проблемные зоны, но без уточнения якорей сильный factual-вывод преждевременен."
|
||||
: input.weakUnits
|
||||
? "Выделены проблемные зоны с ограниченной надежностью; вывод дан в ограниченном режиме."
|
||||
: "Выделены ключевые проблемные зоны и их влияние на учетный контур.";
|
||||
const unitLines = input.units.map((unit) => {
|
||||
const scope = formatAffectedScope(unit);
|
||||
return `- ${unit.title}: ${unit.business_defect_class}; ${scope}; severity=${unit.severity.grade}, confidence=${unit.confidence.grade}.`;
|
||||
});
|
||||
if (unitLines.length === 0) {
|
||||
return `${lead}\nПроблемные кластеры не удалось детализировать в текущем срезе.`;
|
||||
}
|
||||
return [lead, "Проблемные кластеры:", ...unitLines].join("\n");
|
||||
}
|
||||
function buildProblemCentricAnswerStructure(input) {
|
||||
const weakUnits = input.selectedUnits.every((item) => item.confidence.grade === "low");
|
||||
const unitMechanismNotes = uniqueStrings(input.selectedUnits
|
||||
.map((item) => item.mechanism_summary)
|
||||
.filter((item) => typeof item === "string" && item.trim().length > 0), 6);
|
||||
const sourceRefs = uniqueStrings(input.evidenceItems
|
||||
.map((item) => item.source_ref?.canonical_ref)
|
||||
.filter((item) => typeof item === "string" && item.trim().length > 0), 6);
|
||||
const evidenceIds = uniqueStrings(input.evidenceItems.map((item) => item.evidence_id), 10);
|
||||
const mechanismStatus = unitMechanismNotes.length === 0
|
||||
? "unresolved"
|
||||
: weakUnits || input.limitationReasonCodes.includes("missing_mechanism")
|
||||
? "limited"
|
||||
: "grounded";
|
||||
const problemSpecificLimitations = [];
|
||||
if (weakUnits) {
|
||||
problemSpecificLimitations.push("Problem units remain weak-confidence; conclusions are intentionally limited.");
|
||||
}
|
||||
if (input.problemSummary?.duplicate_collapses && input.problemSummary.duplicate_collapses > 0) {
|
||||
problemSpecificLimitations.push("Part of the problem signal was merged due to duplicate collapse.");
|
||||
}
|
||||
const limitations = uniqueStrings([
|
||||
...problemSpecificLimitations,
|
||||
...input.limitationReasonCodes.map((code) => limitationReasonToText(code)),
|
||||
...extractLimitations(input.retrievalResults),
|
||||
...input.groundingCheck.reasons
|
||||
], 10);
|
||||
const openUncertainties = uniqueStrings([
|
||||
...input.groundingCheck.missing_requirements,
|
||||
...(input.mode === "clarification_required" && input.missingAnchors.period ? ["missing_anchor:period"] : []),
|
||||
...(input.mode === "clarification_required" && input.missingAnchors.account ? ["missing_anchor:account"] : []),
|
||||
...(input.mode === "clarification_required" && input.missingAnchors.documentOrObject
|
||||
? ["missing_anchor:document_or_object"]
|
||||
: []),
|
||||
...(input.mode === "clarification_required" && input.missingAnchors.counterparty ? ["missing_anchor:counterparty"] : [])
|
||||
], 8);
|
||||
return {
|
||||
schema_version: "answer_structure_v1_1",
|
||||
answer_summary: buildProblemCentricAnswerSummary({
|
||||
mode: input.mode,
|
||||
weakUnits,
|
||||
summary: input.problemSummary
|
||||
}),
|
||||
direct_answer: buildProblemCentricDirectAnswer({
|
||||
mode: input.mode,
|
||||
units: input.selectedUnits,
|
||||
weakUnits
|
||||
}),
|
||||
mechanism_block: {
|
||||
status: mechanismStatus,
|
||||
mechanism_notes: unitMechanismNotes,
|
||||
limitation_reason_codes: input.limitationReasonCodes
|
||||
},
|
||||
evidence_block: {
|
||||
evidence_ids: evidenceIds,
|
||||
source_refs: sourceRefs,
|
||||
mechanism_notes: unitMechanismNotes,
|
||||
coverage_note: input.coverageReport.requirements_total > 0 &&
|
||||
input.coverageReport.requirements_total === input.coverageReport.requirements_covered &&
|
||||
input.coverageReport.requirements_uncovered.length === 0 &&
|
||||
input.coverageReport.requirements_partially_covered.length === 0
|
||||
? "coverage_full_or_near_full"
|
||||
: "coverage_partial_or_limited",
|
||||
...(input.claimEvidenceLinks.length > 0
|
||||
? {
|
||||
claim_evidence_links: input.claimEvidenceLinks
|
||||
}
|
||||
: {})
|
||||
},
|
||||
uncertainty_block: {
|
||||
open_uncertainties: openUncertainties,
|
||||
limitations
|
||||
},
|
||||
next_step_block: {
|
||||
recommended_actions: buildProblemCentricActions({
|
||||
units: input.selectedUnits,
|
||||
mode: input.mode,
|
||||
missingAnchors: input.missingAnchors,
|
||||
coverageReport: input.coverageReport
|
||||
}),
|
||||
clarification_questions: buildProblemCentricClarifications({
|
||||
units: input.selectedUnits,
|
||||
missingAnchors: input.missingAnchors,
|
||||
coverageReport: input.coverageReport,
|
||||
mode: input.mode
|
||||
})
|
||||
}
|
||||
};
|
||||
}
|
||||
function renderPolicyReply(structure) {
|
||||
const mechanismLines = [`status=${structure.mechanism_block.status}`];
|
||||
if (structure.mechanism_block.mechanism_notes.length > 0) {
|
||||
@@ -414,18 +783,18 @@ function renderPolicyReply(structure) {
|
||||
if (structure.mechanism_block.status === "unresolved" && structure.mechanism_block.mechanism_notes.length === 0) {
|
||||
mechanismLines.push("mechanism_note is intentionally omitted due to weak or missing mechanism evidence");
|
||||
}
|
||||
const sourceRefCount = Array.isArray(structure.evidence_block.source_refs) ? structure.evidence_block.source_refs.length : 0;
|
||||
const claimLinkCount = Array.isArray(structure.evidence_block.claim_evidence_links)
|
||||
? structure.evidence_block.claim_evidence_links.length
|
||||
: 0;
|
||||
const evidenceLines = [
|
||||
`coverage=${structure.evidence_block.coverage_note}`,
|
||||
`evidence_ids=${structure.evidence_block.evidence_ids.length > 0 ? structure.evidence_block.evidence_ids.join(", ") : "none"}`
|
||||
`supporting_evidence_count=${structure.evidence_block.evidence_ids.length}`,
|
||||
`supporting_source_count=${sourceRefCount}`,
|
||||
`claim_support_links=${claimLinkCount}`
|
||||
];
|
||||
if (Array.isArray(structure.evidence_block.source_refs) && structure.evidence_block.source_refs.length > 0) {
|
||||
evidenceLines.push(`source_refs=${structure.evidence_block.source_refs.join(", ")}`);
|
||||
}
|
||||
if (Array.isArray(structure.evidence_block.claim_evidence_links) && structure.evidence_block.claim_evidence_links.length > 0) {
|
||||
const compactLinks = structure.evidence_block.claim_evidence_links
|
||||
.slice(0, 4)
|
||||
.map((item) => `${item.claim_ref}:${item.evidence_ids.join("|")}`);
|
||||
evidenceLines.push(`claim_evidence_links=${compactLinks.join("; ")}`);
|
||||
if (sourceRefCount > 0) {
|
||||
evidenceLines.push("Detailed source references are available in debug payload.");
|
||||
}
|
||||
const uncertaintyLines = [
|
||||
...structure.uncertainty_block.open_uncertainties.map((item) => `open: ${item}`),
|
||||
@@ -441,7 +810,7 @@ function renderPolicyReply(structure) {
|
||||
if (nextStepLines.length === 0) {
|
||||
nextStepLines.push("No additional action is required for this scoped answer.");
|
||||
}
|
||||
return [
|
||||
return sanitizeUserFacingReply([
|
||||
`Answer summary: ${structure.answer_summary}`,
|
||||
`Direct answer:\n${structure.direct_answer}`,
|
||||
`Mechanism block:\n${formatList(mechanismLines)}`,
|
||||
@@ -450,7 +819,7 @@ function renderPolicyReply(structure) {
|
||||
`Next step block:\n${formatList(nextStepLines)}`
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
.join("\n\n"));
|
||||
}
|
||||
function composeAssistantAnswerV11(input) {
|
||||
const fallbackType = fallbackFromSummary(input.routeSummary);
|
||||
@@ -467,8 +836,15 @@ function composeAssistantAnswerV11(input) {
|
||||
const mechanismNotes = uniqueStrings(evidenceItems
|
||||
.map((item) => item.mechanism_note)
|
||||
.filter((item) => typeof item === "string" && item.trim().length > 0), 6);
|
||||
const problemUnits = flattenProblemUnits(input.retrievalResults);
|
||||
const problemUnitSummary = selectProblemUnitSummary(input.retrievalResults);
|
||||
const problemHeavyUnits = problemUnits.filter((item) => PROBLEM_HEAVY_TYPES.has(item.problem_unit_type));
|
||||
const selectedProblemUnits = problemHeavyUnits.slice(0, 4);
|
||||
const claimEvidenceLinks = buildClaimEvidenceLinks(input.retrievalResults);
|
||||
const aggregateEvidenceConfidence = aggregateConfidence(input.retrievalResults, evidenceItems);
|
||||
const lowConfidenceSignals = evidenceItems.filter((item) => item.confidence === "low").length;
|
||||
const lowConfidenceShare = evidenceItems.length > 0 ? lowConfidenceSignals / evidenceItems.length : 0;
|
||||
const lowConfidenceConcentration = lowConfidenceShare >= 0.6;
|
||||
const hasSupport = okResults.length > 0 ||
|
||||
partialResults.length > 0 ||
|
||||
evidenceItems.length > 0 ||
|
||||
@@ -501,6 +877,45 @@ function composeAssistantAnswerV11(input) {
|
||||
policySignals
|
||||
});
|
||||
const missingAnchors = detectMissingAnchors(input.userMessage);
|
||||
const hasProblemWeakSignal = policySignals.narrowing_strength !== "strong" ||
|
||||
policySignals.minimum_evidence_failed ||
|
||||
limitationReasonCodes.includes("missing_mechanism") ||
|
||||
limitationReasonCodes.includes("weak_source_mapping") ||
|
||||
aggregateEvidenceConfidence === "low" ||
|
||||
lowConfidenceConcentration;
|
||||
const hardBlockedMode = decision.mode === "out_of_scope" || decision.mode === "route_mismatch" || decision.mode === "backend_error";
|
||||
const problemCentricModeEligible = decision.mode === "broad_partial" ||
|
||||
decision.mode === "clarification_required" ||
|
||||
(decision.mode === "focused_grounded" && hasProblemWeakSignal);
|
||||
const shouldUseProblemCentricAnswer = Boolean(input.enableProblemCentricAnswerV1) &&
|
||||
!hardBlockedMode &&
|
||||
problemCentricModeEligible &&
|
||||
(!focusedStrong || hasProblemWeakSignal) &&
|
||||
selectedProblemUnits.length > 0;
|
||||
if (shouldUseProblemCentricAnswer) {
|
||||
const problemCentricStructure = buildProblemCentricAnswerStructure({
|
||||
mode: decision.mode,
|
||||
selectedUnits: selectedProblemUnits,
|
||||
problemSummary: problemUnitSummary,
|
||||
evidenceItems,
|
||||
claimEvidenceLinks,
|
||||
limitationReasonCodes,
|
||||
groundingCheck: input.groundingCheck,
|
||||
retrievalResults: input.retrievalResults,
|
||||
missingAnchors,
|
||||
coverageReport: input.coverageReport
|
||||
});
|
||||
return {
|
||||
assistant_reply: renderPolicyReply(problemCentricStructure),
|
||||
fallback_type: decision.fallback_type,
|
||||
reply_type: decision.reply_type,
|
||||
answer_structure_v11: problemCentricStructure,
|
||||
problem_centric_answer_applied: true,
|
||||
problem_units_used_count: selectedProblemUnits.length,
|
||||
problem_answer_mode: "stage2_problem_centric_v1",
|
||||
problem_unit_ids_used: selectedProblemUnits.map((item) => item.problem_unit_id)
|
||||
};
|
||||
}
|
||||
const clarificationQuestions = buildClarificationQuestions({
|
||||
mode: decision.mode,
|
||||
missingAnchors,
|
||||
@@ -577,13 +992,18 @@ function composeAssistantAnswerV11(input) {
|
||||
assistant_reply: renderPolicyReply(answerStructure),
|
||||
fallback_type: decision.fallback_type,
|
||||
reply_type: decision.reply_type,
|
||||
answer_structure_v11: answerStructure
|
||||
answer_structure_v11: answerStructure,
|
||||
problem_centric_answer_applied: false,
|
||||
problem_units_used_count: 0,
|
||||
problem_answer_mode: "stage1_policy_v11"
|
||||
};
|
||||
}
|
||||
function composeExplainableAnswer(input, scopeLabel) {
|
||||
const facts = extractTopFacts(input.retrievalResults);
|
||||
const whyIncluded = extractWhyIncluded(input.retrievalResults);
|
||||
const selectionReasons = extractSelectionReasons(input.retrievalResults);
|
||||
const whyIncludedRaw = extractWhyIncluded(input.retrievalResults);
|
||||
const selectionReasonsRaw = extractSelectionReasons(input.retrievalResults);
|
||||
const whyIncluded = whyIncludedRaw.length > 0 ? whyIncludedRaw : buildFallbackWhyIncluded(input.retrievalResults);
|
||||
const selectionReasons = selectionReasonsRaw.length > 0 ? selectionReasonsRaw : buildFallbackSelectionReasons(input.retrievalResults);
|
||||
const riskFactors = extractRiskFactors(input.retrievalResults);
|
||||
const interpretation = extractBusinessInterpretation(input.retrievalResults);
|
||||
const limitations = uniqueStrings([...extractLimitations(input.retrievalResults), ...input.groundingCheck.reasons]);
|
||||
|
||||
+88
-7
@@ -638,17 +638,17 @@ function hasAccountingSignal(text) {
|
||||
if (/(?:^|[\s,;:])\d{2}(?:\.\d{2})?(?=$|[\s,.;:])/i.test(lower)) {
|
||||
return true;
|
||||
}
|
||||
return /(проводк|документ|реализац|поступлен|взаиморасчет|сальдо|остатк|счет|ндс|амортиз|рбп|ос|контрагент|поставщик|покупател|оплат|банк|выписк|склад|товар|материал|counterparty|supplier|invoice|posting|ledger|account|anomaly|risk)/i.test(lower);
|
||||
return /(РїСЂРѕРІРѕРґРє|документ|реализац|поступлен|взаиморасчет|сальдо|остатк|счет|РЅРґСЃ|амортиз|СЂР±Рї|РѕСЃ|контрагент|поставщик|покупател|оплат|банк|выписк|склад|товар|материал|проводк|документ|реализац|поступлен|взаиморасчет|сальдо|остатк|счет|счёт|ндс|амортиз|рбп|контрагент|поставщик|покупател|оплат|банк|выписк|склад|товар|материал|закрыти|период|postavshchik|kontragent|schet|schetu|period|counterparty|supplier|invoice|posting|ledger|account|anomaly|risk)/i.test(lower);
|
||||
}
|
||||
function hasFollowupMarker(text) {
|
||||
const compact = compactWhitespace(text.toLowerCase());
|
||||
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|plus|also|dobav|utochn|prodolzh)/i.test(compact);
|
||||
return /^(Рё|Р° еще|Р° ещё|еще|ещё|добав|уточн|продолж|также|и|а если|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|plus|also|dobav|utochn|prodolzh)/i.test(compact);
|
||||
}
|
||||
function hasReferentialPointer(text) {
|
||||
return /(по этому|по тому|это же|этой|этим|тому|same thing|that one|po etomu|po tomu)/i.test(text.toLowerCase());
|
||||
return /(РїРѕ этому|РїРѕ тому|это же|этой|этим|тому|по этому|по тому|это же|этой|этим|этому|из этого|в этом|тот же|same thing|that one|po etomu|po tomu)/i.test(text.toLowerCase());
|
||||
}
|
||||
function hasSmallTalkSignal(text) {
|
||||
return /(привет|как дела|спасибо|thanks|thank you|hello|hi)\b/i.test(text.toLowerCase());
|
||||
return /(привет|как дела|спасибо|привет|как дела|спасибо|благодарю|thanks|thank you|hello|hi)\b/i.test(text.toLowerCase());
|
||||
}
|
||||
function countTokens(text) {
|
||||
return compactWhitespace(text)
|
||||
@@ -658,6 +658,37 @@ function countTokens(text) {
|
||||
function hasPeriodLiteral(text) {
|
||||
return /\b(20\d{2}(?:[-/.](?:0[1-9]|1[0-2]))?)\b/.test(text);
|
||||
}
|
||||
function extractNormalizedPeriodLiteral(text) {
|
||||
const monthly = text.match(/\b(20\d{2})[-/.](0[1-9]|1[0-2])\b/);
|
||||
if (monthly) {
|
||||
return `${monthly[1]}-${monthly[2]}`;
|
||||
}
|
||||
const yearly = text.match(/\b(20\d{2})\b/);
|
||||
if (yearly) {
|
||||
return yearly[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function hasStrongFollowupAnchors(userMessage, state) {
|
||||
const explicitPeriod = extractNormalizedPeriodLiteral(userMessage);
|
||||
if (explicitPeriod && state.focus.period && explicitPeriod !== state.focus.period) {
|
||||
const periodLooksLikeFollowupRefinement = hasFollowupMarker(userMessage) || hasReferentialPointer(userMessage);
|
||||
if (!periodLooksLikeFollowupRefinement) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const explicitAccounts = extractAccountTokens(userMessage);
|
||||
if (explicitAccounts.length > 0) {
|
||||
const knownAccounts = new Set(state.focus.primary_accounts.map((item) => item.trim()));
|
||||
if (knownAccounts.size === 0) {
|
||||
return true;
|
||||
}
|
||||
if (explicitAccounts.some((item) => !knownAccounts.has(item))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function routeFromInvestigationState(state) {
|
||||
const rawDomain = compactWhitespace(state.focus.domain ?? "");
|
||||
if (!rawDomain) {
|
||||
@@ -699,7 +730,15 @@ function buildFollowupStateBinding(input) {
|
||||
const referentialPointer = hasReferentialPointer(userMessage);
|
||||
const shortPrompt = countTokens(userMessage) <= 10;
|
||||
const smallTalkSignal = hasSmallTalkSignal(userMessage);
|
||||
const shouldBind = !smallTalkSignal && (followupMarker || referentialPointer || (!strongSignal && shortPrompt));
|
||||
const problemState = input.investigationState.problem_unit_state;
|
||||
const problemContinuityAvailable = config_1.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 &&
|
||||
Boolean(problemState) &&
|
||||
((problemState?.active_problem_units.length ?? 0) > 0 || (problemState?.focus_problem_types.length ?? 0) > 0);
|
||||
const strongNewAnchorDetected = hasStrongFollowupAnchors(userMessage, input.investigationState);
|
||||
const periodRefinementFollowup = hasPeriodLiteral(userMessage) && problemContinuityAvailable;
|
||||
const shouldBind = !smallTalkSignal &&
|
||||
!strongNewAnchorDetected &&
|
||||
(followupMarker || referentialPointer || periodRefinementFollowup || (!strongSignal && shortPrompt));
|
||||
if (!shouldBind) {
|
||||
return {
|
||||
normalizedQuestion: userMessage,
|
||||
@@ -710,6 +749,7 @@ function buildFollowupStateBinding(input) {
|
||||
const context = {
|
||||
...(input.payloadContext ?? {})
|
||||
};
|
||||
const hasExplicitExpectedRoute = Boolean(input.payloadContext?.expected_route);
|
||||
const expectedRouteFromState = !context?.expected_route ? routeFromInvestigationState(input.investigationState) : null;
|
||||
const periodHintFromState = !context?.period_hint ? input.investigationState.focus.period : null;
|
||||
if (expectedRouteFromState) {
|
||||
@@ -720,6 +760,8 @@ function buildFollowupStateBinding(input) {
|
||||
}
|
||||
const subject = withCappedLength(compactWhitespace(input.investigationState.focus.active_query_subject ?? ""), FOLLOWUP_SUBJECT_MAX);
|
||||
const businessContextPatch = ["followup_state_binding_v1"];
|
||||
let problemContinuityApplied = false;
|
||||
let problemContinuitySkippedReason = null;
|
||||
if (input.investigationState.focus.period) {
|
||||
businessContextPatch.push("active_period");
|
||||
}
|
||||
@@ -729,6 +771,21 @@ function buildFollowupStateBinding(input) {
|
||||
if (input.investigationState.focus.primary_accounts.length > 0) {
|
||||
businessContextPatch.push(`focus_accounts:${input.investigationState.focus.primary_accounts.join(",")}`);
|
||||
}
|
||||
if (problemContinuityAvailable) {
|
||||
if (hasExplicitExpectedRoute) {
|
||||
problemContinuitySkippedReason = "explicit_expected_route";
|
||||
}
|
||||
else {
|
||||
const focusTypes = (problemState?.focus_problem_types ?? []).slice(0, 3);
|
||||
const activeCount = problemState?.active_problem_units.length ?? 0;
|
||||
businessContextPatch.push("problem_unit_continuity_v1");
|
||||
if (focusTypes.length > 0) {
|
||||
businessContextPatch.push(`problem_focus_types:${focusTypes.join(",")}`);
|
||||
}
|
||||
businessContextPatch.push(`problem_active_count:${activeCount}`);
|
||||
problemContinuityApplied = true;
|
||||
}
|
||||
}
|
||||
const mergedBusinessContext = mergeBusinessContext(context?.business_context, businessContextPatch);
|
||||
if (mergedBusinessContext) {
|
||||
context.business_context = mergedBusinessContext;
|
||||
@@ -743,6 +800,9 @@ function buildFollowupStateBinding(input) {
|
||||
if (periodHintFromState && !hasPeriodLiteral(userMessage)) {
|
||||
appendParts.push(`Период фокуса: ${periodHintFromState}`);
|
||||
}
|
||||
if (problemContinuityApplied && (problemState?.focus_problem_types.length ?? 0) > 0) {
|
||||
appendParts.push(`Problem focus types: ${(problemState?.focus_problem_types ?? []).slice(0, 3).join(", ")}`);
|
||||
}
|
||||
const appendBlock = withCappedLength(compactWhitespace(appendParts.join("; ")), FOLLOWUP_QUESTION_APPEND_MAX);
|
||||
normalizedQuestion = `${userMessage}\n${appendBlock}`.trim();
|
||||
}
|
||||
@@ -762,7 +822,11 @@ function buildFollowupStateBinding(input) {
|
||||
period_hint_from_state: Boolean(periodHintFromState),
|
||||
expected_route_from_state: Boolean(expectedRouteFromState),
|
||||
business_context_from_state: Boolean(mergedBusinessContext),
|
||||
question_augmented: shouldAugmentQuestion
|
||||
question_augmented: shouldAugmentQuestion,
|
||||
problem_continuity_available: problemContinuityAvailable,
|
||||
problem_continuity_applied: problemContinuityApplied,
|
||||
problem_continuity_skipped_reason: problemContinuityApplied ? null : problemContinuitySkippedReason,
|
||||
strong_new_anchor_detected: strongNewAnchorDetected
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -903,7 +967,8 @@ class AssistantService {
|
||||
requirements: coverageEvaluation.requirements,
|
||||
coverageReport: coverageEvaluation.coverage,
|
||||
groundingCheck,
|
||||
enableAnswerPolicyV11: config_1.FEATURE_ASSISTANT_ANSWER_POLICY_V11
|
||||
enableAnswerPolicyV11: config_1.FEATURE_ASSISTANT_ANSWER_POLICY_V11,
|
||||
enableProblemCentricAnswerV1: config_1.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1
|
||||
});
|
||||
const answerStructureV11 = config_1.FEATURE_ASSISTANT_CONTRACTS_V11
|
||||
? config_1.FEATURE_ASSISTANT_ANSWER_POLICY_V11 && composition.answer_structure_v11
|
||||
@@ -952,6 +1017,14 @@ class AssistantService {
|
||||
answer_grounding_check: groundingCheck,
|
||||
dropped_intent_segments: extractDiscardedIntentSegments(normalized.normalized),
|
||||
...(followupBinding.usage ? { followup_state_usage: followupBinding.usage } : {}),
|
||||
problem_centric_answer_applied: composition.problem_centric_answer_applied ?? false,
|
||||
problem_units_used_count: composition.problem_units_used_count ?? 0,
|
||||
problem_answer_mode: composition.problem_answer_mode ?? "stage1_policy_v11",
|
||||
...(Array.isArray(composition.problem_unit_ids_used) && composition.problem_unit_ids_used.length > 0
|
||||
? {
|
||||
problem_unit_ids_used: composition.problem_unit_ids_used
|
||||
}
|
||||
: {}),
|
||||
answer_structure_v11: answerStructureV11,
|
||||
investigation_state_snapshot: investigationStateSnapshot,
|
||||
normalized: normalized.normalized
|
||||
@@ -1007,6 +1080,14 @@ class AssistantService {
|
||||
clarification_target: coverageEvaluation.coverage.clarification_needed_for,
|
||||
dropped_intent_segments: extractDiscardedIntentSegments(normalized.normalized),
|
||||
...(followupBinding.usage ? { followup_state_usage: followupBinding.usage } : {}),
|
||||
problem_centric_answer_applied: composition.problem_centric_answer_applied ?? false,
|
||||
problem_units_used_count: composition.problem_units_used_count ?? 0,
|
||||
problem_answer_mode: composition.problem_answer_mode ?? "stage1_policy_v11",
|
||||
...(Array.isArray(composition.problem_unit_ids_used) && composition.problem_unit_ids_used.length > 0
|
||||
? {
|
||||
problem_unit_ids_used: composition.problem_unit_ids_used
|
||||
}
|
||||
: {}),
|
||||
answer_structure_v11: answerStructureV11,
|
||||
investigation_state_snapshot: investigationStateSnapshot,
|
||||
fallback_type: composition.fallback_type,
|
||||
|
||||
+780
-1
@@ -9,6 +9,7 @@ const path_1 = __importDefault(require("path"));
|
||||
const nanoid_1 = require("nanoid");
|
||||
const config_1 = require("../config");
|
||||
const stage1Contracts_1 = require("../types/stage1Contracts");
|
||||
const stage2EvalContracts_1 = require("../types/stage2EvalContracts");
|
||||
const http_1 = require("../utils/http");
|
||||
const assistantService_1 = require("./assistantService");
|
||||
const assistantSessionStore_1 = require("./assistantSessionStore");
|
||||
@@ -214,6 +215,20 @@ function isDecisionStateConsistent(decision) {
|
||||
const DEFAULT_ASSISTANT_STAGE1_SUITE_FILE = "assistant_stage1_canonical_v0_1.json";
|
||||
const ASSISTANT_STAGE1_RUN_SCHEMA_VERSION = "assistant_stage1_eval_run_v0_1";
|
||||
const ASSISTANT_STAGE1_COMPARISON_SCHEMA_VERSION = "assistant_stage1_eval_comparison_v0_1";
|
||||
const DEFAULT_ASSISTANT_STAGE2_SUITE_FILE = "assistant_stage2_canonical_v0_1.json";
|
||||
const ASSISTANT_STAGE2_RUN_SCHEMA_VERSION = "assistant_stage2_eval_run_v0_1";
|
||||
const ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION = "assistant_stage2_eval_comparison_v0_1";
|
||||
const KNOWN_PROBLEM_UNIT_TYPES = [
|
||||
"document_conflict",
|
||||
"broken_chain_segment",
|
||||
"lifecycle_anomaly_node",
|
||||
"unresolved_settlement_cluster",
|
||||
"period_risk_cluster",
|
||||
"cross_branch_inconsistency_cluster"
|
||||
];
|
||||
function toProblemUnitType(value) {
|
||||
return KNOWN_PROBLEM_UNIT_TYPES.includes(value) ? value : null;
|
||||
}
|
||||
function round2(value) {
|
||||
return Number(value.toFixed(2));
|
||||
}
|
||||
@@ -258,6 +273,44 @@ function rubricBandForMetric(metric, value) {
|
||||
const score = rateToBandScore(metric, value);
|
||||
return stage1Contracts_1.ACCOUNTANT_SCORING_RUBRIC_V01[metric].find((item) => item.score === score) ?? null;
|
||||
}
|
||||
function rateToBandScoreStage2(metric, value) {
|
||||
if (metric === "problem_unit_precision" || metric === "problem_unit_recall_proxy" || metric === "problem_first_answer_rate") {
|
||||
if (value >= 0.75)
|
||||
return 5;
|
||||
if (value >= 0.45)
|
||||
return 3;
|
||||
return 0;
|
||||
}
|
||||
if (metric === "duplicate_collapse_rate") {
|
||||
if (value >= 0.2)
|
||||
return 5;
|
||||
if (value >= 0.08)
|
||||
return 3;
|
||||
return 0;
|
||||
}
|
||||
if (metric === "entity_leakage_rate") {
|
||||
if (value <= 0.2)
|
||||
return 5;
|
||||
if (value <= 0.4)
|
||||
return 3;
|
||||
return 0;
|
||||
}
|
||||
if (metric === "mechanism_coherence_score" || metric === "problem_clarity_score") {
|
||||
if (value >= 4)
|
||||
return 5;
|
||||
if (value >= 2.5)
|
||||
return 3;
|
||||
return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
function rubricBandForMetricStage2(metric, value) {
|
||||
if (value === null) {
|
||||
return null;
|
||||
}
|
||||
const score = rateToBandScoreStage2(metric, value);
|
||||
return stage2EvalContracts_1.ASSISTANT_STAGE2_SCORING_RUBRIC_V01[metric].find((item) => item.score === score) ?? null;
|
||||
}
|
||||
function buildFeatureProfileSnapshot() {
|
||||
return {
|
||||
FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1: config_1.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1,
|
||||
@@ -266,7 +319,11 @@ function buildFeatureProfileSnapshot() {
|
||||
FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1: process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 ?? null,
|
||||
FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1: process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 ?? null,
|
||||
FEATURE_ASSISTANT_INVESTIGATION_STATE_V1: process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 ?? null,
|
||||
FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1: process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 ?? null
|
||||
FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1: process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 ?? null,
|
||||
FEATURE_ASSISTANT_PROBLEM_UNITS_V1: process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 ?? String(config_1.FEATURE_ASSISTANT_PROBLEM_UNITS_V1),
|
||||
FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1: process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 ?? String(config_1.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1),
|
||||
FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1: process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 ?? String(config_1.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1),
|
||||
FEATURE_ASSISTANT_STAGE2_EVAL_V1: process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 ?? String(config_1.FEATURE_ASSISTANT_STAGE2_EVAL_V1)
|
||||
};
|
||||
}
|
||||
function buildCodeVersionMarker() {
|
||||
@@ -331,6 +388,41 @@ function parseAssistantSuiteFile(inputPath) {
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
function parseAssistantStage2SuiteFile(inputPath) {
|
||||
const filePath = resolveReadablePath(inputPath ?? DEFAULT_ASSISTANT_STAGE2_SUITE_FILE);
|
||||
const raw = fs_1.default.readFileSync(filePath, "utf-8").replace(/^\uFEFF/, "");
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
throw new Error(`Invalid assistant stage2 suite format: ${filePath}`);
|
||||
}
|
||||
if (!Array.isArray(parsed.cases)) {
|
||||
throw new Error(`Assistant stage2 suite cases[] is required: ${filePath}`);
|
||||
}
|
||||
if (!Array.isArray(parsed.case_ids)) {
|
||||
throw new Error(`Assistant stage2 suite case_ids[] is required: ${filePath}`);
|
||||
}
|
||||
if (typeof parsed.suite_id !== "string" || !parsed.suite_id.trim()) {
|
||||
throw new Error(`Assistant stage2 suite_id is required: ${filePath}`);
|
||||
}
|
||||
if (typeof parsed.suite_version !== "string" || !parsed.suite_version.trim()) {
|
||||
throw new Error(`Assistant stage2 suite_version is required: ${filePath}`);
|
||||
}
|
||||
if (parsed.scenario_count !== parsed.cases.length) {
|
||||
throw new Error(`Assistant stage2 scenario_count mismatch: ${filePath}`);
|
||||
}
|
||||
const declaredIds = [...parsed.case_ids].sort();
|
||||
const actualIds = parsed.cases.map((item) => item.case_id).sort();
|
||||
const idsMatch = declaredIds.length === actualIds.length && declaredIds.every((item, index) => item === actualIds[index]);
|
||||
if (!idsMatch) {
|
||||
throw new Error(`Assistant stage2 case_ids do not match cases[]: ${filePath}`);
|
||||
}
|
||||
for (const item of parsed.cases) {
|
||||
if (!Array.isArray(item.turns) || item.turns.length === 0) {
|
||||
throw new Error(`Assistant stage2 case ${item.case_id} must include at least one turn.`);
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
function hasDomainAnchors(text) {
|
||||
const source = String(text ?? "");
|
||||
if (!source.trim()) {
|
||||
@@ -342,6 +434,16 @@ function hasDomainAnchors(text) {
|
||||
const hits = [hasPeriod, hasAccountingObject, hasAccountCode].filter(Boolean).length;
|
||||
return hits >= 2;
|
||||
}
|
||||
function detectEntityLeakage(text) {
|
||||
const source = String(text ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
const uuidHits = source.match(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi)?.length ?? 0;
|
||||
const guidHits = source.match(/\b(?:guid|uuid|entity_id|source_ref|canonical_ref|fragment_id)\b/gi)?.length ?? 0;
|
||||
const longHexHits = source.match(/\b[0-9a-f]{24,}\b/gi)?.length ?? 0;
|
||||
return uuidHits > 0 || guidHits > 1 || longHexHits > 0;
|
||||
}
|
||||
function extractTextList(value) {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
@@ -411,6 +513,50 @@ function buildAssistantEvalMarkdownReport(report) {
|
||||
""
|
||||
].join("\n");
|
||||
}
|
||||
function buildAssistantStage2EvalMarkdownReport(report) {
|
||||
const metrics = (report.metrics ?? {}).raw ?? {};
|
||||
const bands = (report.rubric_bands ?? {});
|
||||
const subsets = (report.subsets ?? {});
|
||||
const scenarioSummary = (report.scenario_summary ?? {});
|
||||
const rows = Object.keys(metrics)
|
||||
.map((key) => {
|
||||
const rawValue = metrics[key];
|
||||
const band = bands[key];
|
||||
const rawPrintable = rawValue === null || rawValue === undefined ? "n/a" : String(rawValue);
|
||||
const bandPrintable = band ? `${String(band.score)} (${String(band.label)})` : "n/a";
|
||||
return `| ${key} | ${rawPrintable} | ${bandPrintable} |`;
|
||||
})
|
||||
.join("\n");
|
||||
return [
|
||||
`# ${String(report.report_title ?? "Assistant Stage 2 Eval Run")}`,
|
||||
"",
|
||||
`- run_id: ${String(report.run_id ?? "")}`,
|
||||
`- eval_target: ${String(report.eval_target ?? "")}`,
|
||||
`- run_timestamp: ${String(report.run_timestamp ?? "")}`,
|
||||
`- suite_id: ${String(report.suite_id ?? "")}`,
|
||||
`- suite_version: ${String(report.suite_version ?? "")}`,
|
||||
`- cases_total: ${String(report.cases_total ?? 0)}`,
|
||||
"",
|
||||
"## Raw Metrics and Rubric Bands",
|
||||
"",
|
||||
"| Metric | Raw | Rubric band |",
|
||||
"|---|---:|---|",
|
||||
rows || "| n/a | n/a | n/a |",
|
||||
"",
|
||||
"## Subsets",
|
||||
"",
|
||||
`- expected_problem_cases_total: ${String(subsets.expected_problem_cases_total ?? 0)}`,
|
||||
`- followup_cases_total: ${String(subsets.followup_cases_total ?? 0)}`,
|
||||
`- candidate_cases_total: ${String(subsets.candidate_cases_total ?? 0)}`,
|
||||
"",
|
||||
"## Scenario Summary",
|
||||
"",
|
||||
`- improved_or_strong: ${String(scenarioSummary.improved_or_strong ?? 0)}`,
|
||||
`- unchanged_or_mixed: ${String(scenarioSummary.unchanged_or_mixed ?? 0)}`,
|
||||
`- weak_or_regressed: ${String(scenarioSummary.weak_or_regressed ?? 0)}`,
|
||||
""
|
||||
].join("\n");
|
||||
}
|
||||
function buildAssistantComparisonMarkdownReport(report) {
|
||||
const metrics = (report.metric_deltas ?? {});
|
||||
const summary = (report.scenario_notes_summary ?? {});
|
||||
@@ -442,6 +588,37 @@ function buildAssistantComparisonMarkdownReport(report) {
|
||||
""
|
||||
].join("\n");
|
||||
}
|
||||
function buildAssistantStage2ComparisonMarkdownReport(report) {
|
||||
const metrics = (report.metric_deltas ?? {});
|
||||
const summary = (report.scenario_notes_summary ?? {});
|
||||
const rows = Object.keys(metrics)
|
||||
.map((key) => {
|
||||
const row = metrics[key];
|
||||
return `| ${key} | ${String(row.baseline ?? "n/a")} | ${String(row.current ?? "n/a")} | ${String(row.delta ?? "n/a")} | ${String(row.trend ?? "n/a")} |`;
|
||||
})
|
||||
.join("\n");
|
||||
return [
|
||||
`# ${String(report.report_title ?? "Assistant Stage 2 Baseline vs Current")}`,
|
||||
"",
|
||||
`- comparison_id: ${String(report.comparison_id ?? "")}`,
|
||||
`- baseline_run_id: ${String(report.baseline_run_id ?? "")}`,
|
||||
`- current_run_id: ${String(report.current_run_id ?? "")}`,
|
||||
`- suite_version: ${String(report.suite_version ?? "")}`,
|
||||
"",
|
||||
"## Metric Deltas",
|
||||
"",
|
||||
"| Metric | Baseline | Current | Delta | Trend |",
|
||||
"|---|---:|---:|---:|---|",
|
||||
rows || "| n/a | n/a | n/a | n/a | n/a |",
|
||||
"",
|
||||
"## Scenario Notes Summary",
|
||||
"",
|
||||
`- improved: ${String(summary.improved ?? 0)}`,
|
||||
`- unchanged: ${String(summary.unchanged ?? 0)}`,
|
||||
`- weakened: ${String(summary.weakened ?? 0)}`,
|
||||
""
|
||||
].join("\n");
|
||||
}
|
||||
class EvalService {
|
||||
normalizerService;
|
||||
constructor(normalizerService) {
|
||||
@@ -806,6 +983,148 @@ class EvalService {
|
||||
uncertainty_limitations_count: uncertaintyLimitationsCount
|
||||
};
|
||||
}
|
||||
collectAssistantStage2Signals(finalResponse, turnResponses) {
|
||||
const base = this.collectAssistantSignals(finalResponse, turnResponses);
|
||||
const debug = finalResponse.debug;
|
||||
const retrievalResults = Array.isArray(debug?.retrieval_results) ? debug.retrieval_results : [];
|
||||
const typeSet = new Set();
|
||||
const mechanismSummaries = new Set();
|
||||
let candidateEvidenceTotal = 0;
|
||||
let problemUnitsTotal = 0;
|
||||
let duplicateCollapsesTotal = 0;
|
||||
for (const result of retrievalResults) {
|
||||
const candidates = Array.isArray(result.candidate_evidence) ? result.candidate_evidence : [];
|
||||
candidateEvidenceTotal += candidates.length;
|
||||
const problemUnits = Array.isArray(result.problem_units) ? result.problem_units : [];
|
||||
problemUnitsTotal += problemUnits.length;
|
||||
for (const unit of problemUnits) {
|
||||
const unitType = toProblemUnitType(unit.problem_unit_type);
|
||||
if (unitType) {
|
||||
typeSet.add(unitType);
|
||||
}
|
||||
const mechanismSummary = String(unit.mechanism_summary ?? "").trim();
|
||||
if (mechanismSummary) {
|
||||
mechanismSummaries.add(mechanismSummary);
|
||||
}
|
||||
}
|
||||
if (result.problem_unit_summary && typeof result.problem_unit_summary.duplicate_collapses === "number") {
|
||||
duplicateCollapsesTotal += Number(result.problem_unit_summary.duplicate_collapses);
|
||||
}
|
||||
}
|
||||
const answerMode = typeof debug?.problem_answer_mode === "string" ? debug.problem_answer_mode : null;
|
||||
const unitsUsedCount = Number(debug?.problem_units_used_count ?? 0);
|
||||
const unitIdsUsed = Array.isArray(debug?.problem_unit_ids_used)
|
||||
? debug.problem_unit_ids_used
|
||||
.map((item) => String(item ?? "").trim())
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
const problemCentricApplied = debug?.problem_centric_answer_applied === true || answerMode === "stage2_problem_centric_v1";
|
||||
return {
|
||||
...base,
|
||||
candidate_evidence_total: candidateEvidenceTotal,
|
||||
problem_units_total: problemUnitsTotal,
|
||||
problem_unit_types: [...typeSet],
|
||||
problem_mechanism_summaries: [...mechanismSummaries],
|
||||
duplicate_collapses_total: duplicateCollapsesTotal,
|
||||
problem_centric_answer_applied: problemCentricApplied,
|
||||
problem_units_used_count: unitsUsedCount,
|
||||
problem_answer_mode: answerMode,
|
||||
problem_unit_ids_used: unitIdsUsed,
|
||||
entity_leakage_detected: detectEntityLeakage(String(finalResponse.assistant_reply ?? ""))
|
||||
};
|
||||
}
|
||||
getExpectedProblemUnitTypes(suiteCase) {
|
||||
const expected = Array.isArray(suiteCase.expected_hints?.expected_problem_unit_types)
|
||||
? suiteCase.expected_hints?.expected_problem_unit_types
|
||||
: [];
|
||||
const output = new Set();
|
||||
for (const value of expected ?? []) {
|
||||
const mapped = toProblemUnitType(value);
|
||||
if (mapped) {
|
||||
output.add(mapped);
|
||||
}
|
||||
}
|
||||
return [...output];
|
||||
}
|
||||
computeProblemUnitPrecision(expectedTypes, detectedTypes) {
|
||||
const uniqueExpected = [...new Set(expectedTypes)];
|
||||
const uniqueDetected = [...new Set(detectedTypes)];
|
||||
if (uniqueDetected.length === 0) {
|
||||
return uniqueExpected.length === 0 ? 1 : 0;
|
||||
}
|
||||
if (uniqueExpected.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const matchedDetected = uniqueDetected.filter((item) => uniqueExpected.includes(item)).length;
|
||||
return round2(matchedDetected / uniqueDetected.length);
|
||||
}
|
||||
computeProblemUnitRecallProxy(expectedTypes, detectedTypes) {
|
||||
const uniqueExpected = [...new Set(expectedTypes)];
|
||||
const uniqueDetected = [...new Set(detectedTypes)];
|
||||
if (uniqueExpected.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (uniqueDetected.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const matchedExpected = uniqueExpected.filter((item) => uniqueDetected.includes(item)).length;
|
||||
return round2(matchedExpected / uniqueExpected.length);
|
||||
}
|
||||
computeDuplicateCollapseRate(candidateTotal, duplicateCollapses) {
|
||||
if (candidateTotal <= 0) {
|
||||
return null;
|
||||
}
|
||||
return round2(Math.min(1, Math.max(0, duplicateCollapses / candidateTotal)));
|
||||
}
|
||||
computeMechanismCoherenceScore(finalResponse, signals) {
|
||||
const mechanismBlock = finalResponse.debug?.answer_structure_v11?.mechanism_block;
|
||||
const mechanismStatus = mechanismBlock?.status;
|
||||
const mechanismNotes = extractTextList(mechanismBlock?.mechanism_notes);
|
||||
const hasProblemMechanism = signals.problem_mechanism_summaries.length > 0;
|
||||
let score = 0;
|
||||
if (mechanismStatus === "grounded" && hasProblemMechanism && mechanismNotes.length > 0) {
|
||||
score = 5;
|
||||
}
|
||||
else if ((mechanismStatus === "limited" || mechanismStatus === "unresolved") && (hasProblemMechanism || mechanismNotes.length > 0)) {
|
||||
score = 3;
|
||||
}
|
||||
else if (hasProblemMechanism || mechanismNotes.length > 0) {
|
||||
score = 2;
|
||||
}
|
||||
if (mechanismStatus === "grounded" && !hasProblemMechanism) {
|
||||
score = Math.min(score, 2);
|
||||
}
|
||||
if (signals.limitation_reason_codes.includes("missing_mechanism")) {
|
||||
score -= 1;
|
||||
}
|
||||
return clampScore(score);
|
||||
}
|
||||
computeProblemClarityScore(finalResponse, signals) {
|
||||
const structure = finalResponse.debug?.answer_structure_v11;
|
||||
const answerSummary = String(structure?.answer_summary ?? "").trim();
|
||||
const directAnswer = String(structure?.direct_answer ?? finalResponse.assistant_reply ?? "").trim();
|
||||
const recommendedActions = extractTextList(structure?.next_step_block?.recommended_actions);
|
||||
const clarificationQuestions = extractTextList(structure?.next_step_block?.clarification_questions);
|
||||
const uncertaintyLimitations = extractTextList(structure?.uncertainty_block?.limitations);
|
||||
let score = 0;
|
||||
if (answerSummary.length > 20)
|
||||
score += 1;
|
||||
if (directAnswer.length > 20)
|
||||
score += 1;
|
||||
if (hasDomainAnchors(`${answerSummary} ${directAnswer}`))
|
||||
score += 1;
|
||||
if (recommendedActions.length > 0 || clarificationQuestions.length > 0)
|
||||
score += 1;
|
||||
if (signals.problem_units_total > 0 || signals.problem_centric_answer_applied)
|
||||
score += 1;
|
||||
if ((signals.minimum_evidence_failed || signals.degraded_to === "clarification") && uncertaintyLimitations.length === 0) {
|
||||
score -= 1;
|
||||
}
|
||||
if (signals.entity_leakage_detected) {
|
||||
score -= 1;
|
||||
}
|
||||
return clampScore(score);
|
||||
}
|
||||
computeAssistantMetrics(input) {
|
||||
const diagnostics = input.diagnostics;
|
||||
const total = Math.max(1, diagnostics.length);
|
||||
@@ -855,6 +1174,68 @@ class EvalService {
|
||||
signature_counts: signatureCounter
|
||||
};
|
||||
}
|
||||
computeAssistantStage2Metrics(input) {
|
||||
const diagnostics = input.diagnostics;
|
||||
const signatureCounter = diagnostics.reduce((acc, item) => {
|
||||
acc[item.signature] = (acc[item.signature] ?? 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
const precisionValues = diagnostics
|
||||
.map((item) => item.problem_unit_precision)
|
||||
.filter((item) => typeof item === "number");
|
||||
const recallValues = diagnostics
|
||||
.map((item) => item.problem_unit_recall_proxy)
|
||||
.filter((item) => typeof item === "number");
|
||||
const collapseValues = diagnostics
|
||||
.map((item) => item.duplicate_collapse_rate)
|
||||
.filter((item) => typeof item === "number");
|
||||
const mechanismValues = diagnostics.map((item) => item.mechanism_coherence_score);
|
||||
const clarityValues = diagnostics.map((item) => item.problem_clarity_score);
|
||||
const firstApplicable = diagnostics.filter((item) => item.problem_first_answer_applied !== null);
|
||||
const firstApplied = firstApplicable.filter((item) => item.problem_first_answer_applied === true).length;
|
||||
const leakageCases = diagnostics.filter((item) => item.entity_leakage).length;
|
||||
const followupCases = diagnostics.filter((item) => item.suite_case.question_type === "followup" || item.turn_count > 1);
|
||||
const candidateCases = diagnostics.filter((item) => item.signals.candidate_evidence_total > 0);
|
||||
const expectedProblemCases = diagnostics.filter((item) => item.expected_problem_first);
|
||||
const average = (values) => {
|
||||
if (values.length === 0)
|
||||
return null;
|
||||
return round2(values.reduce((acc, item) => acc + item, 0) / values.length);
|
||||
};
|
||||
const raw = {
|
||||
problem_unit_precision: average(precisionValues),
|
||||
problem_unit_recall_proxy: average(recallValues),
|
||||
duplicate_collapse_rate: average(collapseValues),
|
||||
mechanism_coherence_score: average(mechanismValues),
|
||||
problem_clarity_score: average(clarityValues),
|
||||
problem_first_answer_rate: firstApplicable.length > 0 ? round2(firstApplied / firstApplicable.length) : null,
|
||||
entity_leakage_rate: diagnostics.length > 0 ? round2(leakageCases / diagnostics.length) : null
|
||||
};
|
||||
const rubric_bands = {
|
||||
problem_unit_precision: rubricBandForMetricStage2("problem_unit_precision", raw.problem_unit_precision),
|
||||
problem_unit_recall_proxy: rubricBandForMetricStage2("problem_unit_recall_proxy", raw.problem_unit_recall_proxy),
|
||||
duplicate_collapse_rate: rubricBandForMetricStage2("duplicate_collapse_rate", raw.duplicate_collapse_rate),
|
||||
mechanism_coherence_score: rubricBandForMetricStage2("mechanism_coherence_score", raw.mechanism_coherence_score),
|
||||
problem_clarity_score: rubricBandForMetricStage2("problem_clarity_score", raw.problem_clarity_score),
|
||||
problem_first_answer_rate: rubricBandForMetricStage2("problem_first_answer_rate", raw.problem_first_answer_rate),
|
||||
entity_leakage_rate: rubricBandForMetricStage2("entity_leakage_rate", raw.entity_leakage_rate)
|
||||
};
|
||||
return {
|
||||
raw,
|
||||
rubric_bands,
|
||||
denominators: {
|
||||
cases_total: diagnostics.length,
|
||||
expected_problem_cases_total: expectedProblemCases.length,
|
||||
followup_cases_total: followupCases.length,
|
||||
candidate_cases_total: candidateCases.length,
|
||||
precision_cases_total: precisionValues.length,
|
||||
recall_cases_total: recallValues.length,
|
||||
duplicate_collapse_cases_total: collapseValues.length,
|
||||
problem_first_applicable_cases_total: firstApplicable.length
|
||||
},
|
||||
signature_counts: signatureCounter
|
||||
};
|
||||
}
|
||||
buildAssistantComparisonReport(input) {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
|
||||
@@ -959,6 +1340,122 @@ class EvalService {
|
||||
}
|
||||
};
|
||||
}
|
||||
buildAssistantStage2ComparisonReport(input) {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
|
||||
const currentReport = input.currentReport;
|
||||
const metricKeys = [
|
||||
"problem_unit_precision",
|
||||
"problem_unit_recall_proxy",
|
||||
"duplicate_collapse_rate",
|
||||
"mechanism_coherence_score",
|
||||
"problem_clarity_score",
|
||||
"problem_first_answer_rate",
|
||||
"entity_leakage_rate"
|
||||
];
|
||||
const lowerIsBetter = new Set(["entity_leakage_rate"]);
|
||||
const baselineRaw = (baselineReport.metrics ?? {}).raw ?? {};
|
||||
const currentRaw = (currentReport.metrics ?? {}).raw ?? {};
|
||||
const deltas = {};
|
||||
for (const metric of metricKeys) {
|
||||
const baseline = typeof baselineRaw[metric] === "number" ? Number(baselineRaw[metric]) : null;
|
||||
const current = typeof currentRaw[metric] === "number" ? Number(currentRaw[metric]) : null;
|
||||
const delta = baseline !== null && current !== null ? round2(current - baseline) : null;
|
||||
let trend = "n/a";
|
||||
if (baseline !== null && current !== null) {
|
||||
const improved = lowerIsBetter.has(metric) ? current < baseline - 0.01 : current > baseline + 0.01;
|
||||
const weakened = lowerIsBetter.has(metric) ? current > baseline + 0.01 : current < baseline - 0.01;
|
||||
trend = improved ? "improved" : weakened ? "weakened" : "unchanged";
|
||||
}
|
||||
deltas[metric] = { baseline, current, delta, trend };
|
||||
}
|
||||
const baselineResults = Array.isArray(baselineReport.results) ? baselineReport.results : [];
|
||||
const currentResults = Array.isArray(currentReport.results) ? currentReport.results : [];
|
||||
const baselineByCase = new Map();
|
||||
for (const row of baselineResults) {
|
||||
baselineByCase.set(String(row.case_id ?? ""), row);
|
||||
}
|
||||
const improvedNotes = [];
|
||||
const unchangedNotes = [];
|
||||
const weakenedNotes = [];
|
||||
const toComposite = (row) => {
|
||||
if (!row || typeof row !== "object")
|
||||
return null;
|
||||
const metricSubscores = row.metric_subscores;
|
||||
if (!metricSubscores)
|
||||
return null;
|
||||
const clarity = typeof metricSubscores.problem_clarity_score === "number" ? Number(metricSubscores.problem_clarity_score) : null;
|
||||
const mechanism = typeof metricSubscores.mechanism_coherence_score === "number" ? Number(metricSubscores.mechanism_coherence_score) : null;
|
||||
const firstRate = typeof metricSubscores.problem_first_answer_rate === "number" ? Number(metricSubscores.problem_first_answer_rate) : null;
|
||||
const leakageRate = typeof metricSubscores.entity_leakage_rate === "number" ? Number(metricSubscores.entity_leakage_rate) : null;
|
||||
if (clarity === null || mechanism === null || firstRate === null || leakageRate === null) {
|
||||
return null;
|
||||
}
|
||||
return round2((clarity + mechanism + firstRate * 5 + (1 - leakageRate) * 5) / 4);
|
||||
};
|
||||
for (const row of currentResults) {
|
||||
const caseId = String(row.case_id ?? "");
|
||||
const currentComposite = toComposite(row);
|
||||
const baselineComposite = toComposite(baselineByCase.get(caseId));
|
||||
if (currentComposite === null || baselineComposite === null) {
|
||||
continue;
|
||||
}
|
||||
const delta = round2(currentComposite - baselineComposite);
|
||||
const note = `${caseId}: composite ${baselineComposite} -> ${currentComposite} (delta ${delta})`;
|
||||
if (delta > 0.25) {
|
||||
improvedNotes.push(note);
|
||||
}
|
||||
else if (delta < -0.25) {
|
||||
weakenedNotes.push(note);
|
||||
}
|
||||
else {
|
||||
unchangedNotes.push(note);
|
||||
}
|
||||
}
|
||||
const comparisonId = `assistant-stage2-compare-${(0, nanoid_1.nanoid)(8)}`;
|
||||
const comparisonReport = {
|
||||
schema_version: ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION,
|
||||
comparison_id: comparisonId,
|
||||
run_timestamp: new Date().toISOString(),
|
||||
baseline_run_id: baselineReport.run_id ?? null,
|
||||
current_run_id: currentReport.run_id ?? null,
|
||||
eval_target: "assistant_stage2",
|
||||
suite_id: currentReport.suite_id ?? baselineReport.suite_id ?? null,
|
||||
suite_version: currentReport.suite_version ?? baselineReport.suite_version ?? null,
|
||||
baseline_report_file: baselinePath,
|
||||
current_report_file: currentReport.artifacts && typeof currentReport.artifacts === "object"
|
||||
? currentReport.artifacts.run_report_json_path ?? null
|
||||
: null,
|
||||
metric_deltas: deltas,
|
||||
scenario_notes_summary: {
|
||||
improved: improvedNotes.length,
|
||||
unchanged: unchangedNotes.length,
|
||||
weakened: weakenedNotes.length
|
||||
},
|
||||
scenario_notes: {
|
||||
improved: improvedNotes,
|
||||
unchanged: unchangedNotes,
|
||||
weakened: weakenedNotes
|
||||
},
|
||||
known_limitations: currentReport.known_limitations ?? [
|
||||
"Stage 2 comparison remains run-to-run and depends on stable feature profile.",
|
||||
"Metrics are Stage 2 Wave 5 heuristics, not final product scorecards."
|
||||
],
|
||||
report_title: "Assistant Stage 2 Baseline vs Current"
|
||||
};
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
const jsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.json`);
|
||||
const mdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.md`);
|
||||
(0, files_1.writeJsonFile)(jsonPath, comparisonReport);
|
||||
fs_1.default.writeFileSync(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport), "utf-8");
|
||||
return {
|
||||
...comparisonReport,
|
||||
artifacts: {
|
||||
comparison_report_json_path: jsonPath,
|
||||
comparison_report_md_path: mdPath
|
||||
}
|
||||
};
|
||||
}
|
||||
async runAssistantStage1(payload) {
|
||||
if (!config_1.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1) {
|
||||
throw new http_1.ApiError("ASSISTANT_STAGE1_EVAL_DISABLED", "Assistant Stage 1 eval target is disabled by FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1.", 409);
|
||||
@@ -1290,6 +1787,278 @@ class EvalService {
|
||||
}
|
||||
return report;
|
||||
}
|
||||
async runAssistantStage2(payload) {
|
||||
if (!config_1.FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
|
||||
throw new http_1.ApiError("ASSISTANT_STAGE2_EVAL_DISABLED", "Assistant Stage 2 eval target is disabled by FEATURE_ASSISTANT_STAGE2_EVAL_V1.", 409);
|
||||
}
|
||||
const suite = parseAssistantStage2SuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage2-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const assistantService = new assistantService_1.AssistantService(this.normalizerService, new assistantSessionStore_1.AssistantSessionStore());
|
||||
const diagnostics = [];
|
||||
let requestsTotal = 0;
|
||||
for (const suiteCase of suiteCases) {
|
||||
const sessionId = `${runId}-${suiteCase.case_id}`;
|
||||
const turnResponses = [];
|
||||
const notes = [];
|
||||
const limitations = [];
|
||||
const expectedProblemUnitTypes = this.getExpectedProblemUnitTypes(suiteCase);
|
||||
const expectedProblemFirst = suiteCase.expected_hints?.expected_problem_first ?? (suiteCase.broadness_level !== "low" || suiteCase.question_type !== "direct");
|
||||
try {
|
||||
for (const turn of suiteCase.turns) {
|
||||
const response = await assistantService.handleMessage({
|
||||
session_id: sessionId,
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
temperature: payload.normalizeConfig.temperature,
|
||||
maxOutputTokens: payload.normalizeConfig.maxOutputTokens,
|
||||
promptVersion: payload.normalizeConfig.promptVersion,
|
||||
systemPrompt: payload.normalizeConfig.systemPrompt,
|
||||
developerPrompt: payload.normalizeConfig.developerPrompt,
|
||||
domainPrompt: payload.normalizeConfig.domainPrompt,
|
||||
fewShotExamples: payload.normalizeConfig.fewShotExamples,
|
||||
useMock: payload.useMock
|
||||
});
|
||||
turnResponses.push(response);
|
||||
requestsTotal += 1;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
diagnostics.push({
|
||||
suite_case: suiteCase,
|
||||
session_id: sessionId,
|
||||
trace_id: null,
|
||||
final_reply_type: "backend_error",
|
||||
turn_count: turnResponses.length,
|
||||
signature: `backend_error|${suiteCase.scenario_tag}`,
|
||||
expected_problem_unit_types: expectedProblemUnitTypes,
|
||||
expected_problem_first: expectedProblemFirst,
|
||||
problem_unit_precision: 0,
|
||||
problem_unit_recall_proxy: expectedProblemUnitTypes.length > 0 ? 0 : null,
|
||||
duplicate_collapse_rate: null,
|
||||
mechanism_coherence_score: 0,
|
||||
problem_clarity_score: 0,
|
||||
problem_first_answer_applied: expectedProblemFirst ? false : null,
|
||||
entity_leakage: false,
|
||||
signals: {
|
||||
broad_query_detected: suiteCase.broadness_level !== "low",
|
||||
broad_result_flag: false,
|
||||
narrowing_strength: null,
|
||||
minimum_evidence_failed: true,
|
||||
degraded_to: "clarification",
|
||||
evidence_confidence: "low",
|
||||
limitation_reason_codes: [],
|
||||
mechanism_status: null,
|
||||
source_refs: [],
|
||||
routes: [],
|
||||
followup_state_applied: false,
|
||||
uncertainty_limitations_count: 0,
|
||||
candidate_evidence_total: 0,
|
||||
problem_units_total: 0,
|
||||
problem_unit_types: [],
|
||||
problem_mechanism_summaries: [],
|
||||
duplicate_collapses_total: 0,
|
||||
problem_centric_answer_applied: false,
|
||||
problem_units_used_count: 0,
|
||||
problem_answer_mode: null,
|
||||
problem_unit_ids_used: [],
|
||||
entity_leakage_detected: false
|
||||
},
|
||||
limitations: [errorMessage],
|
||||
notes: [`Case execution failed: ${errorMessage}`]
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const finalResponse = turnResponses[turnResponses.length - 1];
|
||||
const signals = this.collectAssistantStage2Signals(finalResponse, turnResponses);
|
||||
const problemUnitPrecision = this.computeProblemUnitPrecision(expectedProblemUnitTypes, signals.problem_unit_types);
|
||||
const problemUnitRecallProxy = this.computeProblemUnitRecallProxy(expectedProblemUnitTypes, signals.problem_unit_types);
|
||||
const duplicateCollapseRate = this.computeDuplicateCollapseRate(signals.candidate_evidence_total, signals.duplicate_collapses_total);
|
||||
const mechanismCoherenceScore = this.computeMechanismCoherenceScore(finalResponse, signals);
|
||||
const problemClarityScore = this.computeProblemClarityScore(finalResponse, signals);
|
||||
const problemFirstAnswerApplied = expectedProblemFirst ? signals.problem_centric_answer_applied && signals.problem_units_used_count > 0 : null;
|
||||
if (signals.problem_units_total === 0 && expectedProblemUnitTypes.length > 0) {
|
||||
limitations.push("missing_problem_units");
|
||||
}
|
||||
if (signals.problem_centric_answer_applied && signals.problem_units_used_count <= 0) {
|
||||
limitations.push("problem_mode_without_units");
|
||||
}
|
||||
limitations.push(...signals.limitation_reason_codes.map((item) => `limitation_reason:${item}`));
|
||||
if (signals.entity_leakage_detected) {
|
||||
limitations.push("entity_leakage_detected");
|
||||
}
|
||||
if (problemFirstAnswerApplied === false)
|
||||
notes.push("problem_first_not_applied");
|
||||
if (signals.problem_units_total === 0)
|
||||
notes.push("problem_units_missing");
|
||||
if (signals.problem_unit_types.length > 0)
|
||||
notes.push(`problem_types:${signals.problem_unit_types.join(",")}`);
|
||||
if (signals.entity_leakage_detected)
|
||||
notes.push("entity_leakage");
|
||||
if (signals.degraded_to === "clarification")
|
||||
notes.push("clarification_degraded");
|
||||
diagnostics.push({
|
||||
suite_case: suiteCase,
|
||||
session_id: sessionId,
|
||||
trace_id: finalResponse.debug?.trace_id ?? null,
|
||||
final_reply_type: finalResponse.reply_type,
|
||||
turn_count: suiteCase.turns.length,
|
||||
signature: [
|
||||
finalResponse.reply_type,
|
||||
signals.problem_answer_mode ?? "unknown",
|
||||
signals.problem_unit_types.sort().join(","),
|
||||
signals.degraded_to ?? "none"
|
||||
].join("|"),
|
||||
expected_problem_unit_types: expectedProblemUnitTypes,
|
||||
expected_problem_first: expectedProblemFirst,
|
||||
problem_unit_precision: problemUnitPrecision,
|
||||
problem_unit_recall_proxy: problemUnitRecallProxy,
|
||||
duplicate_collapse_rate: duplicateCollapseRate,
|
||||
mechanism_coherence_score: mechanismCoherenceScore,
|
||||
problem_clarity_score: problemClarityScore,
|
||||
problem_first_answer_applied: problemFirstAnswerApplied,
|
||||
entity_leakage: signals.entity_leakage_detected,
|
||||
signals,
|
||||
limitations: Array.from(new Set(limitations)),
|
||||
notes
|
||||
});
|
||||
}
|
||||
const metrics = this.computeAssistantStage2Metrics({ diagnostics });
|
||||
const caseRecords = diagnostics.map((item) => {
|
||||
const caseMetricVector = {
|
||||
problem_unit_precision: item.problem_unit_precision,
|
||||
problem_unit_recall_proxy: item.problem_unit_recall_proxy,
|
||||
duplicate_collapse_rate: item.duplicate_collapse_rate,
|
||||
mechanism_coherence_score: round2(item.mechanism_coherence_score),
|
||||
problem_clarity_score: round2(item.problem_clarity_score),
|
||||
problem_first_answer_rate: item.problem_first_answer_applied === null ? null : item.problem_first_answer_applied ? 1 : 0,
|
||||
entity_leakage_rate: item.entity_leakage ? 1 : 0
|
||||
};
|
||||
return {
|
||||
schema_version: stage2EvalContracts_1.ASSISTANT_STAGE2_EVAL_RECORD_SCHEMA_VERSION,
|
||||
created_at: new Date().toISOString(),
|
||||
case_id: item.suite_case.case_id,
|
||||
scenario_tag: item.suite_case.scenario_tag,
|
||||
session_id: item.session_id,
|
||||
trace_id: item.trace_id,
|
||||
question_type: item.suite_case.question_type,
|
||||
broadness_level: item.suite_case.broadness_level,
|
||||
expected_problem_unit_types: item.expected_problem_unit_types,
|
||||
expected_problem_first: item.expected_problem_first,
|
||||
problem_units_detected: item.signals.problem_units_total,
|
||||
candidate_evidence_detected: item.signals.candidate_evidence_total,
|
||||
duplicate_collapses_detected: item.signals.duplicate_collapses_total,
|
||||
metric_subscores: caseMetricVector,
|
||||
raw_signals: {
|
||||
final_reply_type: item.final_reply_type,
|
||||
turn_count: item.turn_count,
|
||||
broad_query_detected: item.signals.broad_query_detected,
|
||||
broad_result_flag: item.signals.broad_result_flag,
|
||||
narrowing_strength: item.signals.narrowing_strength,
|
||||
minimum_evidence_failed: item.signals.minimum_evidence_failed,
|
||||
degraded_to: item.signals.degraded_to,
|
||||
evidence_confidence: item.signals.evidence_confidence,
|
||||
limitation_reason_codes: item.signals.limitation_reason_codes,
|
||||
mechanism_status: item.signals.mechanism_status,
|
||||
source_refs: item.signals.source_refs,
|
||||
routes: item.signals.routes,
|
||||
followup_state_applied: item.signals.followup_state_applied,
|
||||
problem_units_total: item.signals.problem_units_total,
|
||||
candidate_evidence_total: item.signals.candidate_evidence_total,
|
||||
problem_unit_types: item.signals.problem_unit_types,
|
||||
duplicate_collapses_total: item.signals.duplicate_collapses_total,
|
||||
problem_centric_answer_applied: item.signals.problem_centric_answer_applied,
|
||||
problem_units_used_count: item.signals.problem_units_used_count,
|
||||
problem_answer_mode: item.signals.problem_answer_mode,
|
||||
problem_unit_ids_used: item.signals.problem_unit_ids_used,
|
||||
entity_leakage_detected: item.signals.entity_leakage_detected
|
||||
},
|
||||
limitations: item.limitations,
|
||||
notes: item.notes
|
||||
};
|
||||
});
|
||||
const strongestSignals = Object.entries(metrics.rubric_bands)
|
||||
.filter(([, band]) => band?.score === 5)
|
||||
.map(([name]) => name);
|
||||
const weakestSignals = Object.entries(metrics.rubric_bands)
|
||||
.filter(([, band]) => band?.score === 0)
|
||||
.map(([name]) => name);
|
||||
const runTimestamp = new Date().toISOString();
|
||||
const report = {
|
||||
schema_version: ASSISTANT_STAGE2_RUN_SCHEMA_VERSION,
|
||||
run_id: runId,
|
||||
run_timestamp: runTimestamp,
|
||||
eval_target: "assistant_stage2",
|
||||
mode: payload.mode,
|
||||
use_mock: Boolean(payload.useMock),
|
||||
prompt_version: payload.normalizeConfig.promptVersion ?? null,
|
||||
suite_id: suite.suite_id,
|
||||
suite_version: suite.suite_version,
|
||||
suite_schema_version: suite.schema_version ?? null,
|
||||
scenario_count: suite.scenario_count,
|
||||
case_ids: suiteCases.map((item) => item.case_id),
|
||||
cases_total: caseRecords.length,
|
||||
feature_profile_snapshot: buildFeatureProfileSnapshot(),
|
||||
code_version: buildCodeVersionMarker(),
|
||||
metrics: {
|
||||
raw: metrics.raw,
|
||||
denominators: metrics.denominators
|
||||
},
|
||||
rubric_bands: metrics.rubric_bands,
|
||||
subsets: {
|
||||
expected_problem_cases_total: metrics.denominators.expected_problem_cases_total,
|
||||
followup_cases_total: metrics.denominators.followup_cases_total,
|
||||
candidate_cases_total: metrics.denominators.candidate_cases_total
|
||||
},
|
||||
budget: {
|
||||
requests_total: requestsTotal
|
||||
},
|
||||
results: caseRecords,
|
||||
scenario_summary: {
|
||||
improved_or_strong: caseRecords.filter((item) => {
|
||||
const clarity = Number(item.metric_subscores.problem_clarity_score ?? 0);
|
||||
const mechanism = Number(item.metric_subscores.mechanism_coherence_score ?? 0);
|
||||
return clarity >= 4 && mechanism >= 3;
|
||||
}).length,
|
||||
unchanged_or_mixed: caseRecords.filter((item) => {
|
||||
const clarity = Number(item.metric_subscores.problem_clarity_score ?? 0);
|
||||
return clarity >= 2.5 && clarity < 4;
|
||||
}).length,
|
||||
weak_or_regressed: caseRecords.filter((item) => Number(item.metric_subscores.problem_clarity_score ?? 0) < 2.5).length
|
||||
},
|
||||
improvement_hints: {
|
||||
strongest_signals: strongestSignals.length > 0 ? strongestSignals.join(", ") : "none",
|
||||
weakest_signals: weakestSignals.length > 0 ? weakestSignals.join(", ") : "none"
|
||||
},
|
||||
known_limitations: [
|
||||
"Stage 2 eval remains heuristic and scoped to problem-unit baseline (no graph/lifecycle/investigation runtime scoring).",
|
||||
"problem_unit_recall_proxy uses suite expected types as lightweight proxy, not full ground-truth labeling.",
|
||||
"Comparison quality depends on stable feature profile and reproducible mock/runtime setup."
|
||||
],
|
||||
report_title: "Assistant Stage 2 Eval Run"
|
||||
};
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
const runJsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.json`);
|
||||
const runMdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.md`);
|
||||
(0, files_1.writeJsonFile)(runJsonPath, report);
|
||||
fs_1.default.writeFileSync(runMdPath, buildAssistantStage2EvalMarkdownReport(report), "utf-8");
|
||||
report.artifacts = {
|
||||
run_report_json_path: runJsonPath,
|
||||
run_report_md_path: runMdPath
|
||||
};
|
||||
if (payload.compareWithReportFile) {
|
||||
report.comparison = this.buildAssistantStage2ComparisonReport({
|
||||
currentReport: report,
|
||||
baselineReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
return report;
|
||||
}
|
||||
async run(payload) {
|
||||
const mode = payload.mode ?? "standard";
|
||||
const evalTarget = payload.evalTarget ?? "normalizer";
|
||||
@@ -1303,6 +2072,16 @@ class EvalService {
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
if (evalTarget === "assistant_stage2") {
|
||||
return this.runAssistantStage2({
|
||||
normalizeConfig: payload.normalizeConfig,
|
||||
caseIds: payload.caseIds,
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
const promptVersion = String(payload.normalizeConfig.promptVersion ?? "").toLowerCase();
|
||||
const schemaVersion = String(payload.normalizeConfig.schemaVersion ?? "").toLowerCase();
|
||||
const isV2 = promptVersion.startsWith("normalizer_v2") || schemaVersion === "v2" || schemaVersion === "v2_0_1" || schemaVersion === "v2_0_2";
|
||||
|
||||
+112
-2
@@ -4,6 +4,7 @@ exports.cloneInvestigationState = cloneInvestigationState;
|
||||
exports.createEmptyInvestigationState = createEmptyInvestigationState;
|
||||
exports.updateInvestigationState = updateInvestigationState;
|
||||
const stage1Contracts_1 = require("../types/stage1Contracts");
|
||||
const stage2ProblemUnits_1 = require("../types/stage2ProblemUnits");
|
||||
function uniqueStrings(values) {
|
||||
return Array.from(new Set(values.map((item) => item.trim()).filter(Boolean)));
|
||||
}
|
||||
@@ -74,10 +75,101 @@ function collectOpenUncertainties(coverageReport, retrievalResults) {
|
||||
const limitationNotes = retrievalResults.flatMap((result) => result.limitations).slice(0, 6);
|
||||
return capStrings([...requirementNotes, ...limitationNotes], stage1Contracts_1.INVESTIGATION_MAX_UNCERTAINTIES);
|
||||
}
|
||||
function normalizeEntityBacklinks(values) {
|
||||
const result = [];
|
||||
const seen = new Set();
|
||||
for (const item of values) {
|
||||
const entity = String(item.entity ?? "").trim();
|
||||
const id = String(item.id ?? "").trim();
|
||||
if (!entity || !id) {
|
||||
continue;
|
||||
}
|
||||
const key = `${entity}::${id}`;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
result.push({
|
||||
entity,
|
||||
id
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
function collectProblemUnits(retrievalResults) {
|
||||
return retrievalResults.flatMap((result) => result.problem_units ?? []);
|
||||
}
|
||||
function capProblemUnitState(state) {
|
||||
return {
|
||||
active_problem_units: capStrings(state.active_problem_units, stage2ProblemUnits_1.INVESTIGATION_MAX_ACTIVE_PROBLEM_UNITS),
|
||||
resolved_problem_units: capStrings(state.resolved_problem_units, stage2ProblemUnits_1.INVESTIGATION_MAX_RESOLVED_PROBLEM_UNITS),
|
||||
problem_unit_backlinks: state.problem_unit_backlinks
|
||||
.map((item) => ({
|
||||
problem_unit_id: String(item.problem_unit_id ?? "").trim(),
|
||||
entity_backlinks: normalizeEntityBacklinks(item.entity_backlinks ?? [])
|
||||
}))
|
||||
.filter((item) => Boolean(item.problem_unit_id) && item.entity_backlinks.length > 0)
|
||||
.slice(0, stage2ProblemUnits_1.INVESTIGATION_MAX_PROBLEM_UNIT_BACKLINKS),
|
||||
focus_problem_types: capStrings(state.focus_problem_types.map((item) => String(item)), stage2ProblemUnits_1.INVESTIGATION_MAX_FOCUS_PROBLEM_TYPES)
|
||||
};
|
||||
}
|
||||
function updateProblemUnitState(previous, retrievalResults) {
|
||||
const previousState = previous.problem_unit_state;
|
||||
const currentProblemUnits = collectProblemUnits(retrievalResults);
|
||||
const currentIds = capStrings(currentProblemUnits.map((item) => String(item.problem_unit_id ?? "")), stage2ProblemUnits_1.INVESTIGATION_MAX_ACTIVE_PROBLEM_UNITS);
|
||||
const currentTypes = capStrings(currentProblemUnits.map((item) => String(item.problem_unit_type ?? "")), stage2ProblemUnits_1.INVESTIGATION_MAX_FOCUS_PROBLEM_TYPES);
|
||||
const currentBacklinksRaw = currentProblemUnits
|
||||
.filter((item) => currentIds.includes(item.problem_unit_id))
|
||||
.map((item) => ({
|
||||
problem_unit_id: item.problem_unit_id,
|
||||
entity_backlinks: normalizeEntityBacklinks(item.entity_backlinks ?? [])
|
||||
}))
|
||||
.filter((item) => item.entity_backlinks.length > 0);
|
||||
const currentBacklinksById = new Map(currentBacklinksRaw.map((item) => [item.problem_unit_id, item.entity_backlinks]));
|
||||
const previousBacklinksById = new Map((previousState?.problem_unit_backlinks ?? []).map((item) => [item.problem_unit_id, item.entity_backlinks]));
|
||||
const active_problem_units = currentIds.length > 0
|
||||
? currentIds
|
||||
: capStrings(previousState?.active_problem_units ?? [], stage2ProblemUnits_1.INVESTIGATION_MAX_ACTIVE_PROBLEM_UNITS);
|
||||
const resolved_problem_units = currentIds.length > 0
|
||||
? capStrings([
|
||||
...(previousState?.active_problem_units ?? []).filter((item) => !currentIds.includes(item)),
|
||||
...(previousState?.resolved_problem_units ?? [])
|
||||
], stage2ProblemUnits_1.INVESTIGATION_MAX_RESOLVED_PROBLEM_UNITS)
|
||||
: capStrings(previousState?.resolved_problem_units ?? [], stage2ProblemUnits_1.INVESTIGATION_MAX_RESOLVED_PROBLEM_UNITS);
|
||||
const problem_unit_backlinks = active_problem_units
|
||||
.map((problemUnitId) => {
|
||||
const entity_backlinks = normalizeEntityBacklinks(currentBacklinksById.get(problemUnitId) ?? previousBacklinksById.get(problemUnitId) ?? []);
|
||||
if (entity_backlinks.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
problem_unit_id: problemUnitId,
|
||||
entity_backlinks
|
||||
};
|
||||
})
|
||||
.filter((item) => item !== null)
|
||||
.slice(0, stage2ProblemUnits_1.INVESTIGATION_MAX_PROBLEM_UNIT_BACKLINKS);
|
||||
const focus_problem_types = currentTypes.length > 0
|
||||
? currentTypes
|
||||
: capStrings((previousState?.focus_problem_types ?? []).map((item) => String(item)), stage2ProblemUnits_1.INVESTIGATION_MAX_FOCUS_PROBLEM_TYPES);
|
||||
const nextState = capProblemUnitState({
|
||||
active_problem_units,
|
||||
resolved_problem_units,
|
||||
problem_unit_backlinks,
|
||||
focus_problem_types
|
||||
});
|
||||
if (nextState.active_problem_units.length === 0 &&
|
||||
nextState.resolved_problem_units.length === 0 &&
|
||||
nextState.problem_unit_backlinks.length === 0 &&
|
||||
nextState.focus_problem_types.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return nextState;
|
||||
}
|
||||
function cloneInvestigationState(state) {
|
||||
if (!state)
|
||||
return null;
|
||||
return {
|
||||
const cloned = {
|
||||
...state,
|
||||
focus: {
|
||||
...state.focus,
|
||||
@@ -92,6 +184,18 @@ function cloneInvestigationState(state) {
|
||||
}
|
||||
: null
|
||||
};
|
||||
if (state.problem_unit_state) {
|
||||
cloned.problem_unit_state = capProblemUnitState({
|
||||
active_problem_units: [...state.problem_unit_state.active_problem_units],
|
||||
resolved_problem_units: [...state.problem_unit_state.resolved_problem_units],
|
||||
problem_unit_backlinks: state.problem_unit_state.problem_unit_backlinks.map((item) => ({
|
||||
problem_unit_id: item.problem_unit_id,
|
||||
entity_backlinks: [...item.entity_backlinks]
|
||||
})),
|
||||
focus_problem_types: [...state.problem_unit_state.focus_problem_types]
|
||||
});
|
||||
}
|
||||
return cloned;
|
||||
}
|
||||
function createEmptyInvestigationState(sessionId, timestamp = new Date().toISOString()) {
|
||||
return {
|
||||
@@ -120,6 +224,7 @@ function updateInvestigationState(input) {
|
||||
const focusFromMessage = capStrings(detectAccounts(input.userMessage), stage1Contracts_1.INVESTIGATION_MAX_PRIMARY_ACCOUNTS);
|
||||
const requirementIds = capStrings(input.requirements.map((item) => item.requirement_id), stage1Contracts_1.INVESTIGATION_MAX_REQUIREMENT_LINKS);
|
||||
const mainRequirement = input.requirements[0]?.requirement_text ?? input.userMessage;
|
||||
const problemUnitState = updateProblemUnitState(previous, input.retrievalResults);
|
||||
return {
|
||||
schema_version: stage1Contracts_1.INVESTIGATION_STATE_SCHEMA_VERSION,
|
||||
session_id: previous.session_id,
|
||||
@@ -142,6 +247,11 @@ function updateInvestigationState(input) {
|
||||
last_user_message: input.userMessage.slice(0, 240),
|
||||
referenced_requirement_ids: requirementIds
|
||||
},
|
||||
query_mode_hint: deriveQueryModeHint(input.routeSummary)
|
||||
query_mode_hint: deriveQueryModeHint(input.routeSummary),
|
||||
...(problemUnitState
|
||||
? {
|
||||
problem_unit_state: problemUnitState
|
||||
}
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.buildCandidateEvidence = buildCandidateEvidence;
|
||||
exports.clusterCandidateEvidence = clusterCandidateEvidence;
|
||||
exports.detectProblemUnitType = detectProblemUnitType;
|
||||
exports.scoreProblemSeverity = scoreProblemSeverity;
|
||||
exports.buildProblemUnit = buildProblemUnit;
|
||||
exports.collapseDuplicates = collapseDuplicates;
|
||||
exports.assembleProblemUnits = assembleProblemUnits;
|
||||
const stage2ProblemUnits_1 = require("../types/stage2ProblemUnits");
|
||||
function toObject(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function uniqueStrings(values) {
|
||||
return Array.from(new Set(values.map((item) => item.trim()).filter(Boolean)));
|
||||
}
|
||||
function clampUnitScore(value) {
|
||||
if (!Number.isFinite(value)) {
|
||||
return 0;
|
||||
}
|
||||
if (value <= 0)
|
||||
return 0;
|
||||
if (value >= 1)
|
||||
return 1;
|
||||
return Number(value.toFixed(2));
|
||||
}
|
||||
function gradeForScore(score) {
|
||||
if (score >= 0.7)
|
||||
return "high";
|
||||
if (score >= 0.4)
|
||||
return "medium";
|
||||
return "low";
|
||||
}
|
||||
function confidenceGradeForScore(score) {
|
||||
if (score >= 0.75)
|
||||
return "high";
|
||||
if (score >= 0.45)
|
||||
return "medium";
|
||||
return "low";
|
||||
}
|
||||
function confidenceToScore(value) {
|
||||
if (value === "high")
|
||||
return 1;
|
||||
if (value === "medium")
|
||||
return 0.6;
|
||||
return 0.3;
|
||||
}
|
||||
function valueFromPayload(item, key) {
|
||||
const value = item.payload[key];
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
function stringArrayFromUnknown(value) {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return uniqueStrings(value.map((entry) => String(entry)));
|
||||
}
|
||||
function stringArrayFromPayload(item, key) {
|
||||
return stringArrayFromUnknown(item.payload[key]);
|
||||
}
|
||||
function extractSemanticProfile(summary) {
|
||||
const semanticProfile = toObject(summary.semantic_profile);
|
||||
return {
|
||||
relation_patterns: stringArrayFromUnknown(semanticProfile?.relation_patterns),
|
||||
anomaly_patterns: stringArrayFromUnknown(semanticProfile?.anomaly_patterns)
|
||||
};
|
||||
}
|
||||
function resolveEntityOverlay(item, rawEntities) {
|
||||
const sourceId = String(item.pointer.source.id ?? "").toLowerCase();
|
||||
if (!sourceId) {
|
||||
return null;
|
||||
}
|
||||
for (const entity of rawEntities) {
|
||||
const candidates = [
|
||||
String(entity.source_id ?? ""),
|
||||
String(entity.entity_id ?? ""),
|
||||
String(entity.id ?? "")
|
||||
]
|
||||
.map((entry) => entry.toLowerCase())
|
||||
.filter(Boolean);
|
||||
if (candidates.includes(sourceId)) {
|
||||
return entity;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function detectRelationPatternHits(item, overlay, context) {
|
||||
const hits = [];
|
||||
const failedEdge = valueFromPayload(item, "failed_expected_edge");
|
||||
if (failedEdge) {
|
||||
hits.push(`failed_edge:${failedEdge}`);
|
||||
}
|
||||
const expectedStep = valueFromPayload(item, "expected_next_step");
|
||||
if (expectedStep) {
|
||||
hits.push(`expected_step:${expectedStep}`);
|
||||
}
|
||||
const relationPattern = valueFromPayload(item, "relation_pattern");
|
||||
if (relationPattern) {
|
||||
hits.push(relationPattern);
|
||||
}
|
||||
hits.push(...stringArrayFromPayload(item, "relation_patterns"));
|
||||
hits.push(...stringArrayFromPayload(item, "relation_pattern_hits"));
|
||||
if (overlay) {
|
||||
hits.push(...stringArrayFromUnknown(overlay.relation_pattern_hits));
|
||||
hits.push(...stringArrayFromUnknown(overlay.relation_types));
|
||||
}
|
||||
hits.push(...context.summary_relation_patterns);
|
||||
if (context.route === "hybrid_store_plus_live" && hits.length === 0) {
|
||||
hits.push("chain_scope");
|
||||
}
|
||||
return uniqueStrings(hits);
|
||||
}
|
||||
function detectAnomalyPatterns(item, overlay, context) {
|
||||
const patterns = [];
|
||||
patterns.push(...stringArrayFromPayload(item, "anomaly_patterns"));
|
||||
patterns.push(...stringArrayFromPayload(item, "risk_factors"));
|
||||
patterns.push(...stringArrayFromPayload(item, "lifecycle_gaps"));
|
||||
patterns.push(...stringArrayFromPayload(item, "lifecycle_markers"));
|
||||
const explicit = valueFromPayload(item, "anomaly_pattern");
|
||||
if (explicit) {
|
||||
patterns.push(explicit);
|
||||
}
|
||||
if (overlay) {
|
||||
patterns.push(...stringArrayFromUnknown(overlay.risk_factors));
|
||||
patterns.push(...stringArrayFromUnknown(overlay.lifecycle_gaps));
|
||||
patterns.push(...stringArrayFromUnknown(overlay.anomaly_patterns));
|
||||
}
|
||||
patterns.push(...context.summary_anomaly_patterns);
|
||||
patterns.push(...context.risk_factors);
|
||||
if (item.evidence_kind === "anomaly_signal") {
|
||||
patterns.push("anomaly_signal");
|
||||
}
|
||||
if (item.limitation?.reason_code === "missing_mechanism") {
|
||||
patterns.push("missing_mechanism");
|
||||
}
|
||||
if (item.limitation?.reason_code === "insufficient_detail") {
|
||||
patterns.push("insufficient_detail");
|
||||
}
|
||||
const defectClass = valueFromPayload(item, "business_defect_class");
|
||||
if (defectClass) {
|
||||
patterns.push(defectClass);
|
||||
}
|
||||
if (context.route === "store_feature_risk") {
|
||||
patterns.push("risk_route");
|
||||
}
|
||||
return uniqueStrings(patterns);
|
||||
}
|
||||
function buildEntityBacklinks(item, overlay) {
|
||||
const backlinks = [];
|
||||
const sourceEntity = String(item.pointer.source.entity ?? "").trim();
|
||||
const sourceId = String(item.pointer.source.id ?? "").trim();
|
||||
if (sourceEntity && sourceId) {
|
||||
backlinks.push({
|
||||
entity: sourceEntity,
|
||||
id: sourceId
|
||||
});
|
||||
}
|
||||
const payloadEntity = valueFromPayload(item, "source_entity");
|
||||
const payloadId = valueFromPayload(item, "source_id");
|
||||
if (payloadEntity && payloadId) {
|
||||
backlinks.push({
|
||||
entity: payloadEntity,
|
||||
id: payloadId
|
||||
});
|
||||
}
|
||||
const overlayEntity = overlay ? String(overlay.source_entity ?? "").trim() : "";
|
||||
const overlayId = overlay ? String(overlay.source_id ?? "").trim() : "";
|
||||
if (overlayEntity && overlayId) {
|
||||
backlinks.push({
|
||||
entity: overlayEntity,
|
||||
id: overlayId
|
||||
});
|
||||
}
|
||||
return Array.from(new Map(backlinks.map((entry) => [`${entry.entity.toLowerCase()}|${entry.id.toLowerCase()}`, entry])).values());
|
||||
}
|
||||
function inferExpectedState(item) {
|
||||
return valueFromPayload(item, "expected_state") ?? valueFromPayload(item, "expected_next_step") ?? undefined;
|
||||
}
|
||||
function inferActualState(item) {
|
||||
return valueFromPayload(item, "actual_state") ?? valueFromPayload(item, "mechanism_of_failure") ?? undefined;
|
||||
}
|
||||
function buildCandidateEvidence(items, route, contextInput) {
|
||||
const context = {
|
||||
route,
|
||||
result_type: contextInput?.result_type,
|
||||
raw_entities: contextInput?.raw_entities ?? [],
|
||||
summary_relation_patterns: contextInput?.summary_relation_patterns ?? [],
|
||||
summary_anomaly_patterns: contextInput?.summary_anomaly_patterns ?? [],
|
||||
risk_factors: contextInput?.risk_factors ?? []
|
||||
};
|
||||
return items.map((item, index) => {
|
||||
const overlay = resolveEntityOverlay(item, context.raw_entities);
|
||||
return {
|
||||
schema_version: stage2ProblemUnits_1.CANDIDATE_EVIDENCE_SCHEMA_VERSION,
|
||||
candidate_id: `cand-${item.evidence_id || `${route}-${index + 1}`}`,
|
||||
route,
|
||||
source_ref: item.source_ref,
|
||||
expected_state: inferExpectedState(item),
|
||||
actual_state: inferActualState(item),
|
||||
relation_pattern_hits: detectRelationPatternHits(item, overlay, context),
|
||||
anomaly_patterns: detectAnomalyPatterns(item, overlay, context),
|
||||
entity_backlinks: buildEntityBacklinks(item, overlay),
|
||||
confidence_hint: item.confidence
|
||||
};
|
||||
});
|
||||
}
|
||||
function clusterSignature(candidate) {
|
||||
const relation = candidate.relation_pattern_hits[0] ?? "none";
|
||||
const anomaly = candidate.anomaly_patterns[0] ?? "none";
|
||||
return [candidate.route, candidate.source_ref.canonical_ref, relation, anomaly].join("|");
|
||||
}
|
||||
function clusterCandidateEvidence(candidates) {
|
||||
const byCluster = new Map();
|
||||
for (const candidate of candidates) {
|
||||
const signature = clusterSignature(candidate);
|
||||
const current = byCluster.get(signature) ?? [];
|
||||
current.push(candidate);
|
||||
byCluster.set(signature, current);
|
||||
}
|
||||
return Array.from(byCluster.entries()).map(([cluster_id, clusterCandidates]) => ({
|
||||
cluster_id,
|
||||
candidates: clusterCandidates
|
||||
}));
|
||||
}
|
||||
function hasAny(value, pattern) {
|
||||
return pattern.test(value);
|
||||
}
|
||||
function detectProblemUnitType(cluster) {
|
||||
const relationText = cluster.candidates.flatMap((item) => item.relation_pattern_hits).join(" ").toLowerCase();
|
||||
const anomalyText = cluster.candidates.flatMap((item) => item.anomaly_patterns).join(" ").toLowerCase();
|
||||
const sourceText = cluster.candidates
|
||||
.map((item) => `${item.source_ref.entity} ${item.source_ref.id}`)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
const routeText = cluster.candidates.map((item) => item.route).join(" ").toLowerCase();
|
||||
if (hasAny(`${anomalyText} ${sourceText}`, /cross[_\s-]?branch|vat|nds|tax|ндс/)) {
|
||||
return "cross_branch_inconsistency_cluster";
|
||||
}
|
||||
if (hasAny(`${anomalyText} ${relationText}`, /period|close[_\s-]?risk|reporting|закрыт|period_close/)) {
|
||||
return "period_risk_cluster";
|
||||
}
|
||||
if (hasAny(anomalyText, /settlement|tail|unresolved|хвост|незакрыт/)) {
|
||||
return "unresolved_settlement_cluster";
|
||||
}
|
||||
if (hasAny(anomalyText, /lifecycle|deferred|broken_lifecycle|списани|амортиз|рбп/)) {
|
||||
return "lifecycle_anomaly_node";
|
||||
}
|
||||
if (hasAny(relationText, /failed_edge|statement_to_document|payment_to_settlement|chain|broken|цепоч|разрыв/)
|
||||
|| (routeText.includes("hybrid_store_plus_live") && relationText.length > 0)) {
|
||||
return "broken_chain_segment";
|
||||
}
|
||||
return "document_conflict";
|
||||
}
|
||||
function scoreProblemSeverity(cluster) {
|
||||
const candidates = cluster.candidates;
|
||||
const averageConfidence = candidates.length > 0
|
||||
? candidates.reduce((acc, item) => acc + confidenceToScore(item.confidence_hint), 0) / candidates.length
|
||||
: 0;
|
||||
const hasEdgeBreak = candidates.some((item) => item.relation_pattern_hits.some((pattern) => /failed_edge|chain|statement_to_document|payment_to_settlement/i.test(pattern)));
|
||||
const hasAnomaly = candidates.some((item) => item.anomaly_patterns.length > 0);
|
||||
const hasPeriodRisk = candidates.some((item) => item.anomaly_patterns.some((pattern) => /period|close|reporting|закрыт/i.test(pattern)));
|
||||
const candidateBoost = Math.min(candidates.length, 5) * 0.08;
|
||||
let severityScore = 0.25 + candidateBoost;
|
||||
if (hasEdgeBreak)
|
||||
severityScore += 0.2;
|
||||
if (hasAnomaly)
|
||||
severityScore += 0.15;
|
||||
if (hasPeriodRisk)
|
||||
severityScore += 0.1;
|
||||
const normalizedSeverity = clampUnitScore(severityScore);
|
||||
let confidenceScore = averageConfidence;
|
||||
if (hasEdgeBreak)
|
||||
confidenceScore += 0.05;
|
||||
const normalizedConfidence = clampUnitScore(confidenceScore);
|
||||
return {
|
||||
severity: {
|
||||
score: normalizedSeverity,
|
||||
grade: gradeForScore(normalizedSeverity)
|
||||
},
|
||||
confidence: {
|
||||
score: normalizedConfidence,
|
||||
grade: confidenceGradeForScore(normalizedConfidence)
|
||||
}
|
||||
};
|
||||
}
|
||||
function unitTitle(type) {
|
||||
if (type === "document_conflict")
|
||||
return "Document conflict detected";
|
||||
if (type === "broken_chain_segment")
|
||||
return "Broken chain segment detected";
|
||||
if (type === "lifecycle_anomaly_node")
|
||||
return "Lifecycle anomaly node detected";
|
||||
if (type === "unresolved_settlement_cluster")
|
||||
return "Unresolved settlement cluster detected";
|
||||
if (type === "period_risk_cluster")
|
||||
return "Period risk cluster detected";
|
||||
return "Cross-branch inconsistency cluster detected";
|
||||
}
|
||||
function mechanismSummary(cluster, type) {
|
||||
const relationHints = uniqueStrings(cluster.candidates.flatMap((item) => item.relation_pattern_hits));
|
||||
const anomalyHints = uniqueStrings(cluster.candidates.flatMap((item) => item.anomaly_patterns));
|
||||
const primaryRelation = relationHints[0];
|
||||
const primaryAnomaly = anomalyHints[0];
|
||||
if (primaryRelation) {
|
||||
return `Mechanism candidate: ${primaryRelation}.`;
|
||||
}
|
||||
if (primaryAnomaly) {
|
||||
return `Mechanism inferred from anomaly pattern: ${primaryAnomaly}.`;
|
||||
}
|
||||
return `Mechanism is currently inferred at baseline level for ${type}.`;
|
||||
}
|
||||
function businessDefectClass(cluster, type) {
|
||||
const patterns = uniqueStrings([
|
||||
...cluster.candidates.flatMap((item) => item.relation_pattern_hits),
|
||||
...cluster.candidates.flatMap((item) => item.anomaly_patterns)
|
||||
]);
|
||||
return patterns[0] ?? type;
|
||||
}
|
||||
function collectAffectedByEntity(backlinks, pattern) {
|
||||
return uniqueStrings(backlinks.filter((entry) => pattern.test(entry.entity)).map((entry) => `${entry.entity}:${entry.id}`));
|
||||
}
|
||||
function parseFailedExpectedEdge(cluster) {
|
||||
const withEdge = cluster.candidates.flatMap((item) => item.relation_pattern_hits).find((pattern) => pattern.startsWith("failed_edge:"));
|
||||
if (!withEdge) {
|
||||
return undefined;
|
||||
}
|
||||
return withEdge.replace(/^failed_edge:/, "").trim() || undefined;
|
||||
}
|
||||
function mergeBacklinks(candidates) {
|
||||
return Array.from(new Map(candidates
|
||||
.flatMap((item) => item.entity_backlinks)
|
||||
.map((entry) => [`${entry.entity.toLowerCase()}|${entry.id.toLowerCase()}`, entry])).values());
|
||||
}
|
||||
function buildProblemUnit(cluster, index) {
|
||||
const type = detectProblemUnitType(cluster);
|
||||
const scored = scoreProblemSeverity(cluster);
|
||||
const backlinks = mergeBacklinks(cluster.candidates);
|
||||
const expectedState = cluster.candidates.find((item) => typeof item.expected_state === "string")?.expected_state;
|
||||
const actualState = cluster.candidates.find((item) => typeof item.actual_state === "string")?.actual_state;
|
||||
const failedExpectedEdge = parseFailedExpectedEdge(cluster);
|
||||
const periodSensitive = cluster.candidates.some((item) => item.anomaly_patterns.some((pattern) => /period|close|reporting|закрыт/i.test(pattern)));
|
||||
const hasLowConfidence = cluster.candidates.some((item) => item.confidence_hint === "low");
|
||||
return {
|
||||
schema_version: stage2ProblemUnits_1.PROBLEM_UNIT_SCHEMA_VERSION,
|
||||
problem_unit_id: `pu-${type}-${index + 1}`,
|
||||
problem_unit_type: type,
|
||||
title: unitTitle(type),
|
||||
mechanism_summary: mechanismSummary(cluster, type),
|
||||
business_defect_class: businessDefectClass(cluster, type),
|
||||
severity: scored.severity,
|
||||
confidence: scored.confidence,
|
||||
affected_entities: uniqueStrings(backlinks.map((entry) => `${entry.entity}:${entry.id}`)),
|
||||
affected_documents: collectAffectedByEntity(backlinks, /doc|document|invoice|плат|реал|поступ/i),
|
||||
affected_postings: collectAffectedByEntity(backlinks, /posting|journal|провод/i),
|
||||
affected_accounts: collectAffectedByEntity(backlinks, /account|счет|сч/i),
|
||||
affected_counterparties: collectAffectedByEntity(backlinks, /counterparty|supplier|buyer|контраг|постав|покуп/i),
|
||||
affected_contracts: collectAffectedByEntity(backlinks, /contract|договор/i),
|
||||
...(expectedState ? { expected_state: expectedState } : {}),
|
||||
...(actualState ? { actual_state: actualState } : {}),
|
||||
...(failedExpectedEdge ? { failed_expected_edge: failedExpectedEdge } : {}),
|
||||
...(periodSensitive
|
||||
? {
|
||||
period_impact: {
|
||||
is_period_sensitive: true,
|
||||
impact_class: "close_risk"
|
||||
}
|
||||
}
|
||||
: {}),
|
||||
evidence_pack: uniqueStrings(cluster.candidates.map((item) => item.candidate_id)),
|
||||
entity_backlinks: backlinks,
|
||||
snapshot_limitations: uniqueStrings(hasLowConfidence ? ["low_confidence_candidates_present"] : [])
|
||||
};
|
||||
}
|
||||
function collapseSignature(unit) {
|
||||
const backlink = unit.entity_backlinks[0] ? `${unit.entity_backlinks[0].entity}|${unit.entity_backlinks[0].id}` : "none";
|
||||
return [unit.problem_unit_type, unit.business_defect_class, unit.failed_expected_edge ?? "none", backlink].join("|");
|
||||
}
|
||||
function collapseDuplicates(units) {
|
||||
const bySignature = new Map();
|
||||
let duplicateCollapses = 0;
|
||||
for (const unit of units) {
|
||||
const signature = collapseSignature(unit);
|
||||
const existing = bySignature.get(signature);
|
||||
if (!existing) {
|
||||
bySignature.set(signature, unit);
|
||||
continue;
|
||||
}
|
||||
duplicateCollapses += 1;
|
||||
bySignature.set(signature, {
|
||||
...existing,
|
||||
evidence_pack: uniqueStrings([...existing.evidence_pack, ...unit.evidence_pack]),
|
||||
entity_backlinks: Array.from(new Map([...existing.entity_backlinks, ...unit.entity_backlinks].map((entry) => [
|
||||
`${entry.entity.toLowerCase()}|${entry.id.toLowerCase()}`,
|
||||
entry
|
||||
])).values()),
|
||||
affected_entities: uniqueStrings([...existing.affected_entities, ...unit.affected_entities]),
|
||||
affected_documents: uniqueStrings([...existing.affected_documents, ...unit.affected_documents]),
|
||||
affected_postings: uniqueStrings([...existing.affected_postings, ...unit.affected_postings]),
|
||||
affected_accounts: uniqueStrings([...existing.affected_accounts, ...unit.affected_accounts]),
|
||||
affected_counterparties: uniqueStrings([...existing.affected_counterparties, ...unit.affected_counterparties]),
|
||||
affected_contracts: uniqueStrings([...existing.affected_contracts, ...unit.affected_contracts]),
|
||||
snapshot_limitations: uniqueStrings([...existing.snapshot_limitations, ...unit.snapshot_limitations]),
|
||||
severity: unit.severity.score > existing.severity.score ? unit.severity : existing.severity,
|
||||
confidence: unit.confidence.score > existing.confidence.score ? unit.confidence : existing.confidence
|
||||
});
|
||||
}
|
||||
return {
|
||||
problem_units: Array.from(bySignature.values()),
|
||||
duplicate_collapses: duplicateCollapses
|
||||
};
|
||||
}
|
||||
function buildSummary(units, duplicateCollapses) {
|
||||
const unitTypes = uniqueStrings(units.map((item) => item.problem_unit_type));
|
||||
const typeDistribution = {};
|
||||
const severityDistribution = {
|
||||
low: 0,
|
||||
medium: 0,
|
||||
high: 0
|
||||
};
|
||||
const confidenceDistribution = {
|
||||
low: 0,
|
||||
medium: 0,
|
||||
high: 0
|
||||
};
|
||||
for (const unit of units) {
|
||||
typeDistribution[unit.problem_unit_type] = (typeDistribution[unit.problem_unit_type] ?? 0) + 1;
|
||||
severityDistribution[unit.severity.grade] += 1;
|
||||
confidenceDistribution[unit.confidence.grade] += 1;
|
||||
}
|
||||
return {
|
||||
schema_version: stage2ProblemUnits_1.PROBLEM_UNIT_SUMMARY_SCHEMA_VERSION,
|
||||
units_total: units.length,
|
||||
duplicate_collapses: duplicateCollapses,
|
||||
unit_types: unitTypes,
|
||||
type_distribution: typeDistribution,
|
||||
severity_distribution: severityDistribution,
|
||||
confidence_distribution: confidenceDistribution,
|
||||
primary_unit_type: units[0]?.problem_unit_type ?? null
|
||||
};
|
||||
}
|
||||
function assembleProblemUnits(input) {
|
||||
const summary = input.summary ?? {};
|
||||
const semanticProfile = extractSemanticProfile(summary);
|
||||
const candidates = buildCandidateEvidence(input.evidence, input.route, {
|
||||
route: input.route,
|
||||
result_type: input.result_type,
|
||||
raw_entities: input.raw_entities ?? [],
|
||||
summary_relation_patterns: semanticProfile.relation_patterns,
|
||||
summary_anomaly_patterns: semanticProfile.anomaly_patterns,
|
||||
risk_factors: uniqueStrings(input.risk_factors ?? [])
|
||||
});
|
||||
const clusters = clusterCandidateEvidence(candidates);
|
||||
const units = clusters.map((cluster, index) => buildProblemUnit(cluster, index));
|
||||
const collapsed = collapseDuplicates(units);
|
||||
return {
|
||||
candidate_evidence: candidates,
|
||||
problem_units: collapsed.problem_units,
|
||||
problem_unit_summary: buildSummary(collapsed.problem_units, collapsed.duplicate_collapses)
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.normalizeRetrievalResult = normalizeRetrievalResult;
|
||||
const config_1 = require("../config");
|
||||
const stage1Contracts_1 = require("../types/stage1Contracts");
|
||||
const problemUnitAssembler_1 = require("./problemUnitAssembler");
|
||||
function toObject(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
@@ -59,6 +60,18 @@ function normalizeStringArray(value) {
|
||||
}
|
||||
return value.map((item) => String(item));
|
||||
}
|
||||
function mergeSummaryWithProblemUnitMeta(summary, input) {
|
||||
return {
|
||||
...summary,
|
||||
problem_units_enabled: true,
|
||||
candidate_evidence_count: input.candidateEvidenceCount,
|
||||
problem_units_count: input.problemUnitsCount,
|
||||
problem_unit_types: input.unitTypes,
|
||||
problem_unit_duplicate_collapses: input.duplicateCollapses,
|
||||
problem_unit_severity_distribution: input.severityDistribution,
|
||||
problem_unit_confidence_distribution: input.confidenceDistribution
|
||||
};
|
||||
}
|
||||
function normalizeConfidence(value) {
|
||||
if (value === "high" || value === "medium" || value === "low") {
|
||||
return value;
|
||||
@@ -359,15 +372,18 @@ function normalizeEvidenceItems(fragmentId, requirementIds, route, value) {
|
||||
});
|
||||
}
|
||||
function normalizeRetrievalResult(fragmentId, requirementIds, route, raw) {
|
||||
return {
|
||||
const items = normalizeObjectArray(raw.items);
|
||||
const summary = normalizeSummary(raw.summary);
|
||||
const evidence = normalizeEvidenceItems(fragmentId, requirementIds, route, raw.evidence);
|
||||
const baseResult = {
|
||||
fragment_id: fragmentId,
|
||||
requirement_ids: requirementIds,
|
||||
route,
|
||||
status: normalizeStatus(raw.status),
|
||||
result_type: normalizeResultType(raw.result_type),
|
||||
items: normalizeObjectArray(raw.items),
|
||||
summary: normalizeSummary(raw.summary),
|
||||
evidence: normalizeEvidenceItems(fragmentId, requirementIds, route, raw.evidence),
|
||||
items,
|
||||
summary,
|
||||
evidence,
|
||||
why_included: normalizeStringArray(raw.why_included),
|
||||
selection_reason: normalizeStringArray(raw.selection_reason),
|
||||
risk_factors: normalizeStringArray(raw.risk_factors),
|
||||
@@ -376,4 +392,33 @@ function normalizeRetrievalResult(fragmentId, requirementIds, route, raw) {
|
||||
limitations: normalizeStringArray(raw.limitations),
|
||||
errors: normalizeErrors(raw.errors)
|
||||
};
|
||||
if (!config_1.FEATURE_ASSISTANT_PROBLEM_UNITS_V1) {
|
||||
return baseResult;
|
||||
}
|
||||
const assembled = (0, problemUnitAssembler_1.assembleProblemUnits)({
|
||||
route,
|
||||
result_type: baseResult.result_type,
|
||||
evidence,
|
||||
raw_entities: items,
|
||||
summary,
|
||||
risk_factors: baseResult.risk_factors,
|
||||
selection_reason: baseResult.selection_reason,
|
||||
business_interpretation: baseResult.business_interpretation
|
||||
});
|
||||
const enrichedSummary = mergeSummaryWithProblemUnitMeta(summary, {
|
||||
candidateEvidenceCount: assembled.candidate_evidence.length,
|
||||
problemUnitsCount: assembled.problem_units.length,
|
||||
unitTypes: assembled.problem_unit_summary.unit_types,
|
||||
duplicateCollapses: assembled.problem_unit_summary.duplicate_collapses,
|
||||
severityDistribution: assembled.problem_unit_summary.severity_distribution,
|
||||
confidenceDistribution: assembled.problem_unit_summary.confidence_distribution
|
||||
});
|
||||
return {
|
||||
...baseResult,
|
||||
summary: enrichedSummary,
|
||||
raw_entities: items,
|
||||
candidate_evidence: assembled.candidate_evidence,
|
||||
problem_units: assembled.problem_units,
|
||||
problem_unit_summary: assembled.problem_unit_summary
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user