Этап 4 / Волна 10: корректировка settlement-кейса — доменная фиксация синтеза, честное покрытие, удержание фокуса / Этап 4 / Волна 11: бизнес-якоря, доменное заземление и устранение утечки дебага
This commit is contained in:
@@ -7,6 +7,11 @@ OPENAI_TEMPERATURE=0
|
||||
OPENAI_MAX_OUTPUT_TOKENS=700
|
||||
DATA_DIR=./data
|
||||
TZ_FALLBACK=Europe/Moscow
|
||||
FEATURE_ASSISTANT_MCP_RUNTIME_V1=0
|
||||
ASSISTANT_MCP_PROXY_URL=http://127.0.0.1:6003
|
||||
ASSISTANT_MCP_CHANNEL=default
|
||||
ASSISTANT_MCP_TIMEOUT_MS=1200
|
||||
ASSISTANT_MCP_LIVE_LIMIT=24
|
||||
|
||||
# Frontend (optional, usually proxy to backend)
|
||||
VITE_API_BASE=/api
|
||||
|
||||
@@ -41,6 +41,14 @@ npm run dev
|
||||
Backend по умолчанию:
|
||||
- `http://localhost:8787`
|
||||
|
||||
Чтобы включить live-probe в 1С через MCP (для `hybrid_store_plus_live` и `live_mcp_drilldown`), задайте переменные перед запуском backend:
|
||||
|
||||
```powershell
|
||||
$env:FEATURE_ASSISTANT_MCP_RUNTIME_V1='1'
|
||||
$env:ASSISTANT_MCP_PROXY_URL='http://127.0.0.1:6003'
|
||||
$env:ASSISTANT_MCP_CHANNEL='default'
|
||||
```
|
||||
|
||||
## Запуск из одной папки (VS Code)
|
||||
|
||||
Открой в VS Code папку:
|
||||
@@ -59,6 +67,13 @@ cd X:\1C\NDC_1C\llm_normalizer
|
||||
start-dev.cmd
|
||||
```
|
||||
|
||||
С live-MCP (прокси 1С на `127.0.0.1:6003`):
|
||||
|
||||
```powershell
|
||||
cd X:\1C\NDC_1C\llm_normalizer
|
||||
start-dev-mcp.cmd
|
||||
```
|
||||
|
||||
Или:
|
||||
|
||||
```powershell
|
||||
|
||||
+20
-7
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ARCH_EXPORT_2020_DIR = exports.SCHEMAS_DIR = exports.EVAL_DATASETS_DIR = exports.REPORTS_DIR = exports.PROMPTS_DIR = exports.ASSISTANT_SESSIONS_DIR = exports.EVAL_CASES_DIR = exports.PRESETS_DIR = exports.TRACES_DIR = exports.DATA_DIR = exports.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = exports.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = exports.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = exports.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = exports.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = exports.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = exports.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = exports.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = exports.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = exports.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = exports.FEATURE_ASSISTANT_BROAD_GUARD_V1 = exports.FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1 = exports.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = exports.FEATURE_ASSISTANT_CONTRACTS_V11 = exports.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = exports.DEFAULT_PROMPT_VERSION = exports.DEFAULT_MAX_OUTPUT_TOKENS = exports.DEFAULT_TEMPERATURE = exports.DEFAULT_MODEL = exports.DEFAULT_OPENAI_BASE_URL = exports.TIMEZONE = exports.PORT = exports.MODULE_ROOT = exports.BACKEND_ROOT = void 0;
|
||||
exports.ARCH_EXPORT_2020_DIR = exports.SCHEMAS_DIR = exports.EVAL_DATASETS_DIR = exports.REPORTS_DIR = exports.PROMPTS_DIR = exports.ASSISTANT_SESSIONS_DIR = exports.EVAL_CASES_DIR = exports.PRESETS_DIR = exports.TRACES_DIR = exports.DATA_DIR = exports.ASSISTANT_MCP_LIVE_LIMIT = exports.ASSISTANT_MCP_TIMEOUT_MS = exports.ASSISTANT_MCP_CHANNEL = exports.ASSISTANT_MCP_PROXY_URL = exports.FEATURE_ASSISTANT_MCP_RUNTIME_V1 = exports.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = exports.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = exports.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = exports.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = exports.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = exports.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = exports.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = exports.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = exports.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = exports.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = exports.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = exports.FEATURE_ASSISTANT_BROAD_GUARD_V1 = exports.FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1 = exports.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = exports.FEATURE_ASSISTANT_CONTRACTS_V11 = exports.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = exports.DEFAULT_PROMPT_VERSION = exports.DEFAULT_MAX_OUTPUT_TOKENS = exports.DEFAULT_TEMPERATURE = exports.DEFAULT_MODEL = exports.DEFAULT_OPENAI_BASE_URL = exports.TIMEZONE = exports.PORT = exports.MODULE_ROOT = exports.BACKEND_ROOT = void 0;
|
||||
const path_1 = __importDefault(require("path"));
|
||||
exports.BACKEND_ROOT = path_1.default.resolve(__dirname, "..");
|
||||
exports.MODULE_ROOT = path_1.default.resolve(exports.BACKEND_ROOT, "..");
|
||||
@@ -14,6 +14,13 @@ function toBooleanFlag(value, defaultValue) {
|
||||
const lowered = value.trim().toLowerCase();
|
||||
return !(lowered === "0" || lowered === "false" || lowered === "off" || lowered === "no");
|
||||
}
|
||||
function toNumberFlag(value, defaultValue) {
|
||||
if (!value || value.trim() === "") {
|
||||
return defaultValue;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : defaultValue;
|
||||
}
|
||||
exports.PORT = Number(process.env.PORT ?? 8787);
|
||||
exports.TIMEZONE = process.env.TZ_FALLBACK ?? "Europe/Moscow";
|
||||
exports.DEFAULT_OPENAI_BASE_URL = process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1";
|
||||
@@ -28,14 +35,20 @@ exports.FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1 = toBooleanFlag(process.env.FEA
|
||||
exports.FEATURE_ASSISTANT_BROAD_GUARD_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1, true);
|
||||
exports.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1, true);
|
||||
exports.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1, true);
|
||||
exports.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = toBooleanFlag(process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11, false);
|
||||
exports.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = toBooleanFlag(process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11, true);
|
||||
exports.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1, true);
|
||||
exports.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1, false);
|
||||
exports.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1, false);
|
||||
exports.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1, false);
|
||||
exports.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1, true);
|
||||
exports.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1, true);
|
||||
exports.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1, true);
|
||||
exports.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1, false);
|
||||
exports.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1, false);
|
||||
exports.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1, false);
|
||||
exports.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1, true);
|
||||
exports.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1, true);
|
||||
exports.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1, true);
|
||||
exports.FEATURE_ASSISTANT_MCP_RUNTIME_V1 = toBooleanFlag(process.env.FEATURE_ASSISTANT_MCP_RUNTIME_V1, false);
|
||||
exports.ASSISTANT_MCP_PROXY_URL = (process.env.ASSISTANT_MCP_PROXY_URL ?? "http://127.0.0.1:6003").replace(/\/+$/, "");
|
||||
exports.ASSISTANT_MCP_CHANNEL = process.env.ASSISTANT_MCP_CHANNEL ?? "default";
|
||||
exports.ASSISTANT_MCP_TIMEOUT_MS = toNumberFlag(process.env.ASSISTANT_MCP_TIMEOUT_MS, 1200);
|
||||
exports.ASSISTANT_MCP_LIVE_LIMIT = Math.max(1, Math.trunc(toNumberFlag(process.env.ASSISTANT_MCP_LIVE_LIMIT, 24)));
|
||||
exports.DATA_DIR = process.env.DATA_DIR ?? path_1.default.resolve(exports.MODULE_ROOT, "data");
|
||||
exports.TRACES_DIR = path_1.default.resolve(exports.DATA_DIR, "traces");
|
||||
exports.PRESETS_DIR = path_1.default.resolve(exports.DATA_DIR, "presets");
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.evaluateP0AcceptanceGate = evaluateP0AcceptanceGate;
|
||||
exports.evaluateP0BaselineStabilityGate = evaluateP0BaselineStabilityGate;
|
||||
const p0_metric_definitions_1 = require("./p0_metric_definitions");
|
||||
function buildMetricChecks(input) {
|
||||
const { metrics, thresholds } = input;
|
||||
return [
|
||||
{
|
||||
metric: "problem_first_answer_rate",
|
||||
value: metrics.problem_first_answer_rate,
|
||||
threshold: thresholds.problem_first_answer_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.problem_first_answer_rate >= thresholds.problem_first_answer_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "entity_leakage_rate",
|
||||
value: metrics.entity_leakage_rate,
|
||||
threshold: thresholds.entity_leakage_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.entity_leakage_rate <= thresholds.entity_leakage_rate_max,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "route_correctness_rate",
|
||||
value: metrics.route_correctness_rate,
|
||||
threshold: thresholds.route_correctness_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.route_correctness_rate >= thresholds.route_correctness_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "domain_purity_rate",
|
||||
value: metrics.domain_purity_rate,
|
||||
threshold: thresholds.domain_purity_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.domain_purity_rate >= thresholds.domain_purity_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "mechanism_coherence_score",
|
||||
value: metrics.mechanism_coherence_score,
|
||||
threshold: thresholds.mechanism_coherence_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.mechanism_coherence_score >= thresholds.mechanism_coherence_score_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "accountant_actionability_score",
|
||||
value: metrics.accountant_actionability_score,
|
||||
threshold: thresholds.accountant_actionability_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.accountant_actionability_score >= thresholds.accountant_actionability_score_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "limitation_honesty_rate",
|
||||
value: metrics.limitation_honesty_rate,
|
||||
threshold: thresholds.limitation_honesty_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.limitation_honesty_rate >= thresholds.limitation_honesty_rate_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "top_problem_unit_match_rate",
|
||||
value: metrics.top_problem_unit_match_rate,
|
||||
threshold: thresholds.top_problem_unit_match_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.top_problem_unit_match_rate >= thresholds.top_problem_unit_match_rate_min,
|
||||
severity: "quality"
|
||||
}
|
||||
];
|
||||
}
|
||||
function toFailureLine(check) {
|
||||
return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
|
||||
}
|
||||
function toQualityGapFailureLine(check) {
|
||||
return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
|
||||
}
|
||||
function buildQualityGapChecks(input) {
|
||||
const { metrics, thresholds } = input;
|
||||
return [
|
||||
{
|
||||
metric: "generic_explanation_rate",
|
||||
value: metrics.generic_explanation_rate,
|
||||
threshold: thresholds.generic_explanation_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.generic_explanation_rate <= thresholds.generic_explanation_rate_max
|
||||
},
|
||||
{
|
||||
metric: "false_confidence_rate",
|
||||
value: metrics.false_confidence_rate,
|
||||
threshold: thresholds.false_confidence_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.false_confidence_rate <= thresholds.false_confidence_rate_max
|
||||
},
|
||||
{
|
||||
metric: "mechanism_specificity_score",
|
||||
value: metrics.mechanism_specificity_score,
|
||||
threshold: thresholds.mechanism_specificity_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.mechanism_specificity_score >= thresholds.mechanism_specificity_score_min
|
||||
},
|
||||
{
|
||||
metric: "followup_context_retention_score",
|
||||
value: metrics.followup_context_retention_score,
|
||||
threshold: thresholds.followup_context_retention_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.followup_context_retention_score >= thresholds.followup_context_retention_score_min
|
||||
}
|
||||
];
|
||||
}
|
||||
function evaluateP0AcceptanceGate(input) {
|
||||
const thresholds = {
|
||||
...p0_metric_definitions_1.P0_DEFAULT_ACCEPTANCE_THRESHOLDS,
|
||||
...(input.thresholds ?? {})
|
||||
};
|
||||
const checks = buildMetricChecks({
|
||||
metrics: input.metrics,
|
||||
thresholds
|
||||
});
|
||||
const blockingFailures = checks.filter((item) => item.severity === "blocking" && !item.passed);
|
||||
const qualityFailures = checks.filter((item) => item.severity === "quality" && !item.passed);
|
||||
let verdict = "P0_NOT_ACCEPTED";
|
||||
if (blockingFailures.length === 0 && qualityFailures.length === 0) {
|
||||
verdict = "P0_ACCEPTED";
|
||||
}
|
||||
else if (blockingFailures.length === 0) {
|
||||
verdict = "P0_ACCEPTED_WITH_LIMITATIONS";
|
||||
}
|
||||
const rationale = [];
|
||||
if (blockingFailures.length > 0) {
|
||||
rationale.push("Blocking metrics failed:");
|
||||
rationale.push(...blockingFailures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (qualityFailures.length > 0) {
|
||||
rationale.push("Quality metrics failed:");
|
||||
rationale.push(...qualityFailures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (rationale.length === 0) {
|
||||
rationale.push("All acceptance thresholds are satisfied.");
|
||||
}
|
||||
return {
|
||||
verdict,
|
||||
passed_all: checks.every((item) => item.passed),
|
||||
blocking_failures: blockingFailures,
|
||||
quality_failures: qualityFailures,
|
||||
metric_checks: checks,
|
||||
rationale
|
||||
};
|
||||
}
|
||||
function evaluateP0BaselineStabilityGate(input) {
|
||||
const acceptance = evaluateP0AcceptanceGate({
|
||||
metrics: input.metrics,
|
||||
thresholds: input.acceptanceThresholds
|
||||
});
|
||||
const qualityGapThresholds = {
|
||||
...p0_metric_definitions_1.P0_DEFAULT_QUALITY_GAP_THRESHOLDS,
|
||||
...(input.qualityGapThresholds ?? {})
|
||||
};
|
||||
const qualityGapChecks = buildQualityGapChecks({
|
||||
metrics: input.qualityGapMetrics,
|
||||
thresholds: qualityGapThresholds
|
||||
});
|
||||
const qualityGapFailures = qualityGapChecks.filter((item) => !item.passed);
|
||||
const baselineIntegrityPassed = acceptance.blocking_failures.length === 0;
|
||||
const qualityGapsOpen = acceptance.quality_failures.length > 0 || qualityGapFailures.length > 0;
|
||||
const verdict = baselineIntegrityPassed && !qualityGapsOpen ? "P0_BASELINE_STABLE" : "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS";
|
||||
const rationale = [];
|
||||
if (!baselineIntegrityPassed) {
|
||||
rationale.push("Blocking baseline regressions detected:");
|
||||
rationale.push(...acceptance.blocking_failures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (acceptance.quality_failures.length > 0) {
|
||||
rationale.push("Legacy quality thresholds still below full acceptance:");
|
||||
rationale.push(...acceptance.quality_failures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (qualityGapFailures.length > 0) {
|
||||
rationale.push("Wave 9 quality-gap thresholds still open:");
|
||||
rationale.push(...qualityGapFailures.map((item) => `- ${toQualityGapFailureLine(item)}`));
|
||||
}
|
||||
if (rationale.length === 0) {
|
||||
rationale.push("Baseline is stable and Wave 9 quality-gap thresholds are satisfied.");
|
||||
}
|
||||
return {
|
||||
verdict,
|
||||
baseline_integrity_passed: baselineIntegrityPassed,
|
||||
quality_gaps_open: qualityGapsOpen,
|
||||
blocking_regressions: acceptance.blocking_failures,
|
||||
legacy_quality_failures: acceptance.quality_failures,
|
||||
quality_gap_failures: qualityGapFailures,
|
||||
quality_gap_checks: qualityGapChecks,
|
||||
rationale
|
||||
};
|
||||
}
|
||||
+1368
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,278 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.P0_DEFAULT_QUALITY_GAP_THRESHOLDS = exports.P0_DEFAULT_ACCEPTANCE_THRESHOLDS = exports.P0_DEFAULT_FORBIDDEN_LEAKAGE_TOKENS = exports.P0_QUALITY_GAP_METRIC_DEFINITIONS = exports.P0_METRIC_DEFINITIONS = void 0;
|
||||
exports.validateP0EvalCorpus = validateP0EvalCorpus;
|
||||
exports.normalizeExpectedRoutes = normalizeExpectedRoutes;
|
||||
exports.P0_METRIC_DEFINITIONS = {
|
||||
problem_first_answer_rate: {
|
||||
description: "Share of cases where answer is problem-first and avoids entity-dump framing.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
mechanism_coherence_score: {
|
||||
description: "Average 0..5 score for mechanism clarity and alignment with expected defect class.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
entity_leakage_rate: {
|
||||
description: "Share of cases with internal/technical leakage in user-facing answer.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
accountant_actionability_score: {
|
||||
description: "Average 0..5 score for first-check concreteness and practical accounting actionability.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
route_correctness_rate: {
|
||||
description: "Share of cases routed to expected path class for query type.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
domain_purity_rate: {
|
||||
description: "Share of cases where top-3 retrieval stays inside expected P0 domain boundary.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
limitation_honesty_rate: {
|
||||
description: "Share of cases where mandatory limitations are explicit when evidence is incomplete.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
top_problem_unit_match_rate: {
|
||||
description: "Share of cases where top problem unit type matches expected annotation.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
}
|
||||
};
|
||||
exports.P0_QUALITY_GAP_METRIC_DEFINITIONS = {
|
||||
generic_explanation_rate: {
|
||||
description: "Share of cases where direct answer stays generic and mechanism/actionability specificity is insufficient.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
false_confidence_rate: {
|
||||
description: "Share of cases with overconfident framing not supported by evidence/limitation state.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
mechanism_specificity_score: {
|
||||
description: "Average 0..5 score of mechanism-level specificity in accountant-facing answer.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
followup_context_retention_score: {
|
||||
description: "Rate of correct context retention in follow-up investigation cases.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
}
|
||||
};
|
||||
exports.P0_DEFAULT_FORBIDDEN_LEAKAGE_TOKENS = [
|
||||
"graph_",
|
||||
"domain_scope",
|
||||
"relation_patterns",
|
||||
"account_scope",
|
||||
"semantic_profile",
|
||||
"route",
|
||||
"profile",
|
||||
"store_canonical",
|
||||
"hybrid_store_plus_live",
|
||||
"deterministic_v2",
|
||||
"candidate_evidence",
|
||||
"problem_unit",
|
||||
"raw_entities",
|
||||
"graph_domain_scope"
|
||||
];
|
||||
exports.P0_DEFAULT_ACCEPTANCE_THRESHOLDS = {
|
||||
problem_first_answer_rate_min: 0.8,
|
||||
mechanism_coherence_score_min: 3.5,
|
||||
entity_leakage_rate_max: 0.08,
|
||||
accountant_actionability_score_min: 3.5,
|
||||
route_correctness_rate_min: 0.85,
|
||||
domain_purity_rate_min: 0.9,
|
||||
limitation_honesty_rate_min: 0.9,
|
||||
top_problem_unit_match_rate_min: 0.65
|
||||
};
|
||||
exports.P0_DEFAULT_QUALITY_GAP_THRESHOLDS = {
|
||||
generic_explanation_rate_max: 0.2,
|
||||
false_confidence_rate_max: 0.12,
|
||||
mechanism_specificity_score_min: 3.0,
|
||||
followup_context_retention_score_min: 0.75
|
||||
};
|
||||
function isProblemUnitType(value) {
|
||||
return (value === "document_conflict" ||
|
||||
value === "broken_chain_segment" ||
|
||||
value === "lifecycle_anomaly_node" ||
|
||||
value === "unresolved_settlement_cluster" ||
|
||||
value === "period_risk_cluster" ||
|
||||
value === "cross_branch_inconsistency_cluster");
|
||||
}
|
||||
function isP0DomainId(value) {
|
||||
return value === "settlements_60_62" || value === "vat_document_register_book" || value === "month_close_costs_20_44";
|
||||
}
|
||||
function isP0QueryClass(value) {
|
||||
return (value === "symptom_first" ||
|
||||
value === "lifecycle_first" ||
|
||||
value === "causal_query" ||
|
||||
value === "mixed_ambiguity" ||
|
||||
value === "followup_investigation" ||
|
||||
value === "noisy_input" ||
|
||||
value === "translit_noisy" ||
|
||||
value === "multi_intent");
|
||||
}
|
||||
function isRouteExpected(value) {
|
||||
return (value === "hybrid_store_plus_live" ||
|
||||
value === "store_feature_risk" ||
|
||||
value === "store_canonical" ||
|
||||
value === "batch_refresh_then_store" ||
|
||||
value === "live_mcp_drilldown" ||
|
||||
value === "no_route");
|
||||
}
|
||||
function assertString(value, fieldPath) {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': non-empty string is required.`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
function assertBoolean(value, fieldPath) {
|
||||
if (typeof value !== "boolean") {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': boolean is required.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function assertStringArray(value, fieldPath) {
|
||||
if (!Array.isArray(value)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': string[] is required.`);
|
||||
}
|
||||
const output = value.map((item, index) => assertString(item, `${fieldPath}[${index}]`));
|
||||
return output;
|
||||
}
|
||||
function assertRouteExpected(value, fieldPath) {
|
||||
if (Array.isArray(value)) {
|
||||
const routes = value.map((item, index) => {
|
||||
if (!isRouteExpected(item)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}[${index}]': unsupported route '${String(item)}'.`);
|
||||
}
|
||||
return item;
|
||||
});
|
||||
if (routes.length === 0) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': route list must not be empty.`);
|
||||
}
|
||||
return routes;
|
||||
}
|
||||
if (!isRouteExpected(value)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': unsupported route '${String(value)}'.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function parseCase(raw, index) {
|
||||
if (!raw || typeof raw !== "object") {
|
||||
throw new Error(`Invalid corpus case at index ${index}: object is required.`);
|
||||
}
|
||||
const row = raw;
|
||||
const expected = row.expected_annotations;
|
||||
if (!expected || typeof expected !== "object") {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations is required.`);
|
||||
}
|
||||
const annotations = expected;
|
||||
const domain = row.domain;
|
||||
if (!isP0DomainId(domain)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported domain '${String(domain)}'.`);
|
||||
}
|
||||
const queryClass = row.query_class;
|
||||
if (!isP0QueryClass(queryClass)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported query_class '${String(queryClass)}'.`);
|
||||
}
|
||||
const expectedProblemType = row.expected_problem_unit_type;
|
||||
if (!isProblemUnitType(expectedProblemType)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': expected_problem_unit_type '${String(expectedProblemType)}' is not supported.`);
|
||||
}
|
||||
const annotationProblemType = annotations.problem_unit_type_expected;
|
||||
if (!isProblemUnitType(annotationProblemType)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.problem_unit_type_expected '${String(annotationProblemType)}' is not supported.`);
|
||||
}
|
||||
const annotationDomain = annotations.domain_expected;
|
||||
if (!isP0DomainId(annotationDomain)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.domain_expected '${String(annotationDomain)}' is not supported.`);
|
||||
}
|
||||
const followupSeedQuery = typeof row.followup_seed_query === "string" && row.followup_seed_query.trim() ? row.followup_seed_query.trim() : undefined;
|
||||
const followupExpectedContextTokens = Array.isArray(row.followup_expected_context_tokens)
|
||||
? assertStringArray(row.followup_expected_context_tokens, `cases[${index}].followup_expected_context_tokens`)
|
||||
: undefined;
|
||||
if (queryClass === "followup_investigation" && !followupSeedQuery) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': followup_investigation requires followup_seed_query.`);
|
||||
}
|
||||
return {
|
||||
case_id: assertString(row.case_id, `cases[${index}].case_id`),
|
||||
domain,
|
||||
user_query_raw: assertString(row.user_query_raw, `cases[${index}].user_query_raw`),
|
||||
query_class: queryClass,
|
||||
expected_problem_unit_type: expectedProblemType,
|
||||
expected_mechanism_summary: assertString(row.expected_mechanism_summary, `cases[${index}].expected_mechanism_summary`),
|
||||
expected_first_check: assertString(row.expected_first_check, `cases[${index}].expected_first_check`),
|
||||
must_include_limitations: assertBoolean(row.must_include_limitations, `cases[${index}].must_include_limitations`),
|
||||
forbidden_leakage_tokens: assertStringArray(row.forbidden_leakage_tokens, `cases[${index}].forbidden_leakage_tokens`),
|
||||
followup_seed_query: followupSeedQuery,
|
||||
followup_expected_context_tokens: followupExpectedContextTokens,
|
||||
notes: assertString(row.notes, `cases[${index}].notes`),
|
||||
expected_annotations: {
|
||||
domain_expected: annotationDomain,
|
||||
route_expected: assertRouteExpected(annotations.route_expected, `cases[${index}].expected_annotations.route_expected`),
|
||||
problem_unit_type_expected: annotationProblemType,
|
||||
mechanism_class_expected: assertString(annotations.mechanism_class_expected, `cases[${index}].expected_annotations.mechanism_class_expected`),
|
||||
evidence_expectation: assertString(annotations.evidence_expectation, `cases[${index}].expected_annotations.evidence_expectation`),
|
||||
must_show_limitation: assertBoolean(annotations.must_show_limitation, `cases[${index}].expected_annotations.must_show_limitation`),
|
||||
must_not_leak_internal: assertBoolean(annotations.must_not_leak_internal, `cases[${index}].expected_annotations.must_not_leak_internal`),
|
||||
first_check_action_expected: assertString(annotations.first_check_action_expected, `cases[${index}].expected_annotations.first_check_action_expected`)
|
||||
}
|
||||
};
|
||||
}
|
||||
function validateP0EvalCorpus(input) {
|
||||
if (!input || typeof input !== "object") {
|
||||
throw new Error("Invalid P0 corpus: root object is required.");
|
||||
}
|
||||
const parsed = input;
|
||||
const casesRaw = parsed.cases;
|
||||
if (!Array.isArray(casesRaw)) {
|
||||
throw new Error("Invalid P0 corpus: cases[] is required.");
|
||||
}
|
||||
const cases = casesRaw.map((item, index) => parseCase(item, index));
|
||||
if (cases.length < 30 || cases.length > 150) {
|
||||
throw new Error(`Invalid P0 corpus: expected 30..150 cases, got ${cases.length}.`);
|
||||
}
|
||||
const caseIds = assertStringArray(parsed.case_ids, "case_ids");
|
||||
const declaredIds = [...caseIds].sort();
|
||||
const actualIds = cases.map((item) => item.case_id).sort();
|
||||
const idsMatch = declaredIds.length === actualIds.length && declaredIds.every((value, index) => value === actualIds[index]);
|
||||
if (!idsMatch) {
|
||||
throw new Error("Invalid P0 corpus: case_ids[] does not match cases[].");
|
||||
}
|
||||
const scenarioCount = Number(parsed.scenario_count ?? 0);
|
||||
if (!Number.isFinite(scenarioCount) || scenarioCount !== cases.length) {
|
||||
throw new Error("Invalid P0 corpus: scenario_count must match cases.length.");
|
||||
}
|
||||
const perDomain = cases.reduce((acc, item) => {
|
||||
acc[item.domain] += 1;
|
||||
return acc;
|
||||
}, {
|
||||
settlements_60_62: 0,
|
||||
vat_document_register_book: 0,
|
||||
month_close_costs_20_44: 0
|
||||
});
|
||||
for (const [domain, count] of Object.entries(perDomain)) {
|
||||
if (count < 10) {
|
||||
throw new Error(`Invalid P0 corpus: domain '${domain}' must contain at least 10 cases, got ${count}.`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
suite_id: assertString(parsed.suite_id, "suite_id"),
|
||||
suite_version: assertString(parsed.suite_version, "suite_version"),
|
||||
schema_version: assertString(parsed.schema_version, "schema_version"),
|
||||
scenario_count: scenarioCount,
|
||||
case_ids: caseIds,
|
||||
cases
|
||||
};
|
||||
}
|
||||
function normalizeExpectedRoutes(value) {
|
||||
return Array.isArray(value) ? value : [value];
|
||||
}
|
||||
+1598
-201
File diff suppressed because it is too large
Load Diff
+1703
-79
File diff suppressed because it is too large
Load Diff
+267
-33
@@ -1,15 +1,51 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || (function () {
|
||||
var ownKeys = function(o) {
|
||||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||||
var ar = [];
|
||||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||||
return ar;
|
||||
};
|
||||
return ownKeys(o);
|
||||
};
|
||||
return function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
})();
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.AssistantService = void 0;
|
||||
const nanoid_1 = require("nanoid");
|
||||
const stage1Contracts_1 = require("../types/stage1Contracts");
|
||||
const config_1 = require("../config");
|
||||
const log_1 = require("../utils/log");
|
||||
const answerComposer_1 = require("./answerComposer");
|
||||
const assistantDataLayer_1 = require("./assistantDataLayer");
|
||||
const assistantSessionLogger_1 = require("./assistantSessionLogger");
|
||||
const investigationState_1 = require("./investigationState");
|
||||
const retrievalResultNormalizer_1 = require("./retrievalResultNormalizer");
|
||||
exports.evaluateCoverageForTests = evaluateCoverageForTests;
|
||||
exports.extractSubjectTokensForTests = extractSubjectTokensForTests;
|
||||
// @ts-nocheck
|
||||
const nanoid_1 = __importStar(require("nanoid"));
|
||||
const stage1Contracts_1 = __importStar(require("../types/stage1Contracts"));
|
||||
const config_1 = __importStar(require("../config"));
|
||||
const log_1 = __importStar(require("../utils/log"));
|
||||
const answerComposer_1 = __importStar(require("./answerComposer"));
|
||||
const assistantDataLayer_1 = __importStar(require("./assistantDataLayer"));
|
||||
const assistantSessionLogger_1 = __importStar(require("./assistantSessionLogger"));
|
||||
const investigationState_1 = __importStar(require("./investigationState"));
|
||||
const retrievalResultNormalizer_1 = __importStar(require("./retrievalResultNormalizer"));
|
||||
function retrievalSummaryForRoute(route) {
|
||||
if (route === "store_canonical")
|
||||
return "Canonical accounting data path selected.";
|
||||
@@ -56,6 +92,25 @@ function extractExecutionState(normalized) {
|
||||
};
|
||||
});
|
||||
}
|
||||
function escapeRegex(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
function enrichFragmentTextWithHints(fragment, text) {
|
||||
const baseText = String(text ?? "").trim();
|
||||
const accountHints = Array.isArray(fragment.account_hints)
|
||||
? Array.from(new Set(fragment.account_hints
|
||||
.map((item) => String(item ?? "").trim())
|
||||
.filter((item) => item.length > 0)))
|
||||
: [];
|
||||
if (accountHints.length === 0) {
|
||||
return baseText;
|
||||
}
|
||||
const hasAccountInText = accountHints.some((account) => new RegExp(`\\b${escapeRegex(account)}\\b`, "i").test(baseText));
|
||||
if (hasAccountInText) {
|
||||
return baseText;
|
||||
}
|
||||
return `${baseText}, по счету ${accountHints.join(", ")}`;
|
||||
}
|
||||
function fragmentTextById(normalized) {
|
||||
const result = new Map();
|
||||
for (const item of extractFragments(normalized)) {
|
||||
@@ -70,7 +125,7 @@ function fragmentTextById(normalized) {
|
||||
const text = (typeof fragment.raw_fragment_text === "string" && fragment.raw_fragment_text.trim()) ||
|
||||
(typeof fragment.normalized_fragment_text === "string" && fragment.normalized_fragment_text.trim()) ||
|
||||
"";
|
||||
result.set(fragmentId, text);
|
||||
result.set(fragmentId, enrichFragmentTextWithHints(fragment, text));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -95,13 +150,18 @@ function extractDiscardedIntentSegments(normalized) {
|
||||
}
|
||||
function collectDateSpans(text) {
|
||||
const spans = [];
|
||||
const datePattern = /\b20\d{2}[-/.](?:0[1-9]|1[0-2])(?:[-/.](?:0[1-9]|[12]\d|3[01]))?\b/g;
|
||||
let match = null;
|
||||
while ((match = datePattern.exec(text)) !== null) {
|
||||
spans.push({
|
||||
start: match.index,
|
||||
end: match.index + match[0].length
|
||||
});
|
||||
const datePatterns = [
|
||||
/\b20\d{2}[-/.](?:0[1-9]|1[0-2])(?:[-/.](?:0[1-9]|[12]\d|3[01]))?\b/g,
|
||||
/\b(?:0?[1-9]|[12]\d|3[01])[./-](?:0?[1-9]|1[0-2])[./-](?:\d{2}|\d{4})\b/g
|
||||
];
|
||||
for (const datePattern of datePatterns) {
|
||||
let match = null;
|
||||
while ((match = datePattern.exec(text)) !== null) {
|
||||
spans.push({
|
||||
start: match.index,
|
||||
end: match.index + match[0].length
|
||||
});
|
||||
}
|
||||
}
|
||||
return spans;
|
||||
}
|
||||
@@ -111,17 +171,80 @@ function intersectsAnySpan(start, end, spans) {
|
||||
function extractAccountTokens(text) {
|
||||
const lower = String(text ?? "").toLowerCase();
|
||||
const explicitAccounts = new Set();
|
||||
const knownAccountPrefixes = new Set([
|
||||
"01",
|
||||
"02",
|
||||
"07",
|
||||
"08",
|
||||
"10",
|
||||
"13",
|
||||
"19",
|
||||
"20",
|
||||
"21",
|
||||
"23",
|
||||
"25",
|
||||
"26",
|
||||
"28",
|
||||
"29",
|
||||
"41",
|
||||
"43",
|
||||
"44",
|
||||
"45",
|
||||
"50",
|
||||
"51",
|
||||
"52",
|
||||
"55",
|
||||
"57",
|
||||
"58",
|
||||
"60",
|
||||
"62",
|
||||
"66",
|
||||
"67",
|
||||
"68",
|
||||
"69",
|
||||
"70",
|
||||
"71",
|
||||
"73",
|
||||
"76",
|
||||
"90",
|
||||
"91",
|
||||
"94",
|
||||
"96",
|
||||
"97"
|
||||
]);
|
||||
const contextualPattern = /(?:\bсчет(?:а|у|ом|ов)?\b|\bсч\.?\b|\baccount(?:s)?\b|\bschet(?:a|u|om|ov)?\b)\s*(?:№|#|:)?\s*(\d{2}(?:\.\d{2})?)/giu;
|
||||
let contextual = null;
|
||||
while ((contextual = contextualPattern.exec(lower)) !== null) {
|
||||
if (contextual[1]) {
|
||||
explicitAccounts.add(contextual[1]);
|
||||
const token = String(contextual[1]).trim();
|
||||
const prefix = token.match(/^(\d{2})/)?.[1];
|
||||
if (prefix && knownAccountPrefixes.has(prefix)) {
|
||||
explicitAccounts.add(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
const pairPattern = /\b(\d{2}\.\d{2})\s*\/\s*(\d{2}\.\d{2})\b/g;
|
||||
let pairMatch = null;
|
||||
while ((pairMatch = pairPattern.exec(lower)) !== null) {
|
||||
const left = String(pairMatch[1] ?? "").trim();
|
||||
const right = String(pairMatch[2] ?? "").trim();
|
||||
const leftPrefix = left.match(/^(\d{2})/)?.[1];
|
||||
const rightPrefix = right.match(/^(\d{2})/)?.[1];
|
||||
if (leftPrefix && knownAccountPrefixes.has(leftPrefix)) {
|
||||
explicitAccounts.add(left);
|
||||
}
|
||||
if (rightPrefix && knownAccountPrefixes.has(rightPrefix)) {
|
||||
explicitAccounts.add(right);
|
||||
}
|
||||
}
|
||||
if (explicitAccounts.size > 0) {
|
||||
return Array.from(explicitAccounts);
|
||||
}
|
||||
const spans = collectDateSpans(lower);
|
||||
const hasAccountingLexeme = /(?:\bсчет(?:а|у|ом|ов)?\b|\bсч\.?\b|\baccount(?:s)?\b|\bschet(?:a|u|om|ov)?\b|оплат|расчет|аванс|долг|settlement|payment|счет|СЃС‡\.?)/iu.test(lower);
|
||||
if (!hasAccountingLexeme) {
|
||||
return [];
|
||||
}
|
||||
const accountList = [];
|
||||
const genericPattern = /\b\d{2}(?:\.\d{2})?\b/g;
|
||||
let generic = null;
|
||||
@@ -132,6 +255,10 @@ function extractAccountTokens(text) {
|
||||
if (intersectsAnySpan(start, end, spans)) {
|
||||
continue;
|
||||
}
|
||||
const prefix = value.match(/^(\d{2})/)?.[1];
|
||||
if (!prefix || !knownAccountPrefixes.has(prefix)) {
|
||||
continue;
|
||||
}
|
||||
accountList.push(value);
|
||||
}
|
||||
return Array.from(new Set(accountList));
|
||||
@@ -406,12 +533,50 @@ function evaluateSubjectTokenMatch(token, corpus, executedRoutes) {
|
||||
}
|
||||
return { matched: corpus.includes(token), critical: false };
|
||||
}
|
||||
function evidenceCountForRequirement(requirementId, result) {
|
||||
const evidence = Array.isArray(result.evidence) ? result.evidence : [];
|
||||
if (evidence.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const tagged = evidence.filter((item) => {
|
||||
const claimRef = typeof item?.claim_ref === "string" ? item.claim_ref : "";
|
||||
return claimRef.toLowerCase() === `requirement:${String(requirementId).toLowerCase()}`;
|
||||
}).length;
|
||||
if (tagged > 0) {
|
||||
return tagged;
|
||||
}
|
||||
if (Array.isArray(result.requirement_ids) &&
|
||||
result.requirement_ids.length === 1 &&
|
||||
result.requirement_ids[0] === requirementId) {
|
||||
return evidence.length;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
function hasSubstantiveCoverageForRequirement(requirementId, result) {
|
||||
const evidenceCount = evidenceCountForRequirement(requirementId, result);
|
||||
if (evidenceCount > 0) {
|
||||
return true;
|
||||
}
|
||||
const problemUnitsCount = Array.isArray(result.problem_units) ? result.problem_units.length : 0;
|
||||
const candidateEvidenceCount = Array.isArray(result.candidate_evidence) ? result.candidate_evidence.length : 0;
|
||||
if (problemUnitsCount > 0 || candidateEvidenceCount > 0) {
|
||||
if (Array.isArray(result.requirement_ids) &&
|
||||
result.requirement_ids.length === 1 &&
|
||||
result.requirement_ids[0] === requirementId) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function evaluateCoverage(requirements, retrievalResults) {
|
||||
const statusByRequirement = new Map();
|
||||
for (const result of retrievalResults) {
|
||||
for (const requirementId of result.requirement_ids) {
|
||||
const list = statusByRequirement.get(requirementId) ?? [];
|
||||
list.push(result.status);
|
||||
list.push({
|
||||
status: result.status,
|
||||
substantive: hasSubstantiveCoverageForRequirement(requirementId, result)
|
||||
});
|
||||
statusByRequirement.set(requirementId, list);
|
||||
}
|
||||
}
|
||||
@@ -419,19 +584,27 @@ function evaluateCoverage(requirements, retrievalResults) {
|
||||
if (requirement.status === "out_of_scope" || requirement.status === "clarification_needed") {
|
||||
return requirement;
|
||||
}
|
||||
const statuses = statusByRequirement.get(requirement.requirement_id) ?? [];
|
||||
if (statuses.length === 0) {
|
||||
const states = statusByRequirement.get(requirement.requirement_id) ?? [];
|
||||
if (states.length === 0) {
|
||||
return { ...requirement, status: "uncovered" };
|
||||
}
|
||||
if (statuses.includes("ok")) {
|
||||
const hasAnySubstantive = states.some((item) => item.substantive);
|
||||
if (!hasAnySubstantive) {
|
||||
return { ...requirement, status: "uncovered" };
|
||||
}
|
||||
const hasOk = states.some((item) => item.status === "ok");
|
||||
const hasPartial = states.some((item) => item.status === "partial");
|
||||
const hasEmpty = states.some((item) => item.status === "empty");
|
||||
const hasError = states.some((item) => item.status === "error");
|
||||
const hasWeakOk = states.some((item) => item.status === "ok" && !item.substantive);
|
||||
const hasSubstantiveOk = states.some((item) => item.status === "ok" && item.substantive);
|
||||
const hasSubstantivePartial = states.some((item) => item.status === "partial" && item.substantive);
|
||||
if (hasSubstantiveOk && !hasSubstantivePartial && !hasWeakOk && !hasEmpty && !hasError) {
|
||||
return { ...requirement, status: "covered" };
|
||||
}
|
||||
if (statuses.includes("partial")) {
|
||||
if (hasSubstantiveOk || hasSubstantivePartial || hasOk || hasPartial) {
|
||||
return { ...requirement, status: "partially_covered" };
|
||||
}
|
||||
if (statuses.includes("empty") && !statuses.includes("error")) {
|
||||
return { ...requirement, status: "covered" };
|
||||
}
|
||||
return { ...requirement, status: "uncovered" };
|
||||
});
|
||||
const requirementsCovered = resolvedRequirements.filter((item) => item.status === "covered").length;
|
||||
@@ -459,6 +632,12 @@ function evaluateCoverage(requirements, retrievalResults) {
|
||||
}
|
||||
};
|
||||
}
|
||||
function evaluateCoverageForTests(requirements, retrievalResults) {
|
||||
return evaluateCoverage(requirements, retrievalResults);
|
||||
}
|
||||
function extractSubjectTokensForTests(text) {
|
||||
return extractSubjectTokens(text);
|
||||
}
|
||||
function checkGrounding(userMessage, requirements, coverage, retrievalResults) {
|
||||
const whyIncludedSummary = summarizeUnique(retrievalResults.flatMap((item) => item.why_included));
|
||||
const selectionReasonSummary = summarizeUnique(retrievalResults.flatMap((item) => item.selection_reason));
|
||||
@@ -627,6 +806,11 @@ function buildAnswerStructureV11(input) {
|
||||
};
|
||||
}
|
||||
const FOLLOWUP_ROUTE_HINTS = new Set(["store_canonical", "store_feature_risk", "hybrid_store_plus_live", "live_mcp_drilldown", "batch_refresh_then_store"]);
|
||||
const FOLLOWUP_ACTIVE_DOMAIN_ROUTE_MAP = {
|
||||
settlements_60_62: "hybrid_store_plus_live",
|
||||
vat_document_register_book: "hybrid_store_plus_live",
|
||||
month_close_costs_20_44: "hybrid_store_plus_live"
|
||||
};
|
||||
const FOLLOWUP_BUSINESS_CONTEXT_MAX = 320;
|
||||
const FOLLOWUP_SUBJECT_MAX = 160;
|
||||
const FOLLOWUP_QUESTION_APPEND_MAX = 260;
|
||||
@@ -669,6 +853,23 @@ function extractNormalizedPeriodLiteral(text) {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function extractFollowupAccountAnchorsLoose(text) {
|
||||
const lower = String(text ?? "").toLowerCase();
|
||||
const spans = collectDateSpans(lower);
|
||||
const anchors = [];
|
||||
const followupAccountPattern = /\b(?:01|02|08|19|20|21|23|25|26|28|29|44|51|60|62|68|76|97)(?:\.\d{2})?\b/g;
|
||||
let match = null;
|
||||
while ((match = followupAccountPattern.exec(lower)) !== null) {
|
||||
const value = String(match[0] ?? "").trim();
|
||||
const start = match.index;
|
||||
const end = start + value.length;
|
||||
if (intersectsAnySpan(start, end, spans)) {
|
||||
continue;
|
||||
}
|
||||
anchors.push(value);
|
||||
}
|
||||
return Array.from(new Set(anchors));
|
||||
}
|
||||
function hasStrongFollowupAnchors(userMessage, state) {
|
||||
const explicitPeriod = extractNormalizedPeriodLiteral(userMessage);
|
||||
if (explicitPeriod && state.focus.period && explicitPeriod !== state.focus.period) {
|
||||
@@ -678,12 +879,13 @@ function hasStrongFollowupAnchors(userMessage, state) {
|
||||
}
|
||||
}
|
||||
const explicitAccounts = extractAccountTokens(userMessage);
|
||||
if (explicitAccounts.length > 0) {
|
||||
const followupAccounts = explicitAccounts.length > 0 ? explicitAccounts : extractFollowupAccountAnchorsLoose(userMessage);
|
||||
if (followupAccounts.length > 0) {
|
||||
const knownAccounts = new Set(state.focus.primary_accounts.map((item) => item.trim()));
|
||||
if (knownAccounts.size === 0) {
|
||||
return true;
|
||||
}
|
||||
if (explicitAccounts.some((item) => !knownAccounts.has(item))) {
|
||||
if (followupAccounts.some((item) => !knownAccounts.has(item))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -692,7 +894,10 @@ function hasStrongFollowupAnchors(userMessage, state) {
|
||||
function routeFromInvestigationState(state) {
|
||||
const rawDomain = compactWhitespace(state.focus.domain ?? "");
|
||||
if (!rawDomain) {
|
||||
return null;
|
||||
const mappedFromFollowup = state.followup_context?.active_domain
|
||||
? FOLLOWUP_ACTIVE_DOMAIN_ROUTE_MAP[compactWhitespace(state.followup_context.active_domain)] ?? null
|
||||
: null;
|
||||
return mappedFromFollowup;
|
||||
}
|
||||
if (FOLLOWUP_ROUTE_HINTS.has(rawDomain)) {
|
||||
return rawDomain;
|
||||
@@ -701,6 +906,15 @@ function routeFromInvestigationState(state) {
|
||||
if (FOLLOWUP_ROUTE_HINTS.has(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(FOLLOWUP_ACTIVE_DOMAIN_ROUTE_MAP, candidate)) {
|
||||
return FOLLOWUP_ACTIVE_DOMAIN_ROUTE_MAP[candidate];
|
||||
}
|
||||
}
|
||||
const mappedFromFollowup = state.followup_context?.active_domain
|
||||
? FOLLOWUP_ACTIVE_DOMAIN_ROUTE_MAP[compactWhitespace(state.followup_context.active_domain)] ?? null
|
||||
: null;
|
||||
if (mappedFromFollowup) {
|
||||
return mappedFromFollowup;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -752,6 +966,7 @@ function buildFollowupStateBinding(input) {
|
||||
const hasExplicitExpectedRoute = Boolean(input.payloadContext?.expected_route);
|
||||
const expectedRouteFromState = !context?.expected_route ? routeFromInvestigationState(input.investigationState) : null;
|
||||
const periodHintFromState = !context?.period_hint ? input.investigationState.focus.period : null;
|
||||
const followupContext = input.investigationState.followup_context;
|
||||
if (expectedRouteFromState) {
|
||||
context.expected_route = expectedRouteFromState;
|
||||
}
|
||||
@@ -771,6 +986,24 @@ function buildFollowupStateBinding(input) {
|
||||
if (input.investigationState.focus.primary_accounts.length > 0) {
|
||||
businessContextPatch.push(`focus_accounts:${input.investigationState.focus.primary_accounts.join(",")}`);
|
||||
}
|
||||
if (followupContext?.active_domain) {
|
||||
businessContextPatch.push(`focus_domain:${followupContext.active_domain}`);
|
||||
}
|
||||
if ((followupContext?.active_requirement_ids?.length ?? 0) > 0) {
|
||||
businessContextPatch.push(`active_requirements:${followupContext.active_requirement_ids.slice(0, 4).join(",")}`);
|
||||
}
|
||||
if ((followupContext?.uncovered_requirement_ids?.length ?? 0) > 0) {
|
||||
businessContextPatch.push(`uncovered_requirements:${followupContext.uncovered_requirement_ids.slice(0, 4).join(",")}`);
|
||||
}
|
||||
if (followupContext?.last_problem_unit_id) {
|
||||
businessContextPatch.push(`last_problem_unit:${followupContext.last_problem_unit_id}`);
|
||||
}
|
||||
if ((followupContext?.evidence_summary?.length ?? 0) > 0) {
|
||||
businessContextPatch.push(`evidence_state:${followupContext.evidence_summary.slice(0, 3).join("|")}`);
|
||||
}
|
||||
if ((followupContext?.settlement_next_actions?.length ?? 0) > 0) {
|
||||
businessContextPatch.push("settlement_focus_retained_v1");
|
||||
}
|
||||
if (problemContinuityAvailable) {
|
||||
if (hasExplicitExpectedRoute) {
|
||||
problemContinuitySkippedReason = "explicit_expected_route";
|
||||
@@ -800,9 +1033,6 @@ function buildFollowupStateBinding(input) {
|
||||
if (periodHintFromState && !hasPeriodLiteral(userMessage)) {
|
||||
appendParts.push(`Период фокуса: ${periodHintFromState}`);
|
||||
}
|
||||
if (problemContinuityApplied && (problemState?.focus_problem_types.length ?? 0) > 0) {
|
||||
appendParts.push(`Problem focus types: ${(problemState?.focus_problem_types ?? []).slice(0, 3).join(", ")}`);
|
||||
}
|
||||
const appendBlock = withCappedLength(compactWhitespace(appendParts.join("; ")), FOLLOWUP_QUESTION_APPEND_MAX);
|
||||
normalizedQuestion = `${userMessage}\n${appendBlock}`.trim();
|
||||
}
|
||||
@@ -922,7 +1152,7 @@ class AssistantService {
|
||||
reason: null
|
||||
});
|
||||
try {
|
||||
const raw = this.dataLayer.executeRoute(planItem.route, planItem.fragment_text);
|
||||
const raw = await this.dataLayer.executeRouteRuntime(planItem.route, planItem.fragment_text);
|
||||
retrievalResultsRaw.push({
|
||||
fragment_id: planItem.fragment_id,
|
||||
route: planItem.route,
|
||||
@@ -960,6 +1190,9 @@ class AssistantService {
|
||||
}
|
||||
const coverageEvaluation = evaluateCoverage(requirementExtraction.requirements, retrievalResults);
|
||||
const groundingCheck = checkGrounding(userMessage, coverageEvaluation.requirements, coverageEvaluation.coverage, retrievalResults);
|
||||
const focusDomainHint = followupBinding.usage?.applied
|
||||
? session.investigation_state?.followup_context?.active_domain ?? session.investigation_state?.focus.domain ?? null
|
||||
: null;
|
||||
const composition = (0, answerComposer_1.composeAssistantAnswer)({
|
||||
userMessage,
|
||||
routeSummary: normalized.route_hint_summary,
|
||||
@@ -967,6 +1200,7 @@ class AssistantService {
|
||||
requirements: coverageEvaluation.requirements,
|
||||
coverageReport: coverageEvaluation.coverage,
|
||||
groundingCheck,
|
||||
focusDomainHint,
|
||||
enableAnswerPolicyV11: config_1.FEATURE_ASSISTANT_ANSWER_POLICY_V11,
|
||||
enableProblemCentricAnswerV1: config_1.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1,
|
||||
enableLifecycleAnswerV1: config_1.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1
|
||||
|
||||
+43
-34
@@ -158,41 +158,50 @@ class AssistantSessionLogger {
|
||||
this.rootDir = rootDir;
|
||||
}
|
||||
persistSession(session) {
|
||||
(0, files_1.ensureDir)(this.rootDir);
|
||||
const filePath = path_1.default.resolve(this.rootDir, `${session.session_id}.json`);
|
||||
const startedAt = session.items[0]?.created_at ?? session.updated_at;
|
||||
const userMessages = session.items.filter((item) => item.role === "user").length;
|
||||
const assistantMessages = session.items.filter((item) => item.role === "assistant").length;
|
||||
const assistantItems = session.items.filter((item) => item.role === "assistant");
|
||||
const lastAssistant = assistantItems.length > 0 ? assistantItems[assistantItems.length - 1] : null;
|
||||
const traceIds = unique(session.items.map((item) => item.trace_id));
|
||||
const replyTypes = Array.from(new Set(session.items
|
||||
.map((item) => item.reply_type)
|
||||
.filter((item) => typeof item === "string" && item.length > 0)));
|
||||
const turns = buildTurns(session.items);
|
||||
const record = {
|
||||
schema_version: "assistant_session_log_v1",
|
||||
session_id: session.session_id,
|
||||
started_at: startedAt,
|
||||
updated_at: session.updated_at,
|
||||
counters: {
|
||||
total_messages: session.items.length,
|
||||
user_messages: userMessages,
|
||||
assistant_messages: assistantMessages
|
||||
},
|
||||
trace_ids: traceIds,
|
||||
reply_types: replyTypes,
|
||||
investigation_state: session.investigation_state,
|
||||
turns,
|
||||
conversation: session.items,
|
||||
last_assistant: {
|
||||
message_id: lastAssistant?.message_id ?? null,
|
||||
reply_type: lastAssistant?.reply_type ?? null,
|
||||
trace_id: lastAssistant?.trace_id ?? null,
|
||||
created_at: lastAssistant?.created_at ?? null
|
||||
try {
|
||||
(0, files_1.ensureDir)(this.rootDir);
|
||||
const filePath = path_1.default.resolve(this.rootDir, `${session.session_id}.json`);
|
||||
const startedAt = session.items[0]?.created_at ?? session.updated_at;
|
||||
const userMessages = session.items.filter((item) => item.role === "user").length;
|
||||
const assistantMessages = session.items.filter((item) => item.role === "assistant").length;
|
||||
const assistantItems = session.items.filter((item) => item.role === "assistant");
|
||||
const lastAssistant = assistantItems.length > 0 ? assistantItems[assistantItems.length - 1] : null;
|
||||
const traceIds = unique(session.items.map((item) => item.trace_id));
|
||||
const replyTypes = Array.from(new Set(session.items
|
||||
.map((item) => item.reply_type)
|
||||
.filter((item) => typeof item === "string" && item.length > 0)));
|
||||
const turns = buildTurns(session.items);
|
||||
const record = {
|
||||
schema_version: "assistant_session_log_v1",
|
||||
session_id: session.session_id,
|
||||
started_at: startedAt,
|
||||
updated_at: session.updated_at,
|
||||
counters: {
|
||||
total_messages: session.items.length,
|
||||
user_messages: userMessages,
|
||||
assistant_messages: assistantMessages
|
||||
},
|
||||
trace_ids: traceIds,
|
||||
reply_types: replyTypes,
|
||||
investigation_state: session.investigation_state,
|
||||
turns,
|
||||
conversation: session.items,
|
||||
last_assistant: {
|
||||
message_id: lastAssistant?.message_id ?? null,
|
||||
reply_type: lastAssistant?.reply_type ?? null,
|
||||
trace_id: lastAssistant?.trace_id ?? null,
|
||||
created_at: lastAssistant?.created_at ?? null
|
||||
}
|
||||
};
|
||||
(0, files_1.writeJsonFile)(filePath, record);
|
||||
}
|
||||
catch (error) {
|
||||
const code = error?.code;
|
||||
if (code === "ENOSPC") {
|
||||
return;
|
||||
}
|
||||
};
|
||||
(0, files_1.writeJsonFile)(filePath, record);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.AssistantSessionLogger = AssistantSessionLogger;
|
||||
|
||||
+156
-38
@@ -8,6 +8,7 @@ const fs_1 = __importDefault(require("fs"));
|
||||
const path_1 = __importDefault(require("path"));
|
||||
const nanoid_1 = require("nanoid");
|
||||
const config_1 = require("../config");
|
||||
const p0_eval_runner_1 = require("../eval/p0_eval_runner");
|
||||
const stage1Contracts_1 = require("../types/stage1Contracts");
|
||||
const stage2EvalContracts_1 = require("../types/stage2EvalContracts");
|
||||
const http_1 = require("../utils/http");
|
||||
@@ -218,6 +219,91 @@ const ASSISTANT_STAGE1_COMPARISON_SCHEMA_VERSION = "assistant_stage1_eval_compar
|
||||
const DEFAULT_ASSISTANT_STAGE2_SUITE_FILE = "assistant_stage2_canonical_v0_1.json";
|
||||
const ASSISTANT_STAGE2_RUN_SCHEMA_VERSION = "assistant_stage2_eval_run_v0_1";
|
||||
const ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION = "assistant_stage2_eval_comparison_v0_1";
|
||||
const INMEM_EVAL_REPORT_PREFIX = "inmem_eval_report:";
|
||||
const INMEM_EVAL_REPORTS = new Map();
|
||||
function isNoSpaceError(error) {
|
||||
const code = error?.code;
|
||||
return code === "ENOSPC";
|
||||
}
|
||||
function tryWriteJsonFile(pathname, value) {
|
||||
try {
|
||||
(0, files_1.writeJsonFile)(pathname, value);
|
||||
return true;
|
||||
}
|
||||
catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function tryWriteTextFile(pathname, value) {
|
||||
try {
|
||||
fs_1.default.writeFileSync(pathname, value, "utf-8");
|
||||
return true;
|
||||
}
|
||||
catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function putInMemoryEvalReport(report) {
|
||||
const key = `${INMEM_EVAL_REPORT_PREFIX}${(0, nanoid_1.nanoid)(12)}`;
|
||||
INMEM_EVAL_REPORTS.set(key, report);
|
||||
return key;
|
||||
}
|
||||
function readEvalReportByRef(ref) {
|
||||
if (ref.startsWith(INMEM_EVAL_REPORT_PREFIX)) {
|
||||
const report = INMEM_EVAL_REPORTS.get(ref);
|
||||
if (!report) {
|
||||
throw new Error(`In-memory eval report not found: ${ref}`);
|
||||
}
|
||||
return {
|
||||
report,
|
||||
resolved_path: ref
|
||||
};
|
||||
}
|
||||
const resolvedPath = resolveReadablePath(ref);
|
||||
const report = JSON.parse(fs_1.default.readFileSync(resolvedPath, "utf-8"));
|
||||
return {
|
||||
report,
|
||||
resolved_path: resolvedPath
|
||||
};
|
||||
}
|
||||
function compactAssistantStage1Report(report) {
|
||||
const results = Array.isArray(report.results) ? report.results : [];
|
||||
const compactResults = results.map((item) => ({
|
||||
case_id: item.case_id ?? null,
|
||||
scenario_tag: item.scenario_tag ?? null,
|
||||
accountant_usefulness_score: item.accountant_usefulness_score ?? null,
|
||||
accountant_metrics: typeof item.accountant_metrics === "object" && item.accountant_metrics !== null ? item.accountant_metrics : null
|
||||
}));
|
||||
return {
|
||||
...report,
|
||||
results: compactResults
|
||||
};
|
||||
}
|
||||
function compactAssistantStage2Report(report) {
|
||||
const results = Array.isArray(report.results) ? report.results : [];
|
||||
const compactResults = results.map((item) => {
|
||||
const metricSubscores = (item.metric_subscores ?? {});
|
||||
return {
|
||||
case_id: item.case_id ?? null,
|
||||
metric_subscores: {
|
||||
problem_clarity_score: metricSubscores.problem_clarity_score ?? null,
|
||||
mechanism_coherence_score: metricSubscores.mechanism_coherence_score ?? null,
|
||||
problem_first_answer_rate: metricSubscores.problem_first_answer_rate ?? null,
|
||||
entity_leakage_rate: metricSubscores.entity_leakage_rate ?? null
|
||||
}
|
||||
};
|
||||
});
|
||||
return {
|
||||
...report,
|
||||
results: compactResults
|
||||
};
|
||||
}
|
||||
const KNOWN_PROBLEM_UNIT_TYPES = [
|
||||
"document_conflict",
|
||||
"broken_chain_segment",
|
||||
@@ -900,7 +986,7 @@ class EvalService {
|
||||
results
|
||||
};
|
||||
(0, files_1.ensureDir)(config_1.EVAL_CASES_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
tryWriteJsonFile(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
return report;
|
||||
}
|
||||
collectAssistantSignals(finalResponse, turnResponses) {
|
||||
@@ -1237,8 +1323,9 @@ class EvalService {
|
||||
};
|
||||
}
|
||||
buildAssistantComparisonReport(input) {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
|
||||
const baselineRef = readEvalReportByRef(input.baselineReportFile);
|
||||
const baselinePath = baselineRef.resolved_path;
|
||||
const baselineReport = baselineRef.report;
|
||||
const currentReport = input.currentReport;
|
||||
const metricKeys = [
|
||||
"retrieval_differentiation_rate",
|
||||
@@ -1330,19 +1417,21 @@ class EvalService {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
const jsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.json`);
|
||||
const mdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.md`);
|
||||
(0, files_1.writeJsonFile)(jsonPath, comparisonReport);
|
||||
fs_1.default.writeFileSync(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport), "utf-8");
|
||||
const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
|
||||
const mdWritten = tryWriteTextFile(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport));
|
||||
const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
|
||||
return {
|
||||
...comparisonReport,
|
||||
artifacts: {
|
||||
comparison_report_json_path: jsonPath,
|
||||
comparison_report_md_path: mdPath
|
||||
comparison_report_json_path: comparisonRef,
|
||||
comparison_report_md_path: mdWritten ? mdPath : null
|
||||
}
|
||||
};
|
||||
}
|
||||
buildAssistantStage2ComparisonReport(input) {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
|
||||
const baselineRef = readEvalReportByRef(input.baselineReportFile);
|
||||
const baselinePath = baselineRef.resolved_path;
|
||||
const baselineReport = baselineRef.report;
|
||||
const currentReport = input.currentReport;
|
||||
const metricKeys = [
|
||||
"problem_unit_precision",
|
||||
@@ -1446,13 +1535,14 @@ class EvalService {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
const jsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.json`);
|
||||
const mdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.md`);
|
||||
(0, files_1.writeJsonFile)(jsonPath, comparisonReport);
|
||||
fs_1.default.writeFileSync(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport), "utf-8");
|
||||
const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
|
||||
const mdWritten = tryWriteTextFile(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport));
|
||||
const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
|
||||
return {
|
||||
...comparisonReport,
|
||||
artifacts: {
|
||||
comparison_report_json_path: jsonPath,
|
||||
comparison_report_md_path: mdPath
|
||||
comparison_report_json_path: comparisonRef,
|
||||
comparison_report_md_path: mdWritten ? mdPath : null
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1473,7 +1563,7 @@ class EvalService {
|
||||
const limitations = [];
|
||||
try {
|
||||
for (const turn of suiteCase.turns) {
|
||||
const response = await assistantService.handleMessage({
|
||||
const response = (await assistantService.handleMessage({
|
||||
session_id: sessionId,
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
@@ -1489,7 +1579,7 @@ class EvalService {
|
||||
domainPrompt: payload.normalizeConfig.domainPrompt,
|
||||
fewShotExamples: payload.normalizeConfig.fewShotExamples,
|
||||
useMock: payload.useMock
|
||||
});
|
||||
}));
|
||||
turnResponses.push(response);
|
||||
requestsTotal += 1;
|
||||
}
|
||||
@@ -1773,11 +1863,13 @@ class EvalService {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
const runJsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.json`);
|
||||
const runMdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.md`);
|
||||
(0, files_1.writeJsonFile)(runJsonPath, report);
|
||||
fs_1.default.writeFileSync(runMdPath, buildAssistantEvalMarkdownReport(report), "utf-8");
|
||||
const compactReport = compactAssistantStage1Report(report);
|
||||
const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
|
||||
const mdWritten = tryWriteTextFile(runMdPath, buildAssistantEvalMarkdownReport(compactReport));
|
||||
const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
|
||||
report.artifacts = {
|
||||
run_report_json_path: runJsonPath,
|
||||
run_report_md_path: runMdPath
|
||||
run_report_json_path: runReportRef,
|
||||
run_report_md_path: mdWritten ? runMdPath : null
|
||||
};
|
||||
if (payload.compareWithReportFile) {
|
||||
report.comparison = this.buildAssistantComparisonReport({
|
||||
@@ -1806,7 +1898,7 @@ class EvalService {
|
||||
const expectedProblemFirst = suiteCase.expected_hints?.expected_problem_first ?? (suiteCase.broadness_level !== "low" || suiteCase.question_type !== "direct");
|
||||
try {
|
||||
for (const turn of suiteCase.turns) {
|
||||
const response = await assistantService.handleMessage({
|
||||
const response = (await assistantService.handleMessage({
|
||||
session_id: sessionId,
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
@@ -1822,7 +1914,7 @@ class EvalService {
|
||||
domainPrompt: payload.normalizeConfig.domainPrompt,
|
||||
fewShotExamples: payload.normalizeConfig.fewShotExamples,
|
||||
useMock: payload.useMock
|
||||
});
|
||||
}));
|
||||
turnResponses.push(response);
|
||||
requestsTotal += 1;
|
||||
}
|
||||
@@ -2045,11 +2137,13 @@ class EvalService {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
const runJsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.json`);
|
||||
const runMdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.md`);
|
||||
(0, files_1.writeJsonFile)(runJsonPath, report);
|
||||
fs_1.default.writeFileSync(runMdPath, buildAssistantStage2EvalMarkdownReport(report), "utf-8");
|
||||
const compactReport = compactAssistantStage2Report(report);
|
||||
const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
|
||||
const mdWritten = tryWriteTextFile(runMdPath, buildAssistantStage2EvalMarkdownReport(compactReport));
|
||||
const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
|
||||
report.artifacts = {
|
||||
run_report_json_path: runJsonPath,
|
||||
run_report_md_path: runMdPath
|
||||
run_report_json_path: runReportRef,
|
||||
run_report_md_path: mdWritten ? runMdPath : null
|
||||
};
|
||||
if (payload.compareWithReportFile) {
|
||||
report.comparison = this.buildAssistantStage2ComparisonReport({
|
||||
@@ -2059,6 +2153,20 @@ class EvalService {
|
||||
}
|
||||
return report;
|
||||
}
|
||||
async runAssistantP0(payload) {
|
||||
if (!config_1.FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
|
||||
throw new http_1.ApiError("ASSISTANT_P0_EVAL_DISABLED", "Assistant P0 eval target is disabled by FEATURE_ASSISTANT_STAGE2_EVAL_V1.", 409);
|
||||
}
|
||||
const runner = new p0_eval_runner_1.P0EvalRunner(this.normalizerService);
|
||||
return runner.run({
|
||||
normalizeConfig: payload.normalizeConfig,
|
||||
caseIds: payload.caseIds,
|
||||
useMock: payload.useMock,
|
||||
mode: payload.mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
async run(payload) {
|
||||
const mode = payload.mode ?? "standard";
|
||||
const evalTarget = payload.evalTarget ?? "normalizer";
|
||||
@@ -2082,6 +2190,16 @@ class EvalService {
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
if (evalTarget === "assistant_p0") {
|
||||
return this.runAssistantP0({
|
||||
normalizeConfig: payload.normalizeConfig,
|
||||
caseIds: payload.caseIds,
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
const promptVersion = String(payload.normalizeConfig.promptVersion ?? "").toLowerCase();
|
||||
const schemaVersion = String(payload.normalizeConfig.schemaVersion ?? "").toLowerCase();
|
||||
const isV2 = promptVersion.startsWith("normalizer_v2") || schemaVersion === "v2" || schemaVersion === "v2_0_1" || schemaVersion === "v2_0_2";
|
||||
@@ -2269,17 +2387,17 @@ class EvalService {
|
||||
results
|
||||
};
|
||||
(0, files_1.ensureDir)(config_1.EVAL_CASES_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
tryWriteJsonFile(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
const shouldWriteV11Artifacts = mode === "single-pass-strict" &&
|
||||
Boolean(payload.caseSetFile) &&
|
||||
path_1.default.basename(String(payload.caseSetFile)).toLowerCase() === "normalizer_eval_v1_1_30cases.json";
|
||||
if (shouldWriteV11Artifacts) {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
|
||||
fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.md"), buildMarkdownReport({
|
||||
tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
|
||||
tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.md"), buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1 Eval Run"
|
||||
}), "utf-8");
|
||||
}));
|
||||
}
|
||||
const shouldWriteV1121EvalArtifacts = mode === "single-pass-strict" &&
|
||||
String(payload.normalizeConfig.promptVersion ?? "") === "normalizer_v1_1_2_1" &&
|
||||
@@ -2287,33 +2405,33 @@ class EvalService {
|
||||
path_1.default.basename(String(payload.caseSetFile)).toLowerCase() === "normalizer_eval_v1_1_2_1_30cases.json";
|
||||
if (shouldWriteV1121EvalArtifacts) {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
|
||||
fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.md"), buildMarkdownReport({
|
||||
tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
|
||||
tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.md"), buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.2.1 Eval Run"
|
||||
}), "utf-8");
|
||||
}));
|
||||
}
|
||||
const shouldWriteV111MicroArtifacts = mode === "single-pass-strict" &&
|
||||
String(payload.normalizeConfig.promptVersion ?? "") === "normalizer_v1_1_1" &&
|
||||
isSameCaseSet(payload.caseIds, V111_MICRO_CASE_IDS);
|
||||
if (shouldWriteV111MicroArtifacts) {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
|
||||
fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.md"), buildMarkdownReport({
|
||||
tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
|
||||
tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.md"), buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.1 Micro Eval"
|
||||
}), "utf-8");
|
||||
}));
|
||||
}
|
||||
const shouldWriteV112MicroArtifacts = mode === "single-pass-strict" &&
|
||||
String(payload.normalizeConfig.promptVersion ?? "") === "normalizer_v1_1_2" &&
|
||||
isSameCaseSet(payload.caseIds, V112_MICRO_CASE_IDS);
|
||||
if (shouldWriteV112MicroArtifacts) {
|
||||
(0, files_1.ensureDir)(config_1.REPORTS_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
|
||||
fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.md"), buildMarkdownReport({
|
||||
tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
|
||||
tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.md"), buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.2 Micro Eval"
|
||||
}), "utf-8");
|
||||
}));
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
+114
-4
@@ -75,6 +75,78 @@ function collectOpenUncertainties(coverageReport, retrievalResults) {
|
||||
const limitationNotes = retrievalResults.flatMap((result) => result.limitations).slice(0, 6);
|
||||
return capStrings([...requirementNotes, ...limitationNotes], stage1Contracts_1.INVESTIGATION_MAX_UNCERTAINTIES);
|
||||
}
|
||||
function normalizeAccountPrefix(value) {
|
||||
const account = String(value ?? "").trim();
|
||||
if (!account) {
|
||||
return null;
|
||||
}
|
||||
const match = account.match(/^(\d{2})/);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
function isSettlementAccount(value) {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
return prefix === "60" || prefix === "62" || prefix === "51" || prefix === "76";
|
||||
}
|
||||
function isVatAccount(value) {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
return prefix === "19" || prefix === "68";
|
||||
}
|
||||
function isCloseCostsAccount(value) {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
if (!prefix) {
|
||||
return false;
|
||||
}
|
||||
const account = Number(prefix);
|
||||
return (account >= 20 && account <= 44) || prefix === "97";
|
||||
}
|
||||
function inferFollowupActiveDomain(input) {
|
||||
const corpus = `${input.userMessage} ${input.previous.focus.active_query_subject ?? ""}`.toLowerCase();
|
||||
const hasSettlementSignal = input.focusAccounts.some((item) => isSettlementAccount(item)) ||
|
||||
/(60(?:\.\d{2})?|62(?:\.\d{2})?|оплат|расчет|расч[её]т|зачет|зач[её]т|аванс|долг|поставщ|покупат|settlement|payment|supplier|customer)/i.test(corpus);
|
||||
if (hasSettlementSignal) {
|
||||
return "settlements_60_62";
|
||||
}
|
||||
const hasVatSignal = input.focusAccounts.some((item) => isVatAccount(item)) ||
|
||||
/(ндс|счет[\s-]?фактур|сч[её]т[\s-]?фактур|книг[аи]|vat|invoice|book|register)/i.test(corpus);
|
||||
if (hasVatSignal) {
|
||||
return "vat_document_register_book";
|
||||
}
|
||||
const hasCloseSignal = input.focusAccounts.some((item) => isCloseCostsAccount(item)) ||
|
||||
/(закрыти|закрытие|месяц|затрат|распредел|списан|period\s*close|month\s*close|allocation|residual|cost)/i.test(corpus);
|
||||
if (hasCloseSignal) {
|
||||
return "month_close_costs_20_44";
|
||||
}
|
||||
const routeDomain = deriveDomain(input.routeSummary);
|
||||
if (routeDomain && routeDomain !== "no_route") {
|
||||
return routeDomain;
|
||||
}
|
||||
return input.previous.followup_context?.active_domain ?? input.previous.focus.domain ?? null;
|
||||
}
|
||||
function collectUncoveredRequirementIds(coverageReport) {
|
||||
return capStrings([
|
||||
...coverageReport.requirements_uncovered,
|
||||
...coverageReport.requirements_partially_covered,
|
||||
...coverageReport.clarification_needed_for,
|
||||
...coverageReport.out_of_scope_requirements
|
||||
], stage1Contracts_1.INVESTIGATION_MAX_REQUIREMENT_LINKS);
|
||||
}
|
||||
function collectEvidenceSummary(retrievalResults) {
|
||||
const lines = retrievalResults.map((result) => {
|
||||
const requirementRef = result.requirement_ids[0] ?? result.fragment_id;
|
||||
return `${requirementRef}:${result.status}:${result.route}`;
|
||||
});
|
||||
return capStrings(lines, 6);
|
||||
}
|
||||
function settlementFocusActions(activeDomain) {
|
||||
if (activeDomain !== "settlements_60_62") {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
"Проверьте договор и объект расчетов по платежу.",
|
||||
"Сверьте регистр расчетов и привязку платежа к закрывающему документу.",
|
||||
"Проверьте зачет аванса или взаимозачет по связке 60/62."
|
||||
];
|
||||
}
|
||||
function normalizeEntityBacklinks(values) {
|
||||
const result = [];
|
||||
const seen = new Set();
|
||||
@@ -180,7 +252,27 @@ function cloneInvestigationState(state) {
|
||||
followup_context: state.followup_context
|
||||
? {
|
||||
...state.followup_context,
|
||||
referenced_requirement_ids: [...state.followup_context.referenced_requirement_ids]
|
||||
referenced_requirement_ids: [...state.followup_context.referenced_requirement_ids],
|
||||
...(state.followup_context.active_requirement_ids
|
||||
? {
|
||||
active_requirement_ids: [...state.followup_context.active_requirement_ids]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.uncovered_requirement_ids
|
||||
? {
|
||||
uncovered_requirement_ids: [...state.followup_context.uncovered_requirement_ids]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.settlement_next_actions
|
||||
? {
|
||||
settlement_next_actions: [...state.followup_context.settlement_next_actions]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.evidence_summary
|
||||
? {
|
||||
evidence_summary: [...state.followup_context.evidence_summary]
|
||||
}
|
||||
: {})
|
||||
}
|
||||
: null
|
||||
};
|
||||
@@ -222,9 +314,21 @@ function createEmptyInvestigationState(sessionId, timestamp = new Date().toISOSt
|
||||
function updateInvestigationState(input) {
|
||||
const previous = input.previous;
|
||||
const focusFromMessage = capStrings(detectAccounts(input.userMessage), stage1Contracts_1.INVESTIGATION_MAX_PRIMARY_ACCOUNTS);
|
||||
const mergedFocusAccounts = capStrings([...focusFromMessage, ...previous.focus.primary_accounts], stage1Contracts_1.INVESTIGATION_MAX_PRIMARY_ACCOUNTS);
|
||||
const requirementIds = capStrings(input.requirements.map((item) => item.requirement_id), stage1Contracts_1.INVESTIGATION_MAX_REQUIREMENT_LINKS);
|
||||
const mainRequirement = input.requirements[0]?.requirement_text ?? input.userMessage;
|
||||
const problemUnitState = updateProblemUnitState(previous, input.retrievalResults);
|
||||
const uncoveredRequirementIds = collectUncoveredRequirementIds(input.coverageReport);
|
||||
const activeDomain = inferFollowupActiveDomain({
|
||||
userMessage: input.userMessage,
|
||||
focusAccounts: mergedFocusAccounts,
|
||||
routeSummary: input.routeSummary,
|
||||
previous
|
||||
});
|
||||
const focusDomain = activeDomain ?? deriveDomain(input.routeSummary) ?? previous.focus.domain;
|
||||
const settlementNextActions = settlementFocusActions(activeDomain);
|
||||
const lastProblemUnitId = problemUnitState?.active_problem_units[0] ?? null;
|
||||
const evidenceSummary = collectEvidenceSummary(input.retrievalResults);
|
||||
return {
|
||||
schema_version: stage1Contracts_1.INVESTIGATION_STATE_SCHEMA_VERSION,
|
||||
session_id: previous.session_id,
|
||||
@@ -233,9 +337,9 @@ function updateInvestigationState(input) {
|
||||
updated_at: input.timestamp,
|
||||
question_id: input.questionId,
|
||||
focus: {
|
||||
domain: deriveDomain(input.routeSummary) ?? previous.focus.domain,
|
||||
domain: focusDomain,
|
||||
period: detectPeriod(input.userMessage) ?? previous.focus.period,
|
||||
primary_accounts: capStrings([...focusFromMessage, ...previous.focus.primary_accounts], stage1Contracts_1.INVESTIGATION_MAX_PRIMARY_ACCOUNTS),
|
||||
primary_accounts: mergedFocusAccounts,
|
||||
active_query_subject: mainRequirement.slice(0, 180)
|
||||
},
|
||||
narrowing_status: deriveNarrowingStatus(input.routeSummary, input.coverageReport),
|
||||
@@ -245,7 +349,13 @@ function updateInvestigationState(input) {
|
||||
followup_context: {
|
||||
previous_question_id: previous.question_id,
|
||||
last_user_message: input.userMessage.slice(0, 240),
|
||||
referenced_requirement_ids: requirementIds
|
||||
referenced_requirement_ids: requirementIds,
|
||||
active_domain: activeDomain,
|
||||
active_requirement_ids: requirementIds,
|
||||
uncovered_requirement_ids: uncoveredRequirementIds,
|
||||
last_problem_unit_id: lastProblemUnitId,
|
||||
settlement_next_actions: settlementNextActions,
|
||||
evidence_summary: evidenceSummary
|
||||
},
|
||||
query_mode_hint: deriveQueryModeHint(input.routeSummary),
|
||||
...(problemUnitState
|
||||
|
||||
@@ -4,6 +4,7 @@ exports.normalizeRetrievalResult = normalizeRetrievalResult;
|
||||
const config_1 = require("../config");
|
||||
const stage1Contracts_1 = require("../types/stage1Contracts");
|
||||
const problemUnitAssembler_1 = require("./problemUnitAssembler");
|
||||
const stage4GraphRuntime_1 = require("./stage4GraphRuntime");
|
||||
function toObject(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
@@ -75,6 +76,21 @@ function mergeSummaryWithProblemUnitMeta(summary, input) {
|
||||
problem_unit_lifecycle_defect_distribution: input.lifecycleDefectDistribution
|
||||
};
|
||||
}
|
||||
function mergeSummaryWithGraphMeta(summary, graphSummary) {
|
||||
return {
|
||||
...summary,
|
||||
graph_runtime_enabled: true,
|
||||
graph_total_units: graphSummary.total_units,
|
||||
graph_bound_units: graphSummary.bound_units,
|
||||
graph_nodes_count: graphSummary.node_count,
|
||||
graph_edges_count: graphSummary.edge_count,
|
||||
graph_missing_links_count: graphSummary.missing_links_count,
|
||||
graph_conflicting_links_count: graphSummary.conflicting_links_count,
|
||||
graph_coverage_grade: graphSummary.graph_coverage_grade,
|
||||
graph_domain_distribution: graphSummary.domain_distribution,
|
||||
graph_relation_distribution: graphSummary.relation_distribution
|
||||
};
|
||||
}
|
||||
function normalizeConfidence(value) {
|
||||
if (value === "high" || value === "medium" || value === "low") {
|
||||
return value;
|
||||
@@ -408,23 +424,55 @@ function normalizeRetrievalResult(fragmentId, requirementIds, route, raw) {
|
||||
selection_reason: baseResult.selection_reason,
|
||||
business_interpretation: baseResult.business_interpretation
|
||||
});
|
||||
const enrichedSummary = mergeSummaryWithProblemUnitMeta(summary, {
|
||||
candidateEvidenceCount: assembled.candidate_evidence.length,
|
||||
problemUnitsCount: assembled.problem_units.length,
|
||||
unitTypes: assembled.problem_unit_summary.unit_types,
|
||||
duplicateCollapses: assembled.problem_unit_summary.duplicate_collapses,
|
||||
severityDistribution: assembled.problem_unit_summary.severity_distribution,
|
||||
confidenceDistribution: assembled.problem_unit_summary.confidence_distribution,
|
||||
lifecycleEnrichedUnits: assembled.problem_unit_summary.lifecycle_enriched_units ?? 0,
|
||||
lifecycleDomainDistribution: (assembled.problem_unit_summary.lifecycle_domain_distribution ?? {}),
|
||||
lifecycleDefectDistribution: (assembled.problem_unit_summary.lifecycle_defect_distribution ?? {})
|
||||
const graphBuild = config_1.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1
|
||||
? (0, stage4GraphRuntime_1.buildAccountingGraph)({
|
||||
route,
|
||||
candidateEvidence: assembled.candidate_evidence,
|
||||
problemUnits: assembled.problem_units
|
||||
})
|
||||
: null;
|
||||
const graphBindingByUnitId = new Map(graphBuild?.unit_bindings.map((item) => [item.problem_unit_id, item]) ?? []);
|
||||
const graphBoundProblemUnits = assembled.problem_units.map((unit) => {
|
||||
const binding = graphBindingByUnitId.get(unit.problem_unit_id);
|
||||
if (!binding) {
|
||||
return unit;
|
||||
}
|
||||
return {
|
||||
...unit,
|
||||
graph_binding: binding
|
||||
};
|
||||
});
|
||||
const graphBoundSummary = graphBuild
|
||||
? {
|
||||
...assembled.problem_unit_summary,
|
||||
graph_summary: graphBuild.summary
|
||||
}
|
||||
: assembled.problem_unit_summary;
|
||||
let enrichedSummary = mergeSummaryWithProblemUnitMeta(summary, {
|
||||
candidateEvidenceCount: assembled.candidate_evidence.length,
|
||||
problemUnitsCount: graphBoundProblemUnits.length,
|
||||
unitTypes: graphBoundSummary.unit_types,
|
||||
duplicateCollapses: graphBoundSummary.duplicate_collapses,
|
||||
severityDistribution: graphBoundSummary.severity_distribution,
|
||||
confidenceDistribution: graphBoundSummary.confidence_distribution,
|
||||
lifecycleEnrichedUnits: graphBoundSummary.lifecycle_enriched_units ?? 0,
|
||||
lifecycleDomainDistribution: (graphBoundSummary.lifecycle_domain_distribution ?? {}),
|
||||
lifecycleDefectDistribution: (graphBoundSummary.lifecycle_defect_distribution ?? {})
|
||||
});
|
||||
if (graphBuild) {
|
||||
enrichedSummary = mergeSummaryWithGraphMeta(enrichedSummary, graphBuild.summary);
|
||||
}
|
||||
return {
|
||||
...baseResult,
|
||||
summary: enrichedSummary,
|
||||
raw_entities: items,
|
||||
candidate_evidence: assembled.candidate_evidence,
|
||||
problem_units: assembled.problem_units,
|
||||
problem_unit_summary: assembled.problem_unit_summary
|
||||
problem_units: graphBoundProblemUnits,
|
||||
problem_unit_summary: graphBoundSummary,
|
||||
...(graphBuild
|
||||
? {
|
||||
accounting_graph: graphBuild
|
||||
}
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
+206
-91
@@ -1,5 +1,6 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ROUTE_DISCIPLINE_RULE_TABLE = void 0;
|
||||
exports.simulateDeterministicRouting = simulateDeterministicRouting;
|
||||
exports.toRouteHintSummary = toRouteHintSummary;
|
||||
exports.toRouterInput = toRouterInput;
|
||||
@@ -28,6 +29,188 @@ function toRouteHintSummaryV1(normalized) {
|
||||
}
|
||||
};
|
||||
}
|
||||
const ACCOUNT_HINT_PATTERN = /(?:\b(?:account|acct|schet|счет|сч)\s*[:#]?\s*(?:[1-9][0-9](?:[./-][0-9]{1,2})?)|\b(?:19|20|21|23|25|26|28|29|44|51|60|62|68)\b)/i;
|
||||
const PERIOD_PATTERN = /\b20\d{2}(?:[-./](?:0[1-9]|1[0-2]))?\b/i;
|
||||
const SYMPTOM_MARKER_PATTERN = /(?:\bsymptom\b|\banomaly\b|\bproblem\b|\bissue\b|\btail\b|\bhanging\b|\bblocked\b|\bincomplete\b|remains?\s+open|not\s+(?:confirmed|observed|resolved|closed)|не\s+(?:подтвержден|закрыт|наблюдается)|хвост|сбой|проблем)/i;
|
||||
const LIFECYCLE_MARKER_PATTERN = /(?:\blifecycle\b|\bchain\b|\btransition\b|\bstep\b|\btrace\b|цепоч|этап|переход|связк|где\s+разрыв)/i;
|
||||
const CHAIN_BREAK_PATTERN = /(?:\bbreak\b|\bbroken\b|\bgap\b|missing\s+(?:transition|step|link)|chain\s+break|разрыв|обрыв|нет\s+переход|не\s+дошл|не\s+наблюд)/i;
|
||||
const PERIOD_IMPACT_PATTERN = /(?:period\s*close|month\s*close|month-end|residual|allocation|20[/-]44|закрыти|остатк|распредел|конец\s+месяц)/i;
|
||||
const CAUSAL_PATTERN = /(?:\bwhy\b|\bbecause\b|\breason\b|explain\s+mechanism|почему|объясни|механизм|причин)/i;
|
||||
const AMBIGUITY_PATTERN = /(?:\bmaybe\b|\bperhaps\b|not\s+sure|i\s+only\s+know|part\s+may\s+be\s+missing|возможно|может\s+быть|не\s+уверен|не\s+знаю|часть\s+цепочки\s+не\s+подтвержд)/i;
|
||||
const TRANSLIT_PROBLEM_PATTERN = /(?:raschet|oplata|zakryt|nds|vychet|zatrat|ostatok|cepoch|perehod|pochemu|prichin|period)/i;
|
||||
const DOMAIN_LEXICAL_ANCHOR_PATTERN = /(?:\b(?:settlement|payment|bank|supplier|customer|vat|nds|invoice|register|book|period\s*close|month\s*close|close\s*operation|allocation|residual|cost|expenses?)\b|оплат|расчет|РЅРґСЃ|СЃС‡[её]С‚.?фактур|РєРЅРёРі[аи]|затрат|закрыт|остатк)/i;
|
||||
exports.ROUTE_DISCIPLINE_RULE_TABLE = [
|
||||
{
|
||||
query_class: "exact_object_trace",
|
||||
required_route: "live_mcp_drilldown",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical", "hybrid_store_plus_live", "store_feature_risk", "batch_refresh_then_store"],
|
||||
description: "Exact object trace queries always run via live drilldown."
|
||||
},
|
||||
{
|
||||
query_class: "ranking_or_period_summary",
|
||||
required_route: "batch_refresh_then_store",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical", "hybrid_store_plus_live"],
|
||||
description: "Ranking and period summary queries require analytical batch path."
|
||||
},
|
||||
{
|
||||
query_class: "symptom_first",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Symptom-first intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "lifecycle_first",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Lifecycle-first intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "chain_break",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Chain-break intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "period_impact",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Period-impact problem intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "causal_query",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Causal/mechanism intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "mixed_ambiguity",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Mixed ambiguity keeps hybrid as primary route, with explicit no-route fallback."
|
||||
},
|
||||
{
|
||||
query_class: "rule_check_without_symptom",
|
||||
required_route: "store_feature_risk",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Rule checks without symptom/lifecycle signals run via risk profile path."
|
||||
},
|
||||
{
|
||||
query_class: "canonical_fact_lookup",
|
||||
required_route: "store_canonical",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["hybrid_store_plus_live"],
|
||||
description: "Only plain factual lookups are allowed to stay on canonical path."
|
||||
}
|
||||
];
|
||||
const ROUTE_DISCIPLINE_RULE_MAP = new Map(exports.ROUTE_DISCIPLINE_RULE_TABLE.map((item) => [item.query_class, item]));
|
||||
function mergedFragmentText(fragment) {
|
||||
return `${fragment.raw_fragment_text ?? ""} ${fragment.normalized_fragment_text ?? ""}`.toLowerCase();
|
||||
}
|
||||
function hasLifecycleDomainHint(fragment, lowerText) {
|
||||
const accountHints = Array.isArray(fragment.account_hints) ? fragment.account_hints.map((item) => String(item)) : [];
|
||||
if (accountHints.some((item) => /^(97|01|02|08|19|20|21|23|25|26|28|29|44|68(?:\.\d+)?|51|60|62)$/.test(item))) {
|
||||
return true;
|
||||
}
|
||||
return (fragment.candidate_labels.includes("anomaly_probe") ||
|
||||
fragment.candidate_labels.includes("period_close_risk") ||
|
||||
PERIOD_IMPACT_PATTERN.test(lowerText));
|
||||
}
|
||||
function hasSymptomSignal(fragment, lowerText) {
|
||||
return (fragment.flags.asks_for_anomaly_scan ||
|
||||
fragment.candidate_labels.includes("anomaly_probe") ||
|
||||
fragment.candidate_labels.includes("period_close_risk") ||
|
||||
SYMPTOM_MARKER_PATTERN.test(lowerText) ||
|
||||
TRANSLIT_PROBLEM_PATTERN.test(lowerText));
|
||||
}
|
||||
function hasLifecycleSignal(fragment, lowerText) {
|
||||
return (fragment.flags.asks_for_chain_explanation ||
|
||||
fragment.flags.mentions_period_close_context ||
|
||||
LIFECYCLE_MARKER_PATTERN.test(lowerText) ||
|
||||
hasLifecycleDomainHint(fragment, lowerText));
|
||||
}
|
||||
function hasChainBreakSignal(lowerText) {
|
||||
return CHAIN_BREAK_PATTERN.test(lowerText);
|
||||
}
|
||||
function hasPeriodImpactSignal(lowerText) {
|
||||
return PERIOD_IMPACT_PATTERN.test(lowerText);
|
||||
}
|
||||
function hasCausalSignal(lowerText) {
|
||||
return CAUSAL_PATTERN.test(lowerText);
|
||||
}
|
||||
function hasAmbiguitySignal(fragment, lowerText) {
|
||||
return (AMBIGUITY_PATTERN.test(lowerText) ||
|
||||
fragment.confidence === "low" ||
|
||||
fragment.domain_relevance === "unclear" ||
|
||||
fragment.business_scope === "unclear");
|
||||
}
|
||||
function hasAccountOrPeriodAnchor(fragment, lowerText) {
|
||||
return fragment.account_hints.length > 0 || ACCOUNT_HINT_PATTERN.test(lowerText) || PERIOD_PATTERN.test(lowerText);
|
||||
}
|
||||
function resolveRouteClass(fragment) {
|
||||
const lowerText = mergedFragmentText(fragment);
|
||||
const symptomSignal = hasSymptomSignal(fragment, lowerText);
|
||||
const lifecycleSignal = hasLifecycleSignal(fragment, lowerText);
|
||||
const chainBreakSignal = hasChainBreakSignal(lowerText);
|
||||
const periodImpactSignal = hasPeriodImpactSignal(lowerText);
|
||||
const causalSignal = hasCausalSignal(lowerText);
|
||||
const ambiguitySignal = hasAmbiguitySignal(fragment, lowerText);
|
||||
const accountOrPeriodAnchor = hasAccountOrPeriodAnchor(fragment, lowerText);
|
||||
if (fragment.flags.asks_for_exact_object_trace) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("exact_object_trace");
|
||||
}
|
||||
if (fragment.flags.asks_for_ranking_or_top || fragment.flags.asks_for_period_summary) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("ranking_or_period_summary");
|
||||
}
|
||||
if (ambiguitySignal && (symptomSignal || lifecycleSignal || chainBreakSignal || periodImpactSignal || causalSignal)) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("mixed_ambiguity");
|
||||
}
|
||||
if (chainBreakSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("chain_break");
|
||||
}
|
||||
if (periodImpactSignal && accountOrPeriodAnchor) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("period_impact");
|
||||
}
|
||||
if (lifecycleSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("lifecycle_first");
|
||||
}
|
||||
if (symptomSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("symptom_first");
|
||||
}
|
||||
if (causalSignal && accountOrPeriodAnchor) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("causal_query");
|
||||
}
|
||||
if (fragment.flags.asks_for_rule_check) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("rule_check_without_symptom");
|
||||
}
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("canonical_fact_lookup");
|
||||
}
|
||||
function shouldPromoteFromNoRoute(fragment, rule) {
|
||||
if (rule.required_route === "store_canonical") {
|
||||
return false;
|
||||
}
|
||||
if (explicitNoRouteReason(fragment) === "out_of_scope") {
|
||||
return false;
|
||||
}
|
||||
const lowerText = mergedFragmentText(fragment);
|
||||
const hasProblemSignal = hasSymptomSignal(fragment, lowerText) ||
|
||||
hasLifecycleSignal(fragment, lowerText) ||
|
||||
hasChainBreakSignal(lowerText) ||
|
||||
hasPeriodImpactSignal(lowerText) ||
|
||||
hasCausalSignal(lowerText);
|
||||
const hasAnchor = hasAccountOrPeriodAnchor(fragment, lowerText) ||
|
||||
fragment.candidate_labels.includes("cross_entity") ||
|
||||
DOMAIN_LEXICAL_ANCHOR_PATTERN.test(lowerText);
|
||||
return hasProblemSignal && hasAnchor;
|
||||
}
|
||||
function reasonForNoRoute(noRouteReason) {
|
||||
if (noRouteReason === "out_of_scope") {
|
||||
return "Fragment is out-of-scope for company-specific accounting contour.";
|
||||
@@ -77,32 +260,30 @@ function decideRouteForFragment(fragment) {
|
||||
const readiness = executionReadiness(fragment);
|
||||
const clarification = clarificationReason(fragment);
|
||||
const soft = softAssumptions(fragment);
|
||||
if (status === "no_route") {
|
||||
return buildNoRouteDecision(fragment, noRouteReason);
|
||||
}
|
||||
if (readiness === "needs_clarification" || readiness === "no_route") {
|
||||
return buildNoRouteDecision(fragment, noRouteReason ?? "insufficient_specificity");
|
||||
}
|
||||
if (fragment.domain_relevance !== "in_scope") {
|
||||
const routeRule = resolveRouteClass(fragment);
|
||||
if (fragment.domain_relevance === "out_of_scope") {
|
||||
return buildNoRouteDecision(fragment, "out_of_scope");
|
||||
}
|
||||
if (fragment.flags.asks_for_exact_object_trace) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "live_mcp_drilldown",
|
||||
reason: "Exact object trace requested."
|
||||
};
|
||||
if (status === "no_route" || readiness === "needs_clarification" || readiness === "no_route") {
|
||||
if (shouldPromoteFromNoRoute(fragment, routeRule)) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: routeRule.required_route,
|
||||
reason: `${routeRule.description} Query class: ${routeRule.query_class}. Promoted from no-route by anchor/symptom guardrail.`
|
||||
};
|
||||
}
|
||||
return buildNoRouteDecision(fragment, noRouteReason);
|
||||
}
|
||||
if (fragment.flags.asks_for_ranking_or_top || fragment.flags.asks_for_period_summary) {
|
||||
if (status === "routed" || status === null) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
@@ -114,74 +295,8 @@ function decideRouteForFragment(fragment) {
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "batch_refresh_then_store",
|
||||
reason: "Ranking/summary semantics require batch analytical route."
|
||||
};
|
||||
}
|
||||
if (fragment.flags.has_multi_entity_scope && fragment.flags.asks_for_chain_explanation) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "hybrid_store_plus_live",
|
||||
reason: "Multi-entity causal chain requested."
|
||||
};
|
||||
}
|
||||
if (fragment.flags.asks_for_rule_check && !fragment.flags.asks_for_chain_explanation) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_feature_risk",
|
||||
reason: "Rule-control check without causal decomposition."
|
||||
};
|
||||
}
|
||||
if (fragment.flags.asks_for_anomaly_scan &&
|
||||
!fragment.flags.asks_for_ranking_or_top &&
|
||||
!(fragment.flags.has_multi_entity_scope && fragment.flags.asks_for_chain_explanation)) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_feature_risk",
|
||||
reason: "Anomaly scan without heavy ranking or causal chain."
|
||||
};
|
||||
}
|
||||
if (status === "routed") {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_canonical",
|
||||
reason: "Routed fragment without deep analytical or causal signals."
|
||||
route: routeRule.required_route,
|
||||
reason: `${routeRule.description} Query class: ${routeRule.query_class}. Allowed fallback: ${routeRule.allowed_fallback.join(", ")}. Forbidden fallback: ${routeRule.forbidden_fallback.join(", ")}.`
|
||||
};
|
||||
}
|
||||
return buildNoRouteDecision(fragment, "missing_mapping");
|
||||
|
||||
@@ -0,0 +1,574 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.buildAccountingGraph = buildAccountingGraph;
|
||||
const stage4Graph_1 = require("../types/stage4Graph");
|
||||
const GRAPH_CONFIDENCE_ORDER = {
|
||||
low: 1,
|
||||
medium: 2,
|
||||
high: 3
|
||||
};
|
||||
const DOMAIN_PATH_HINTS = {
|
||||
bank_settlement: ["payment_to_settlement", "wrong_closing_document_type"],
|
||||
customer_settlement: ["invoice_to_payment", "payment_to_closure"],
|
||||
deferred_expense: ["deferred_expense_to_writeoff", "writeoff_sequence"],
|
||||
fixed_asset: ["asset_card_to_depreciation", "card_document_register_alignment"],
|
||||
vat_flow: ["invoice_to_vat_register", "cross_branch_alignment"],
|
||||
period_close: ["period_close_dependency_chain", "closure_blocker_transition"]
|
||||
};
|
||||
function uniqueStrings(values, limit = 16) {
|
||||
return Array.from(new Set(values.map((item) => String(item ?? "").trim()).filter(Boolean))).slice(0, limit);
|
||||
}
|
||||
function compactToken(value) {
|
||||
const normalized = value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
return normalized.length > 0 ? normalized.slice(0, 48) : "x";
|
||||
}
|
||||
function stableNodeId(type, domain, stableKey) {
|
||||
return `gnd-${compactToken(type)}-${compactToken(domain)}-${compactToken(stableKey)}`;
|
||||
}
|
||||
function stableEdgeId(relation, fromNode, toNode) {
|
||||
return `ged-${compactToken(relation)}-${compactToken(fromNode)}-${compactToken(toNode)}`;
|
||||
}
|
||||
function mergeConfidence(left, right) {
|
||||
return GRAPH_CONFIDENCE_ORDER[right] > GRAPH_CONFIDENCE_ORDER[left] ? right : left;
|
||||
}
|
||||
function mergeProvenance(left, right, routeFallback) {
|
||||
return {
|
||||
route: String(right?.route ?? left.route ?? routeFallback),
|
||||
candidate_ids: uniqueStrings([...(left.candidate_ids ?? []), ...(right?.candidate_ids ?? [])], 24),
|
||||
evidence_ids: uniqueStrings([...(left.evidence_ids ?? []), ...(right?.evidence_ids ?? [])], 24)
|
||||
};
|
||||
}
|
||||
class GraphAccumulator {
|
||||
route;
|
||||
nodesById = new Map();
|
||||
edgesById = new Map();
|
||||
constructor(route) {
|
||||
this.route = route;
|
||||
}
|
||||
upsertNode(input) {
|
||||
const node_id = stableNodeId(input.node_type, input.domain, input.stable_key);
|
||||
const existing = this.nodesById.get(node_id);
|
||||
if (!existing) {
|
||||
const created = {
|
||||
node_id,
|
||||
node_type: input.node_type,
|
||||
domain: input.domain,
|
||||
label: input.label,
|
||||
confidence: input.confidence,
|
||||
attributes: input.attributes ?? {},
|
||||
provenance: {
|
||||
route: String(input.provenance?.route ?? this.route),
|
||||
candidate_ids: uniqueStrings(input.provenance?.candidate_ids ?? [], 24),
|
||||
evidence_ids: uniqueStrings(input.provenance?.evidence_ids ?? [], 24)
|
||||
}
|
||||
};
|
||||
this.nodesById.set(node_id, created);
|
||||
return created;
|
||||
}
|
||||
existing.confidence = mergeConfidence(existing.confidence, input.confidence);
|
||||
existing.provenance = mergeProvenance(existing.provenance, input.provenance, this.route);
|
||||
if (Object.keys(input.attributes ?? {}).length > 0) {
|
||||
existing.attributes = {
|
||||
...existing.attributes,
|
||||
...input.attributes
|
||||
};
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
upsertEdge(input) {
|
||||
const edge_id = stableEdgeId(input.relation_type, input.from_node_id, input.to_node_id);
|
||||
const existing = this.edgesById.get(edge_id);
|
||||
if (!existing) {
|
||||
const created = {
|
||||
edge_id,
|
||||
relation_type: input.relation_type,
|
||||
from_node_id: input.from_node_id,
|
||||
to_node_id: input.to_node_id,
|
||||
domain: input.domain,
|
||||
confidence: input.confidence,
|
||||
flags: uniqueStrings(input.flags ?? [], 8),
|
||||
provenance: {
|
||||
route: String(input.provenance?.route ?? this.route),
|
||||
candidate_ids: uniqueStrings(input.provenance?.candidate_ids ?? [], 24),
|
||||
evidence_ids: uniqueStrings(input.provenance?.evidence_ids ?? [], 24)
|
||||
}
|
||||
};
|
||||
this.edgesById.set(edge_id, created);
|
||||
return created;
|
||||
}
|
||||
existing.confidence = mergeConfidence(existing.confidence, input.confidence);
|
||||
existing.flags = uniqueStrings([...(existing.flags ?? []), ...(input.flags ?? [])], 8);
|
||||
existing.provenance = mergeProvenance(existing.provenance, input.provenance, this.route);
|
||||
return existing;
|
||||
}
|
||||
export() {
|
||||
return {
|
||||
nodes: Array.from(this.nodesById.values()).sort((left, right) => left.node_id.localeCompare(right.node_id)),
|
||||
edges: Array.from(this.edgesById.values()).sort((left, right) => left.edge_id.localeCompare(right.edge_id))
|
||||
};
|
||||
}
|
||||
}
|
||||
function inferDomainFromUnit(unit) {
|
||||
if (unit.lifecycle_domain) {
|
||||
return unit.lifecycle_domain;
|
||||
}
|
||||
const accountText = unit.affected_accounts.join(" ").toLowerCase();
|
||||
if (/\b97\b/.test(accountText) || unit.problem_unit_type === "lifecycle_anomaly_node") {
|
||||
return "deferred_expense";
|
||||
}
|
||||
if (/\b(01|02|08)\b/.test(accountText)) {
|
||||
return "fixed_asset";
|
||||
}
|
||||
if (/\b(19|68)\b/.test(accountText) || unit.problem_unit_type === "cross_branch_inconsistency_cluster") {
|
||||
return "vat_flow";
|
||||
}
|
||||
if (unit.problem_unit_type === "period_risk_cluster" || unit.period_impact?.impact_class === "close_risk") {
|
||||
return "period_close";
|
||||
}
|
||||
if (/\b62\b/.test(accountText)) {
|
||||
return "customer_settlement";
|
||||
}
|
||||
if (/\b(51|60|76)\b/.test(accountText) || unit.problem_unit_type === "unresolved_settlement_cluster") {
|
||||
return "bank_settlement";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
function relationPathHints(domain, unit) {
|
||||
const path = [`domain:${domain}`];
|
||||
if (unit.current_lifecycle_state && unit.expected_lifecycle_state) {
|
||||
path.push(`state:${unit.current_lifecycle_state}->${unit.expected_lifecycle_state}`);
|
||||
}
|
||||
else if (unit.current_lifecycle_state) {
|
||||
path.push(`state:${unit.current_lifecycle_state}`);
|
||||
}
|
||||
if (domain !== "unknown") {
|
||||
path.push(...(DOMAIN_PATH_HINTS[domain] ?? []));
|
||||
}
|
||||
if (unit.missing_transition) {
|
||||
path.push(`missing:${unit.missing_transition}`);
|
||||
}
|
||||
if (unit.invalid_transition) {
|
||||
path.push(`conflict:${unit.invalid_transition}`);
|
||||
}
|
||||
return uniqueStrings(path, 10);
|
||||
}
|
||||
function graphConfidenceFromUnit(unit) {
|
||||
return unit.lifecycle_confidence?.grade ?? unit.confidence.grade;
|
||||
}
|
||||
function coverageGrade(boundUnits, totalUnits) {
|
||||
if (totalUnits <= 0) {
|
||||
return "low";
|
||||
}
|
||||
const ratio = boundUnits / totalUnits;
|
||||
if (ratio >= 0.8)
|
||||
return "high";
|
||||
if (ratio >= 0.4)
|
||||
return "medium";
|
||||
return "low";
|
||||
}
|
||||
function buildSummary(input) {
|
||||
const domain_distribution = {};
|
||||
const relation_distribution = {};
|
||||
for (const binding of input.bindings) {
|
||||
const domainMarker = binding.relation_path.find((item) => item.startsWith("domain:")) ?? "domain:unknown";
|
||||
const domainKey = domainMarker.replace(/^domain:/, "");
|
||||
domain_distribution[domainKey] = (domain_distribution[domainKey] ?? 0) + 1;
|
||||
}
|
||||
for (const edge of input.edges) {
|
||||
relation_distribution[edge.relation_type] = (relation_distribution[edge.relation_type] ?? 0) + 1;
|
||||
}
|
||||
const missing_links_count = input.bindings.reduce((acc, item) => acc + item.missing_links.length, 0);
|
||||
const conflicting_links_count = input.bindings.reduce((acc, item) => acc + item.conflicting_links.length, 0);
|
||||
const bound_units = input.bindings.length;
|
||||
return {
|
||||
total_units: input.totalUnits,
|
||||
bound_units,
|
||||
node_count: input.nodes.length,
|
||||
edge_count: input.edges.length,
|
||||
missing_links_count,
|
||||
conflicting_links_count,
|
||||
graph_coverage_grade: coverageGrade(bound_units, input.totalUnits),
|
||||
domain_distribution,
|
||||
relation_distribution
|
||||
};
|
||||
}
|
||||
function buildAccountingGraph(input) {
|
||||
const accumulator = new GraphAccumulator(input.route);
|
||||
const candidateById = new Map(input.candidateEvidence.map((item) => [item.candidate_id, item]));
|
||||
const bindings = [];
|
||||
const issues = [];
|
||||
if (input.problemUnits.length === 0) {
|
||||
issues.push("no_problem_units_for_graph_build");
|
||||
}
|
||||
for (const unit of input.problemUnits) {
|
||||
const domain = inferDomainFromUnit(unit);
|
||||
const confidence = graphConfidenceFromUnit(unit);
|
||||
const candidateIds = uniqueStrings(unit.evidence_pack, 12).filter((item) => candidateById.has(item));
|
||||
const evidenceIds = uniqueStrings(unit.evidence_pack, 12);
|
||||
const domainNode = accumulator.upsertNode({
|
||||
node_type: "domain",
|
||||
domain,
|
||||
stable_key: `domain:${domain}`,
|
||||
label: domain,
|
||||
confidence,
|
||||
attributes: {
|
||||
domain
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
const problemNode = accumulator.upsertNode({
|
||||
node_type: "problem_unit",
|
||||
domain,
|
||||
stable_key: `problem:${unit.problem_unit_id}`,
|
||||
label: unit.title || unit.problem_unit_id,
|
||||
confidence,
|
||||
attributes: {
|
||||
problem_unit_id: unit.problem_unit_id,
|
||||
problem_unit_type: unit.problem_unit_type,
|
||||
business_defect_class: unit.business_defect_class,
|
||||
lifecycle_defect_type: unit.lifecycle_defect_type ?? null
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "belongs_to_domain",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: domainNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
for (const account of uniqueStrings(unit.affected_accounts, 4)) {
|
||||
const accountNode = accumulator.upsertNode({
|
||||
node_type: "account",
|
||||
domain,
|
||||
stable_key: `account:${account}`,
|
||||
label: account,
|
||||
confidence,
|
||||
attributes: {
|
||||
account
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_account",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: accountNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
for (const document of uniqueStrings(unit.affected_documents, 3)) {
|
||||
const documentNode = accumulator.upsertNode({
|
||||
node_type: "document",
|
||||
domain,
|
||||
stable_key: `document:${document}`,
|
||||
label: document,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_document",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: documentNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
for (const counterparty of uniqueStrings(unit.affected_counterparties, 3)) {
|
||||
const counterpartyNode = accumulator.upsertNode({
|
||||
node_type: "counterparty",
|
||||
domain,
|
||||
stable_key: `counterparty:${counterparty}`,
|
||||
label: counterparty,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_counterparty",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: counterpartyNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
if (unit.current_lifecycle_state) {
|
||||
const currentNode = accumulator.upsertNode({
|
||||
node_type: "lifecycle_state",
|
||||
domain,
|
||||
stable_key: `current_state:${unit.current_lifecycle_state}`,
|
||||
label: unit.current_lifecycle_state,
|
||||
confidence,
|
||||
attributes: {
|
||||
state_role: "current"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "current_state",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: currentNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
if (unit.expected_lifecycle_state) {
|
||||
const expectedNode = accumulator.upsertNode({
|
||||
node_type: "lifecycle_state",
|
||||
domain,
|
||||
stable_key: `expected_state:${unit.expected_lifecycle_state}`,
|
||||
label: unit.expected_lifecycle_state,
|
||||
confidence,
|
||||
attributes: {
|
||||
state_role: "expected"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "expected_state",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: expectedNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["expected_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
if (unit.missing_transition) {
|
||||
const missingNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `missing_transition:${unit.missing_transition}`,
|
||||
label: unit.missing_transition,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "missing"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "missing_transition",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: missingNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["missing_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
if (unit.invalid_transition) {
|
||||
const invalidNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `invalid_transition:${unit.invalid_transition}`,
|
||||
label: unit.invalid_transition,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "invalid"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "invalid_transition",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: invalidNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["conflict_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
if (unit.lifecycle_defect_type) {
|
||||
const defectNode = accumulator.upsertNode({
|
||||
node_type: "defect",
|
||||
domain,
|
||||
stable_key: `defect:${unit.lifecycle_defect_type}`,
|
||||
label: unit.lifecycle_defect_type,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "has_defect",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: defectNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: unit.lifecycle_defect_type === "cross_branch_state_conflict" ? ["conflict_link"] : ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
for (const candidateId of candidateIds.slice(0, 6)) {
|
||||
const candidate = candidateById.get(candidateId);
|
||||
const evidenceLabel = candidate
|
||||
? `${candidate.source_ref.entity}:${candidate.source_ref.id}`
|
||||
: `candidate:${candidateId}`;
|
||||
const evidenceNode = accumulator.upsertNode({
|
||||
node_type: "evidence",
|
||||
domain,
|
||||
stable_key: `evidence:${candidateId}`,
|
||||
label: evidenceLabel,
|
||||
confidence,
|
||||
attributes: {
|
||||
candidate_id: candidateId
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "supported_by_evidence",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: evidenceNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
if (candidate && candidate.relation_pattern_hits.length > 0) {
|
||||
for (const relationHint of uniqueStrings(candidate.relation_pattern_hits, 2)) {
|
||||
const hintNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `hint:${relationHint}`,
|
||||
label: relationHint,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "hint"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "supports_path",
|
||||
from_node_id: evidenceNode.node_id,
|
||||
to_node_id: hintNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const missing_links = uniqueStrings([
|
||||
unit.missing_transition,
|
||||
...(unit.lifecycle_resolution?.missing_transitions ?? [])
|
||||
]);
|
||||
const conflicting_links = uniqueStrings([
|
||||
unit.invalid_transition,
|
||||
...(unit.lifecycle_resolution?.invalid_transitions ?? [])
|
||||
]);
|
||||
bindings.push({
|
||||
problem_unit_id: unit.problem_unit_id,
|
||||
graph_node_id: problemNode.node_id,
|
||||
relation_path: relationPathHints(domain, unit),
|
||||
missing_links,
|
||||
conflicting_links,
|
||||
provenance_evidence_ids: evidenceIds,
|
||||
graph_confidence: confidence
|
||||
});
|
||||
}
|
||||
const exported = accumulator.export();
|
||||
const summary = buildSummary({
|
||||
nodes: exported.nodes,
|
||||
edges: exported.edges,
|
||||
bindings,
|
||||
totalUnits: input.problemUnits.length
|
||||
});
|
||||
if (summary.bound_units < summary.total_units) {
|
||||
issues.push("some_problem_units_not_bound_to_graph");
|
||||
}
|
||||
if (summary.node_count === 0 || summary.edge_count === 0) {
|
||||
issues.push("graph_runtime_empty");
|
||||
}
|
||||
return {
|
||||
schema_version: stage4Graph_1.ACCOUNTING_GRAPH_SCHEMA_VERSION,
|
||||
nodes: exported.nodes,
|
||||
edges: exported.edges,
|
||||
unit_bindings: bindings,
|
||||
summary,
|
||||
issues: uniqueStrings(issues, 8)
|
||||
};
|
||||
}
|
||||
+34
-7
@@ -19,10 +19,22 @@ function redactSecrets(payload) {
|
||||
delete output.apiKey;
|
||||
return output;
|
||||
}
|
||||
function isNoSpaceError(error) {
|
||||
const code = error?.code;
|
||||
return code === "ENOSPC";
|
||||
}
|
||||
function saveTrace(record) {
|
||||
(0, files_1.ensureDir)(config_1.TRACES_DIR);
|
||||
const target = path_1.default.resolve(config_1.TRACES_DIR, `${record.trace_id}.json`);
|
||||
(0, files_1.writeJsonFile)(target, record);
|
||||
try {
|
||||
(0, files_1.ensureDir)(config_1.TRACES_DIR);
|
||||
const target = path_1.default.resolve(config_1.TRACES_DIR, `${record.trace_id}.json`);
|
||||
(0, files_1.writeJsonFile)(target, record);
|
||||
}
|
||||
catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function listTraces(limit = 100) {
|
||||
(0, files_1.ensureDir)(config_1.TRACES_DIR);
|
||||
@@ -60,8 +72,16 @@ function getTrace(traceId) {
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
function savePreset(preset) {
|
||||
(0, files_1.ensureDir)(config_1.PRESETS_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.PRESETS_DIR, `${preset.id}.json`), preset);
|
||||
try {
|
||||
(0, files_1.ensureDir)(config_1.PRESETS_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.PRESETS_DIR, `${preset.id}.json`), preset);
|
||||
}
|
||||
catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
function listPresets() {
|
||||
(0, files_1.ensureDir)(config_1.PRESETS_DIR);
|
||||
@@ -75,9 +95,16 @@ function listPresets() {
|
||||
.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
||||
}
|
||||
function saveEvalCase(casePayload) {
|
||||
(0, files_1.ensureDir)(config_1.EVAL_CASES_DIR);
|
||||
const id = String(casePayload.case_id ?? `NQ-${Date.now()}`);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${id}.json`), casePayload);
|
||||
try {
|
||||
(0, files_1.ensureDir)(config_1.EVAL_CASES_DIR);
|
||||
(0, files_1.writeJsonFile)(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${id}.json`), casePayload);
|
||||
}
|
||||
catch (error) {
|
||||
if (!isNoSpaceError(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return id;
|
||||
}
|
||||
function redactRequestPayload(payload) {
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ACCOUNTING_GRAPH_RELATION_TYPES = exports.ACCOUNTING_GRAPH_NODE_TYPES = exports.ACCOUNTING_GRAPH_SCHEMA_VERSION = void 0;
|
||||
exports.ACCOUNTING_GRAPH_SCHEMA_VERSION = "accounting_graph_v0_1";
|
||||
exports.ACCOUNTING_GRAPH_NODE_TYPES = [
|
||||
"domain",
|
||||
"problem_unit",
|
||||
"account",
|
||||
"document",
|
||||
"counterparty",
|
||||
"lifecycle_state",
|
||||
"transition",
|
||||
"defect",
|
||||
"evidence"
|
||||
];
|
||||
exports.ACCOUNTING_GRAPH_RELATION_TYPES = [
|
||||
"belongs_to_domain",
|
||||
"affects_account",
|
||||
"affects_document",
|
||||
"affects_counterparty",
|
||||
"current_state",
|
||||
"expected_state",
|
||||
"missing_transition",
|
||||
"invalid_transition",
|
||||
"has_defect",
|
||||
"supported_by_evidence",
|
||||
"supports_path"
|
||||
];
|
||||
+10
-1
@@ -27,5 +27,14 @@ function logJson(entry) {
|
||||
details: redactObject(entry.details)
|
||||
};
|
||||
// Structured JSON logs for diagnostics/trace aggregation.
|
||||
process.stdout.write(JSON.stringify(safe) + "\n");
|
||||
try {
|
||||
process.stdout.write(JSON.stringify(safe) + "\n");
|
||||
}
|
||||
catch (error) {
|
||||
const code = error?.code;
|
||||
if (code === "ENOSPC" || code === "EPIPE") {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,14 @@ function toBooleanFlag(value: string | undefined, defaultValue: boolean): boolea
|
||||
return !(lowered === "0" || lowered === "false" || lowered === "off" || lowered === "no");
|
||||
}
|
||||
|
||||
function toNumberFlag(value: string | undefined, defaultValue: number): number {
|
||||
if (!value || value.trim() === "") {
|
||||
return defaultValue;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : defaultValue;
|
||||
}
|
||||
|
||||
export const PORT = Number(process.env.PORT ?? 8787);
|
||||
export const TIMEZONE = process.env.TZ_FALLBACK ?? "Europe/Moscow";
|
||||
export const DEFAULT_OPENAI_BASE_URL = process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1";
|
||||
@@ -45,7 +53,7 @@ export const FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = toBooleanFlag(
|
||||
);
|
||||
export const FEATURE_ASSISTANT_ANSWER_POLICY_V11 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1,
|
||||
@@ -53,15 +61,15 @@ export const FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = toBooleanFlag(
|
||||
);
|
||||
export const FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_STAGE2_EVAL_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1,
|
||||
@@ -69,12 +77,27 @@ export const FEATURE_ASSISTANT_STAGE2_EVAL_V1 = toBooleanFlag(
|
||||
);
|
||||
export const FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1,
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1,
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_MCP_RUNTIME_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_MCP_RUNTIME_V1,
|
||||
false
|
||||
);
|
||||
export const ASSISTANT_MCP_PROXY_URL = (process.env.ASSISTANT_MCP_PROXY_URL ?? "http://127.0.0.1:6003").replace(
|
||||
/\/+$/,
|
||||
""
|
||||
);
|
||||
export const ASSISTANT_MCP_CHANNEL = process.env.ASSISTANT_MCP_CHANNEL ?? "default";
|
||||
export const ASSISTANT_MCP_TIMEOUT_MS = toNumberFlag(process.env.ASSISTANT_MCP_TIMEOUT_MS, 1200);
|
||||
export const ASSISTANT_MCP_LIVE_LIMIT = Math.max(1, Math.trunc(toNumberFlag(process.env.ASSISTANT_MCP_LIVE_LIMIT, 24)));
|
||||
|
||||
export const DATA_DIR = process.env.DATA_DIR ?? path.resolve(MODULE_ROOT, "data");
|
||||
export const TRACES_DIR = path.resolve(DATA_DIR, "traces");
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
import {
|
||||
P0_DEFAULT_ACCEPTANCE_THRESHOLDS,
|
||||
P0_DEFAULT_QUALITY_GAP_THRESHOLDS,
|
||||
type P0AcceptanceThresholds,
|
||||
type P0EvalMetricVector,
|
||||
type P0QualityGapMetricVector,
|
||||
type P0QualityGapThresholds
|
||||
} from "./p0_metric_definitions";
|
||||
|
||||
export type P0AcceptanceVerdict = "P0_ACCEPTED" | "P0_ACCEPTED_WITH_LIMITATIONS" | "P0_NOT_ACCEPTED";
|
||||
export type P0BaselineStabilityVerdict = "P0_BASELINE_STABLE" | "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS";
|
||||
|
||||
interface MetricCheck {
|
||||
metric: keyof P0EvalMetricVector;
|
||||
value: number;
|
||||
threshold: number;
|
||||
comparator: ">=" | "<=";
|
||||
passed: boolean;
|
||||
severity: "blocking" | "quality";
|
||||
}
|
||||
|
||||
interface QualityGapMetricCheck {
|
||||
metric: keyof P0QualityGapMetricVector;
|
||||
value: number;
|
||||
threshold: number;
|
||||
comparator: ">=" | "<=";
|
||||
passed: boolean;
|
||||
}
|
||||
|
||||
export interface P0AcceptanceResult {
|
||||
verdict: P0AcceptanceVerdict;
|
||||
passed_all: boolean;
|
||||
blocking_failures: MetricCheck[];
|
||||
quality_failures: MetricCheck[];
|
||||
metric_checks: MetricCheck[];
|
||||
rationale: string[];
|
||||
}
|
||||
|
||||
export interface P0BaselineStabilityResult {
|
||||
verdict: P0BaselineStabilityVerdict;
|
||||
baseline_integrity_passed: boolean;
|
||||
quality_gaps_open: boolean;
|
||||
blocking_regressions: MetricCheck[];
|
||||
legacy_quality_failures: MetricCheck[];
|
||||
quality_gap_failures: QualityGapMetricCheck[];
|
||||
quality_gap_checks: QualityGapMetricCheck[];
|
||||
rationale: string[];
|
||||
}
|
||||
|
||||
function buildMetricChecks(input: {
|
||||
metrics: P0EvalMetricVector;
|
||||
thresholds: P0AcceptanceThresholds;
|
||||
}): MetricCheck[] {
|
||||
const { metrics, thresholds } = input;
|
||||
return [
|
||||
{
|
||||
metric: "problem_first_answer_rate",
|
||||
value: metrics.problem_first_answer_rate,
|
||||
threshold: thresholds.problem_first_answer_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.problem_first_answer_rate >= thresholds.problem_first_answer_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "entity_leakage_rate",
|
||||
value: metrics.entity_leakage_rate,
|
||||
threshold: thresholds.entity_leakage_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.entity_leakage_rate <= thresholds.entity_leakage_rate_max,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "route_correctness_rate",
|
||||
value: metrics.route_correctness_rate,
|
||||
threshold: thresholds.route_correctness_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.route_correctness_rate >= thresholds.route_correctness_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "domain_purity_rate",
|
||||
value: metrics.domain_purity_rate,
|
||||
threshold: thresholds.domain_purity_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.domain_purity_rate >= thresholds.domain_purity_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "mechanism_coherence_score",
|
||||
value: metrics.mechanism_coherence_score,
|
||||
threshold: thresholds.mechanism_coherence_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.mechanism_coherence_score >= thresholds.mechanism_coherence_score_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "accountant_actionability_score",
|
||||
value: metrics.accountant_actionability_score,
|
||||
threshold: thresholds.accountant_actionability_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.accountant_actionability_score >= thresholds.accountant_actionability_score_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "limitation_honesty_rate",
|
||||
value: metrics.limitation_honesty_rate,
|
||||
threshold: thresholds.limitation_honesty_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.limitation_honesty_rate >= thresholds.limitation_honesty_rate_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "top_problem_unit_match_rate",
|
||||
value: metrics.top_problem_unit_match_rate,
|
||||
threshold: thresholds.top_problem_unit_match_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.top_problem_unit_match_rate >= thresholds.top_problem_unit_match_rate_min,
|
||||
severity: "quality"
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
function toFailureLine(check: MetricCheck): string {
|
||||
return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
|
||||
}
|
||||
|
||||
function toQualityGapFailureLine(check: QualityGapMetricCheck): string {
|
||||
return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
|
||||
}
|
||||
|
||||
function buildQualityGapChecks(input: {
|
||||
metrics: P0QualityGapMetricVector;
|
||||
thresholds: P0QualityGapThresholds;
|
||||
}): QualityGapMetricCheck[] {
|
||||
const { metrics, thresholds } = input;
|
||||
return [
|
||||
{
|
||||
metric: "generic_explanation_rate",
|
||||
value: metrics.generic_explanation_rate,
|
||||
threshold: thresholds.generic_explanation_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.generic_explanation_rate <= thresholds.generic_explanation_rate_max
|
||||
},
|
||||
{
|
||||
metric: "false_confidence_rate",
|
||||
value: metrics.false_confidence_rate,
|
||||
threshold: thresholds.false_confidence_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.false_confidence_rate <= thresholds.false_confidence_rate_max
|
||||
},
|
||||
{
|
||||
metric: "mechanism_specificity_score",
|
||||
value: metrics.mechanism_specificity_score,
|
||||
threshold: thresholds.mechanism_specificity_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.mechanism_specificity_score >= thresholds.mechanism_specificity_score_min
|
||||
},
|
||||
{
|
||||
metric: "followup_context_retention_score",
|
||||
value: metrics.followup_context_retention_score,
|
||||
threshold: thresholds.followup_context_retention_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.followup_context_retention_score >= thresholds.followup_context_retention_score_min
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
export function evaluateP0AcceptanceGate(input: {
|
||||
metrics: P0EvalMetricVector;
|
||||
thresholds?: Partial<P0AcceptanceThresholds>;
|
||||
}): P0AcceptanceResult {
|
||||
const thresholds: P0AcceptanceThresholds = {
|
||||
...P0_DEFAULT_ACCEPTANCE_THRESHOLDS,
|
||||
...(input.thresholds ?? {})
|
||||
};
|
||||
const checks = buildMetricChecks({
|
||||
metrics: input.metrics,
|
||||
thresholds
|
||||
});
|
||||
const blockingFailures = checks.filter((item) => item.severity === "blocking" && !item.passed);
|
||||
const qualityFailures = checks.filter((item) => item.severity === "quality" && !item.passed);
|
||||
|
||||
let verdict: P0AcceptanceVerdict = "P0_NOT_ACCEPTED";
|
||||
if (blockingFailures.length === 0 && qualityFailures.length === 0) {
|
||||
verdict = "P0_ACCEPTED";
|
||||
} else if (blockingFailures.length === 0) {
|
||||
verdict = "P0_ACCEPTED_WITH_LIMITATIONS";
|
||||
}
|
||||
|
||||
const rationale: string[] = [];
|
||||
if (blockingFailures.length > 0) {
|
||||
rationale.push("Blocking metrics failed:");
|
||||
rationale.push(...blockingFailures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (qualityFailures.length > 0) {
|
||||
rationale.push("Quality metrics failed:");
|
||||
rationale.push(...qualityFailures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (rationale.length === 0) {
|
||||
rationale.push("All acceptance thresholds are satisfied.");
|
||||
}
|
||||
|
||||
return {
|
||||
verdict,
|
||||
passed_all: checks.every((item) => item.passed),
|
||||
blocking_failures: blockingFailures,
|
||||
quality_failures: qualityFailures,
|
||||
metric_checks: checks,
|
||||
rationale
|
||||
};
|
||||
}
|
||||
|
||||
export function evaluateP0BaselineStabilityGate(input: {
|
||||
metrics: P0EvalMetricVector;
|
||||
qualityGapMetrics: P0QualityGapMetricVector;
|
||||
acceptanceThresholds?: Partial<P0AcceptanceThresholds>;
|
||||
qualityGapThresholds?: Partial<P0QualityGapThresholds>;
|
||||
}): P0BaselineStabilityResult {
|
||||
const acceptance = evaluateP0AcceptanceGate({
|
||||
metrics: input.metrics,
|
||||
thresholds: input.acceptanceThresholds
|
||||
});
|
||||
const qualityGapThresholds: P0QualityGapThresholds = {
|
||||
...P0_DEFAULT_QUALITY_GAP_THRESHOLDS,
|
||||
...(input.qualityGapThresholds ?? {})
|
||||
};
|
||||
const qualityGapChecks = buildQualityGapChecks({
|
||||
metrics: input.qualityGapMetrics,
|
||||
thresholds: qualityGapThresholds
|
||||
});
|
||||
const qualityGapFailures = qualityGapChecks.filter((item) => !item.passed);
|
||||
|
||||
const baselineIntegrityPassed = acceptance.blocking_failures.length === 0;
|
||||
const qualityGapsOpen = acceptance.quality_failures.length > 0 || qualityGapFailures.length > 0;
|
||||
const verdict: P0BaselineStabilityVerdict =
|
||||
baselineIntegrityPassed && !qualityGapsOpen ? "P0_BASELINE_STABLE" : "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS";
|
||||
|
||||
const rationale: string[] = [];
|
||||
if (!baselineIntegrityPassed) {
|
||||
rationale.push("Blocking baseline regressions detected:");
|
||||
rationale.push(...acceptance.blocking_failures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (acceptance.quality_failures.length > 0) {
|
||||
rationale.push("Legacy quality thresholds still below full acceptance:");
|
||||
rationale.push(...acceptance.quality_failures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (qualityGapFailures.length > 0) {
|
||||
rationale.push("Wave 9 quality-gap thresholds still open:");
|
||||
rationale.push(...qualityGapFailures.map((item) => `- ${toQualityGapFailureLine(item)}`));
|
||||
}
|
||||
if (rationale.length === 0) {
|
||||
rationale.push("Baseline is stable and Wave 9 quality-gap thresholds are satisfied.");
|
||||
}
|
||||
|
||||
return {
|
||||
verdict,
|
||||
baseline_integrity_passed: baselineIntegrityPassed,
|
||||
quality_gaps_open: qualityGapsOpen,
|
||||
blocking_regressions: acceptance.blocking_failures,
|
||||
legacy_quality_failures: acceptance.quality_failures,
|
||||
quality_gap_failures: qualityGapFailures,
|
||||
quality_gap_checks: qualityGapChecks,
|
||||
rationale
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,425 @@
|
||||
import type { ProblemUnitType } from "../types/stage2ProblemUnits";
|
||||
|
||||
export type P0DomainId = "settlements_60_62" | "vat_document_register_book" | "month_close_costs_20_44";
|
||||
|
||||
export type P0QueryClass =
|
||||
| "symptom_first"
|
||||
| "lifecycle_first"
|
||||
| "causal_query"
|
||||
| "mixed_ambiguity"
|
||||
| "followup_investigation"
|
||||
| "noisy_input"
|
||||
| "translit_noisy"
|
||||
| "multi_intent";
|
||||
|
||||
export type P0RouteExpected = "hybrid_store_plus_live" | "store_feature_risk" | "store_canonical" | "batch_refresh_then_store" | "live_mcp_drilldown" | "no_route";
|
||||
|
||||
export interface P0ExpectedAnnotations {
|
||||
domain_expected: P0DomainId;
|
||||
route_expected: P0RouteExpected | P0RouteExpected[];
|
||||
problem_unit_type_expected: ProblemUnitType;
|
||||
mechanism_class_expected: string;
|
||||
evidence_expectation: string;
|
||||
must_show_limitation: boolean;
|
||||
must_not_leak_internal: boolean;
|
||||
first_check_action_expected: string;
|
||||
}
|
||||
|
||||
export interface P0EvalCase {
|
||||
case_id: string;
|
||||
domain: P0DomainId;
|
||||
user_query_raw: string;
|
||||
query_class: P0QueryClass;
|
||||
expected_problem_unit_type: ProblemUnitType;
|
||||
expected_mechanism_summary: string;
|
||||
expected_first_check: string;
|
||||
must_include_limitations: boolean;
|
||||
forbidden_leakage_tokens: string[];
|
||||
followup_seed_query?: string;
|
||||
followup_expected_context_tokens?: string[];
|
||||
notes: string;
|
||||
expected_annotations: P0ExpectedAnnotations;
|
||||
}
|
||||
|
||||
export interface P0EvalCorpusFile {
|
||||
suite_id: string;
|
||||
suite_version: string;
|
||||
schema_version: string;
|
||||
scenario_count: number;
|
||||
case_ids: string[];
|
||||
cases: P0EvalCase[];
|
||||
}
|
||||
|
||||
export interface P0EvalMetricVector {
|
||||
problem_first_answer_rate: number;
|
||||
mechanism_coherence_score: number;
|
||||
entity_leakage_rate: number;
|
||||
accountant_actionability_score: number;
|
||||
route_correctness_rate: number;
|
||||
domain_purity_rate: number;
|
||||
limitation_honesty_rate: number;
|
||||
top_problem_unit_match_rate: number;
|
||||
}
|
||||
|
||||
export type P0MetricName = keyof P0EvalMetricVector;
|
||||
|
||||
export interface P0QualityGapMetricVector {
|
||||
generic_explanation_rate: number;
|
||||
false_confidence_rate: number;
|
||||
mechanism_specificity_score: number;
|
||||
followup_context_retention_score: number;
|
||||
}
|
||||
|
||||
export type P0QualityGapMetricName = keyof P0QualityGapMetricVector;
|
||||
|
||||
export interface P0MetricDefinition {
|
||||
description: string;
|
||||
unit: "rate" | "score";
|
||||
direction: "higher_is_better" | "lower_is_better";
|
||||
}
|
||||
|
||||
export interface P0QualityGapMetricDefinition {
|
||||
description: string;
|
||||
unit: "rate" | "score";
|
||||
direction: "higher_is_better" | "lower_is_better";
|
||||
}
|
||||
|
||||
export const P0_METRIC_DEFINITIONS: Record<P0MetricName, P0MetricDefinition> = {
|
||||
problem_first_answer_rate: {
|
||||
description: "Share of cases where answer is problem-first and avoids entity-dump framing.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
mechanism_coherence_score: {
|
||||
description: "Average 0..5 score for mechanism clarity and alignment with expected defect class.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
entity_leakage_rate: {
|
||||
description: "Share of cases with internal/technical leakage in user-facing answer.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
accountant_actionability_score: {
|
||||
description: "Average 0..5 score for first-check concreteness and practical accounting actionability.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
route_correctness_rate: {
|
||||
description: "Share of cases routed to expected path class for query type.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
domain_purity_rate: {
|
||||
description: "Share of cases where top-3 retrieval stays inside expected P0 domain boundary.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
limitation_honesty_rate: {
|
||||
description: "Share of cases where mandatory limitations are explicit when evidence is incomplete.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
top_problem_unit_match_rate: {
|
||||
description: "Share of cases where top problem unit type matches expected annotation.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
}
|
||||
};
|
||||
|
||||
export const P0_QUALITY_GAP_METRIC_DEFINITIONS: Record<P0QualityGapMetricName, P0QualityGapMetricDefinition> = {
|
||||
generic_explanation_rate: {
|
||||
description: "Share of cases where direct answer stays generic and mechanism/actionability specificity is insufficient.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
false_confidence_rate: {
|
||||
description: "Share of cases with overconfident framing not supported by evidence/limitation state.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
mechanism_specificity_score: {
|
||||
description: "Average 0..5 score of mechanism-level specificity in accountant-facing answer.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
followup_context_retention_score: {
|
||||
description: "Rate of correct context retention in follow-up investigation cases.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
}
|
||||
};
|
||||
|
||||
export const P0_DEFAULT_FORBIDDEN_LEAKAGE_TOKENS = [
|
||||
"graph_",
|
||||
"domain_scope",
|
||||
"relation_patterns",
|
||||
"account_scope",
|
||||
"semantic_profile",
|
||||
"route",
|
||||
"profile",
|
||||
"store_canonical",
|
||||
"hybrid_store_plus_live",
|
||||
"deterministic_v2",
|
||||
"candidate_evidence",
|
||||
"problem_unit",
|
||||
"raw_entities",
|
||||
"graph_domain_scope"
|
||||
];
|
||||
|
||||
export interface P0AcceptanceThresholds {
|
||||
problem_first_answer_rate_min: number;
|
||||
mechanism_coherence_score_min: number;
|
||||
entity_leakage_rate_max: number;
|
||||
accountant_actionability_score_min: number;
|
||||
route_correctness_rate_min: number;
|
||||
domain_purity_rate_min: number;
|
||||
limitation_honesty_rate_min: number;
|
||||
top_problem_unit_match_rate_min: number;
|
||||
}
|
||||
|
||||
export interface P0QualityGapThresholds {
|
||||
generic_explanation_rate_max: number;
|
||||
false_confidence_rate_max: number;
|
||||
mechanism_specificity_score_min: number;
|
||||
followup_context_retention_score_min: number;
|
||||
}
|
||||
|
||||
export const P0_DEFAULT_ACCEPTANCE_THRESHOLDS: P0AcceptanceThresholds = {
|
||||
problem_first_answer_rate_min: 0.8,
|
||||
mechanism_coherence_score_min: 3.5,
|
||||
entity_leakage_rate_max: 0.08,
|
||||
accountant_actionability_score_min: 3.5,
|
||||
route_correctness_rate_min: 0.85,
|
||||
domain_purity_rate_min: 0.9,
|
||||
limitation_honesty_rate_min: 0.9,
|
||||
top_problem_unit_match_rate_min: 0.65
|
||||
};
|
||||
|
||||
export const P0_DEFAULT_QUALITY_GAP_THRESHOLDS: P0QualityGapThresholds = {
|
||||
generic_explanation_rate_max: 0.2,
|
||||
false_confidence_rate_max: 0.12,
|
||||
mechanism_specificity_score_min: 3.0,
|
||||
followup_context_retention_score_min: 0.75
|
||||
};
|
||||
|
||||
function isProblemUnitType(value: unknown): value is ProblemUnitType {
|
||||
return (
|
||||
value === "document_conflict" ||
|
||||
value === "broken_chain_segment" ||
|
||||
value === "lifecycle_anomaly_node" ||
|
||||
value === "unresolved_settlement_cluster" ||
|
||||
value === "period_risk_cluster" ||
|
||||
value === "cross_branch_inconsistency_cluster"
|
||||
);
|
||||
}
|
||||
|
||||
function isP0DomainId(value: unknown): value is P0DomainId {
|
||||
return value === "settlements_60_62" || value === "vat_document_register_book" || value === "month_close_costs_20_44";
|
||||
}
|
||||
|
||||
function isP0QueryClass(value: unknown): value is P0QueryClass {
|
||||
return (
|
||||
value === "symptom_first" ||
|
||||
value === "lifecycle_first" ||
|
||||
value === "causal_query" ||
|
||||
value === "mixed_ambiguity" ||
|
||||
value === "followup_investigation" ||
|
||||
value === "noisy_input" ||
|
||||
value === "translit_noisy" ||
|
||||
value === "multi_intent"
|
||||
);
|
||||
}
|
||||
|
||||
function isRouteExpected(value: unknown): value is P0RouteExpected {
|
||||
return (
|
||||
value === "hybrid_store_plus_live" ||
|
||||
value === "store_feature_risk" ||
|
||||
value === "store_canonical" ||
|
||||
value === "batch_refresh_then_store" ||
|
||||
value === "live_mcp_drilldown" ||
|
||||
value === "no_route"
|
||||
);
|
||||
}
|
||||
|
||||
function assertString(value: unknown, fieldPath: string): string {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': non-empty string is required.`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function assertBoolean(value: unknown, fieldPath: string): boolean {
|
||||
if (typeof value !== "boolean") {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': boolean is required.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertStringArray(value: unknown, fieldPath: string): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': string[] is required.`);
|
||||
}
|
||||
const output = value.map((item, index) => assertString(item, `${fieldPath}[${index}]`));
|
||||
return output;
|
||||
}
|
||||
|
||||
function assertRouteExpected(value: unknown, fieldPath: string): P0RouteExpected | P0RouteExpected[] {
|
||||
if (Array.isArray(value)) {
|
||||
const routes = value.map((item, index) => {
|
||||
if (!isRouteExpected(item)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}[${index}]': unsupported route '${String(item)}'.`);
|
||||
}
|
||||
return item;
|
||||
});
|
||||
if (routes.length === 0) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': route list must not be empty.`);
|
||||
}
|
||||
return routes;
|
||||
}
|
||||
if (!isRouteExpected(value)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': unsupported route '${String(value)}'.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseCase(raw: unknown, index: number): P0EvalCase {
|
||||
if (!raw || typeof raw !== "object") {
|
||||
throw new Error(`Invalid corpus case at index ${index}: object is required.`);
|
||||
}
|
||||
const row = raw as Record<string, unknown>;
|
||||
const expected = row.expected_annotations;
|
||||
if (!expected || typeof expected !== "object") {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations is required.`);
|
||||
}
|
||||
const annotations = expected as Record<string, unknown>;
|
||||
|
||||
const domain = row.domain;
|
||||
if (!isP0DomainId(domain)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported domain '${String(domain)}'.`);
|
||||
}
|
||||
const queryClass = row.query_class;
|
||||
if (!isP0QueryClass(queryClass)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported query_class '${String(queryClass)}'.`);
|
||||
}
|
||||
const expectedProblemType = row.expected_problem_unit_type;
|
||||
if (!isProblemUnitType(expectedProblemType)) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': expected_problem_unit_type '${String(expectedProblemType)}' is not supported.`
|
||||
);
|
||||
}
|
||||
const annotationProblemType = annotations.problem_unit_type_expected;
|
||||
if (!isProblemUnitType(annotationProblemType)) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.problem_unit_type_expected '${String(annotationProblemType)}' is not supported.`
|
||||
);
|
||||
}
|
||||
const annotationDomain = annotations.domain_expected;
|
||||
if (!isP0DomainId(annotationDomain)) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.domain_expected '${String(annotationDomain)}' is not supported.`
|
||||
);
|
||||
}
|
||||
|
||||
const followupSeedQuery =
|
||||
typeof row.followup_seed_query === "string" && row.followup_seed_query.trim() ? row.followup_seed_query.trim() : undefined;
|
||||
const followupExpectedContextTokens = Array.isArray(row.followup_expected_context_tokens)
|
||||
? assertStringArray(row.followup_expected_context_tokens, `cases[${index}].followup_expected_context_tokens`)
|
||||
: undefined;
|
||||
if (queryClass === "followup_investigation" && !followupSeedQuery) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': followup_investigation requires followup_seed_query.`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
case_id: assertString(row.case_id, `cases[${index}].case_id`),
|
||||
domain,
|
||||
user_query_raw: assertString(row.user_query_raw, `cases[${index}].user_query_raw`),
|
||||
query_class: queryClass,
|
||||
expected_problem_unit_type: expectedProblemType,
|
||||
expected_mechanism_summary: assertString(row.expected_mechanism_summary, `cases[${index}].expected_mechanism_summary`),
|
||||
expected_first_check: assertString(row.expected_first_check, `cases[${index}].expected_first_check`),
|
||||
must_include_limitations: assertBoolean(row.must_include_limitations, `cases[${index}].must_include_limitations`),
|
||||
forbidden_leakage_tokens: assertStringArray(row.forbidden_leakage_tokens, `cases[${index}].forbidden_leakage_tokens`),
|
||||
followup_seed_query: followupSeedQuery,
|
||||
followup_expected_context_tokens: followupExpectedContextTokens,
|
||||
notes: assertString(row.notes, `cases[${index}].notes`),
|
||||
expected_annotations: {
|
||||
domain_expected: annotationDomain,
|
||||
route_expected: assertRouteExpected(annotations.route_expected, `cases[${index}].expected_annotations.route_expected`),
|
||||
problem_unit_type_expected: annotationProblemType,
|
||||
mechanism_class_expected: assertString(
|
||||
annotations.mechanism_class_expected,
|
||||
`cases[${index}].expected_annotations.mechanism_class_expected`
|
||||
),
|
||||
evidence_expectation: assertString(annotations.evidence_expectation, `cases[${index}].expected_annotations.evidence_expectation`),
|
||||
must_show_limitation: assertBoolean(annotations.must_show_limitation, `cases[${index}].expected_annotations.must_show_limitation`),
|
||||
must_not_leak_internal: assertBoolean(
|
||||
annotations.must_not_leak_internal,
|
||||
`cases[${index}].expected_annotations.must_not_leak_internal`
|
||||
),
|
||||
first_check_action_expected: assertString(
|
||||
annotations.first_check_action_expected,
|
||||
`cases[${index}].expected_annotations.first_check_action_expected`
|
||||
)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function validateP0EvalCorpus(input: unknown): P0EvalCorpusFile {
|
||||
if (!input || typeof input !== "object") {
|
||||
throw new Error("Invalid P0 corpus: root object is required.");
|
||||
}
|
||||
const parsed = input as Record<string, unknown>;
|
||||
const casesRaw = parsed.cases;
|
||||
if (!Array.isArray(casesRaw)) {
|
||||
throw new Error("Invalid P0 corpus: cases[] is required.");
|
||||
}
|
||||
const cases = casesRaw.map((item, index) => parseCase(item, index));
|
||||
if (cases.length < 30 || cases.length > 150) {
|
||||
throw new Error(`Invalid P0 corpus: expected 30..150 cases, got ${cases.length}.`);
|
||||
}
|
||||
|
||||
const caseIds = assertStringArray(parsed.case_ids, "case_ids");
|
||||
const declaredIds = [...caseIds].sort();
|
||||
const actualIds = cases.map((item) => item.case_id).sort();
|
||||
const idsMatch = declaredIds.length === actualIds.length && declaredIds.every((value, index) => value === actualIds[index]);
|
||||
if (!idsMatch) {
|
||||
throw new Error("Invalid P0 corpus: case_ids[] does not match cases[].");
|
||||
}
|
||||
const scenarioCount = Number(parsed.scenario_count ?? 0);
|
||||
if (!Number.isFinite(scenarioCount) || scenarioCount !== cases.length) {
|
||||
throw new Error("Invalid P0 corpus: scenario_count must match cases.length.");
|
||||
}
|
||||
|
||||
const perDomain = cases.reduce<Record<P0DomainId, number>>(
|
||||
(acc, item) => {
|
||||
acc[item.domain] += 1;
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
settlements_60_62: 0,
|
||||
vat_document_register_book: 0,
|
||||
month_close_costs_20_44: 0
|
||||
}
|
||||
);
|
||||
for (const [domain, count] of Object.entries(perDomain)) {
|
||||
if (count < 10) {
|
||||
throw new Error(`Invalid P0 corpus: domain '${domain}' must contain at least 10 cases, got ${count}.`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
suite_id: assertString(parsed.suite_id, "suite_id"),
|
||||
suite_version: assertString(parsed.suite_version, "suite_version"),
|
||||
schema_version: assertString(parsed.schema_version, "schema_version"),
|
||||
scenario_count: scenarioCount,
|
||||
case_ids: caseIds,
|
||||
cases
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeExpectedRoutes(value: P0RouteExpected | P0RouteExpected[]): P0RouteExpected[] {
|
||||
return Array.isArray(value) ? value : [value];
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -214,48 +214,56 @@ export class AssistantSessionLogger {
|
||||
constructor(private readonly rootDir: string = ASSISTANT_SESSIONS_DIR) {}
|
||||
|
||||
public persistSession(session: AssistantSessionState): void {
|
||||
ensureDir(this.rootDir);
|
||||
const filePath = path.resolve(this.rootDir, `${session.session_id}.json`);
|
||||
try {
|
||||
ensureDir(this.rootDir);
|
||||
const filePath = path.resolve(this.rootDir, `${session.session_id}.json`);
|
||||
|
||||
const startedAt = session.items[0]?.created_at ?? session.updated_at;
|
||||
const userMessages = session.items.filter((item) => item.role === "user").length;
|
||||
const assistantMessages = session.items.filter((item) => item.role === "assistant").length;
|
||||
const assistantItems = session.items.filter((item) => item.role === "assistant");
|
||||
const lastAssistant = assistantItems.length > 0 ? assistantItems[assistantItems.length - 1] : null;
|
||||
const startedAt = session.items[0]?.created_at ?? session.updated_at;
|
||||
const userMessages = session.items.filter((item) => item.role === "user").length;
|
||||
const assistantMessages = session.items.filter((item) => item.role === "assistant").length;
|
||||
const assistantItems = session.items.filter((item) => item.role === "assistant");
|
||||
const lastAssistant = assistantItems.length > 0 ? assistantItems[assistantItems.length - 1] : null;
|
||||
|
||||
const traceIds = unique(session.items.map((item) => item.trace_id));
|
||||
const replyTypes = Array.from(
|
||||
new Set(
|
||||
session.items
|
||||
.map((item) => item.reply_type)
|
||||
.filter((item): item is AssistantReplyType => typeof item === "string" && item.length > 0)
|
||||
)
|
||||
);
|
||||
const turns = buildTurns(session.items);
|
||||
const traceIds = unique(session.items.map((item) => item.trace_id));
|
||||
const replyTypes = Array.from(
|
||||
new Set(
|
||||
session.items
|
||||
.map((item) => item.reply_type)
|
||||
.filter((item): item is AssistantReplyType => typeof item === "string" && item.length > 0)
|
||||
)
|
||||
);
|
||||
const turns = buildTurns(session.items);
|
||||
|
||||
const record: AssistantSessionLogRecord = {
|
||||
schema_version: "assistant_session_log_v1",
|
||||
session_id: session.session_id,
|
||||
started_at: startedAt,
|
||||
updated_at: session.updated_at,
|
||||
counters: {
|
||||
total_messages: session.items.length,
|
||||
user_messages: userMessages,
|
||||
assistant_messages: assistantMessages
|
||||
},
|
||||
trace_ids: traceIds,
|
||||
reply_types: replyTypes,
|
||||
investigation_state: session.investigation_state,
|
||||
turns,
|
||||
conversation: session.items,
|
||||
last_assistant: {
|
||||
message_id: lastAssistant?.message_id ?? null,
|
||||
reply_type: lastAssistant?.reply_type ?? null,
|
||||
trace_id: lastAssistant?.trace_id ?? null,
|
||||
created_at: lastAssistant?.created_at ?? null
|
||||
const record: AssistantSessionLogRecord = {
|
||||
schema_version: "assistant_session_log_v1",
|
||||
session_id: session.session_id,
|
||||
started_at: startedAt,
|
||||
updated_at: session.updated_at,
|
||||
counters: {
|
||||
total_messages: session.items.length,
|
||||
user_messages: userMessages,
|
||||
assistant_messages: assistantMessages
|
||||
},
|
||||
trace_ids: traceIds,
|
||||
reply_types: replyTypes,
|
||||
investigation_state: session.investigation_state,
|
||||
turns,
|
||||
conversation: session.items,
|
||||
last_assistant: {
|
||||
message_id: lastAssistant?.message_id ?? null,
|
||||
reply_type: lastAssistant?.reply_type ?? null,
|
||||
trace_id: lastAssistant?.trace_id ?? null,
|
||||
created_at: lastAssistant?.created_at ?? null
|
||||
}
|
||||
};
|
||||
|
||||
writeJsonFile(filePath, record);
|
||||
} catch (error) {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
if (code === "ENOSPC") {
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
writeJsonFile(filePath, record);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
FEATURE_ASSISTANT_STAGE2_EVAL_V1,
|
||||
REPORTS_DIR
|
||||
} from "../config";
|
||||
import { P0EvalRunner } from "../eval/p0_eval_runner";
|
||||
import type { AssistantMessageResponsePayload } from "../types/assistant";
|
||||
import type {
|
||||
EvalTarget,
|
||||
@@ -328,6 +329,98 @@ const ASSISTANT_STAGE1_COMPARISON_SCHEMA_VERSION = "assistant_stage1_eval_compar
|
||||
const DEFAULT_ASSISTANT_STAGE2_SUITE_FILE = "assistant_stage2_canonical_v0_1.json";
|
||||
const ASSISTANT_STAGE2_RUN_SCHEMA_VERSION = "assistant_stage2_eval_run_v0_1";
|
||||
const ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION = "assistant_stage2_eval_comparison_v0_1";
|
||||
const INMEM_EVAL_REPORT_PREFIX = "inmem_eval_report:";
|
||||
const INMEM_EVAL_REPORTS = new Map<string, Record<string, unknown>>();
|
||||
|
||||
function isNoSpaceError(error: unknown): boolean {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
return code === "ENOSPC";
|
||||
}
|
||||
|
||||
function tryWriteJsonFile(pathname: string, value: unknown): boolean {
|
||||
try {
|
||||
writeJsonFile(pathname, value);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function tryWriteTextFile(pathname: string, value: string): boolean {
|
||||
try {
|
||||
fs.writeFileSync(pathname, value, "utf-8");
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function putInMemoryEvalReport(report: Record<string, unknown>): string {
|
||||
const key = `${INMEM_EVAL_REPORT_PREFIX}${nanoid(12)}`;
|
||||
INMEM_EVAL_REPORTS.set(key, report);
|
||||
return key;
|
||||
}
|
||||
|
||||
function readEvalReportByRef(ref: string): { report: Record<string, unknown>; resolved_path: string } {
|
||||
if (ref.startsWith(INMEM_EVAL_REPORT_PREFIX)) {
|
||||
const report = INMEM_EVAL_REPORTS.get(ref);
|
||||
if (!report) {
|
||||
throw new Error(`In-memory eval report not found: ${ref}`);
|
||||
}
|
||||
return {
|
||||
report,
|
||||
resolved_path: ref
|
||||
};
|
||||
}
|
||||
|
||||
const resolvedPath = resolveReadablePath(ref);
|
||||
const report = JSON.parse(fs.readFileSync(resolvedPath, "utf-8")) as Record<string, unknown>;
|
||||
return {
|
||||
report,
|
||||
resolved_path: resolvedPath
|
||||
};
|
||||
}
|
||||
|
||||
function compactAssistantStage1Report(report: Record<string, unknown>): Record<string, unknown> {
|
||||
const results = Array.isArray(report.results) ? (report.results as Array<Record<string, unknown>>) : [];
|
||||
const compactResults = results.map((item) => ({
|
||||
case_id: item.case_id ?? null,
|
||||
scenario_tag: item.scenario_tag ?? null,
|
||||
accountant_usefulness_score: item.accountant_usefulness_score ?? null,
|
||||
accountant_metrics:
|
||||
typeof item.accountant_metrics === "object" && item.accountant_metrics !== null ? item.accountant_metrics : null
|
||||
}));
|
||||
return {
|
||||
...report,
|
||||
results: compactResults
|
||||
};
|
||||
}
|
||||
|
||||
function compactAssistantStage2Report(report: Record<string, unknown>): Record<string, unknown> {
|
||||
const results = Array.isArray(report.results) ? (report.results as Array<Record<string, unknown>>) : [];
|
||||
const compactResults = results.map((item) => {
|
||||
const metricSubscores = (item.metric_subscores ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
case_id: item.case_id ?? null,
|
||||
metric_subscores: {
|
||||
problem_clarity_score: metricSubscores.problem_clarity_score ?? null,
|
||||
mechanism_coherence_score: metricSubscores.mechanism_coherence_score ?? null,
|
||||
problem_first_answer_rate: metricSubscores.problem_first_answer_rate ?? null,
|
||||
entity_leakage_rate: metricSubscores.entity_leakage_rate ?? null
|
||||
}
|
||||
};
|
||||
});
|
||||
return {
|
||||
...report,
|
||||
results: compactResults
|
||||
};
|
||||
}
|
||||
|
||||
type AssistantMetricKey = keyof AssistantEvalMetricVector;
|
||||
type AssistantStage2MetricKey = keyof AssistantStage2MetricVector;
|
||||
@@ -1140,7 +1233,7 @@ export class EvalService {
|
||||
};
|
||||
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
tryWriteJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
|
||||
return report;
|
||||
}
|
||||
@@ -1534,8 +1627,9 @@ export class EvalService {
|
||||
currentReport: Record<string, unknown>;
|
||||
baselineReportFile: string;
|
||||
}): Record<string, unknown> {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs.readFileSync(baselinePath, "utf-8")) as Record<string, unknown>;
|
||||
const baselineRef = readEvalReportByRef(input.baselineReportFile);
|
||||
const baselinePath = baselineRef.resolved_path;
|
||||
const baselineReport = baselineRef.report;
|
||||
const currentReport = input.currentReport;
|
||||
const metricKeys: AssistantMetricKey[] = [
|
||||
"retrieval_differentiation_rate",
|
||||
@@ -1633,14 +1727,15 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const jsonPath = path.resolve(REPORTS_DIR, `${comparisonId}.json`);
|
||||
const mdPath = path.resolve(REPORTS_DIR, `${comparisonId}.md`);
|
||||
writeJsonFile(jsonPath, comparisonReport);
|
||||
fs.writeFileSync(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport), "utf-8");
|
||||
const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
|
||||
const mdWritten = tryWriteTextFile(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport));
|
||||
const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
|
||||
|
||||
return {
|
||||
...comparisonReport,
|
||||
artifacts: {
|
||||
comparison_report_json_path: jsonPath,
|
||||
comparison_report_md_path: mdPath
|
||||
comparison_report_json_path: comparisonRef,
|
||||
comparison_report_md_path: mdWritten ? mdPath : null
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1649,8 +1744,9 @@ export class EvalService {
|
||||
currentReport: Record<string, unknown>;
|
||||
baselineReportFile: string;
|
||||
}): Record<string, unknown> {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs.readFileSync(baselinePath, "utf-8")) as Record<string, unknown>;
|
||||
const baselineRef = readEvalReportByRef(input.baselineReportFile);
|
||||
const baselinePath = baselineRef.resolved_path;
|
||||
const baselineReport = baselineRef.report;
|
||||
const currentReport = input.currentReport;
|
||||
const metricKeys: AssistantStage2MetricKey[] = [
|
||||
"problem_unit_precision",
|
||||
@@ -1760,14 +1856,15 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const jsonPath = path.resolve(REPORTS_DIR, `${comparisonId}.json`);
|
||||
const mdPath = path.resolve(REPORTS_DIR, `${comparisonId}.md`);
|
||||
writeJsonFile(jsonPath, comparisonReport);
|
||||
fs.writeFileSync(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport), "utf-8");
|
||||
const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
|
||||
const mdWritten = tryWriteTextFile(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport));
|
||||
const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
|
||||
|
||||
return {
|
||||
...comparisonReport,
|
||||
artifacts: {
|
||||
comparison_report_json_path: jsonPath,
|
||||
comparison_report_md_path: mdPath
|
||||
comparison_report_json_path: comparisonRef,
|
||||
comparison_report_md_path: mdWritten ? mdPath : null
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1803,7 +1900,7 @@ export class EvalService {
|
||||
|
||||
try {
|
||||
for (const turn of suiteCase.turns) {
|
||||
const response = await assistantService.handleMessage({
|
||||
const response = (await assistantService.handleMessage({
|
||||
session_id: sessionId,
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
@@ -1819,7 +1916,7 @@ export class EvalService {
|
||||
domainPrompt: payload.normalizeConfig.domainPrompt,
|
||||
fewShotExamples: payload.normalizeConfig.fewShotExamples,
|
||||
useMock: payload.useMock
|
||||
});
|
||||
})) as AssistantMessageResponsePayload;
|
||||
turnResponses.push(response);
|
||||
requestsTotal += 1;
|
||||
}
|
||||
@@ -2099,12 +2196,14 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const runJsonPath = path.resolve(REPORTS_DIR, `${runId}.json`);
|
||||
const runMdPath = path.resolve(REPORTS_DIR, `${runId}.md`);
|
||||
writeJsonFile(runJsonPath, report);
|
||||
fs.writeFileSync(runMdPath, buildAssistantEvalMarkdownReport(report), "utf-8");
|
||||
const compactReport = compactAssistantStage1Report(report);
|
||||
const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
|
||||
const mdWritten = tryWriteTextFile(runMdPath, buildAssistantEvalMarkdownReport(compactReport));
|
||||
const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
|
||||
|
||||
report.artifacts = {
|
||||
run_report_json_path: runJsonPath,
|
||||
run_report_md_path: runMdPath
|
||||
run_report_json_path: runReportRef,
|
||||
run_report_md_path: mdWritten ? runMdPath : null
|
||||
};
|
||||
|
||||
if (payload.compareWithReportFile) {
|
||||
@@ -2151,7 +2250,7 @@ export class EvalService {
|
||||
|
||||
try {
|
||||
for (const turn of suiteCase.turns) {
|
||||
const response = await assistantService.handleMessage({
|
||||
const response = (await assistantService.handleMessage({
|
||||
session_id: sessionId,
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
@@ -2167,7 +2266,7 @@ export class EvalService {
|
||||
domainPrompt: payload.normalizeConfig.domainPrompt,
|
||||
fewShotExamples: payload.normalizeConfig.fewShotExamples,
|
||||
useMock: payload.useMock
|
||||
});
|
||||
})) as AssistantMessageResponsePayload;
|
||||
turnResponses.push(response);
|
||||
requestsTotal += 1;
|
||||
}
|
||||
@@ -2393,12 +2492,14 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const runJsonPath = path.resolve(REPORTS_DIR, `${runId}.json`);
|
||||
const runMdPath = path.resolve(REPORTS_DIR, `${runId}.md`);
|
||||
writeJsonFile(runJsonPath, report);
|
||||
fs.writeFileSync(runMdPath, buildAssistantStage2EvalMarkdownReport(report), "utf-8");
|
||||
const compactReport = compactAssistantStage2Report(report);
|
||||
const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
|
||||
const mdWritten = tryWriteTextFile(runMdPath, buildAssistantStage2EvalMarkdownReport(compactReport));
|
||||
const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
|
||||
|
||||
report.artifacts = {
|
||||
run_report_json_path: runJsonPath,
|
||||
run_report_md_path: runMdPath
|
||||
run_report_json_path: runReportRef,
|
||||
run_report_md_path: mdWritten ? runMdPath : null
|
||||
};
|
||||
|
||||
if (payload.compareWithReportFile) {
|
||||
@@ -2411,6 +2512,33 @@ export class EvalService {
|
||||
return report;
|
||||
}
|
||||
|
||||
private async runAssistantP0(payload: {
|
||||
normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
|
||||
caseIds?: string[];
|
||||
useMock?: boolean;
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
compareWithReportFile?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
if (!FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
|
||||
throw new ApiError(
|
||||
"ASSISTANT_P0_EVAL_DISABLED",
|
||||
"Assistant P0 eval target is disabled by FEATURE_ASSISTANT_STAGE2_EVAL_V1.",
|
||||
409
|
||||
);
|
||||
}
|
||||
|
||||
const runner = new P0EvalRunner(this.normalizerService);
|
||||
return runner.run({
|
||||
normalizeConfig: payload.normalizeConfig,
|
||||
caseIds: payload.caseIds,
|
||||
useMock: payload.useMock,
|
||||
mode: payload.mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
|
||||
public async run(payload: {
|
||||
normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
|
||||
caseIds?: string[];
|
||||
@@ -2446,6 +2574,17 @@ export class EvalService {
|
||||
});
|
||||
}
|
||||
|
||||
if (evalTarget === "assistant_p0") {
|
||||
return this.runAssistantP0({
|
||||
normalizeConfig: payload.normalizeConfig,
|
||||
caseIds: payload.caseIds,
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
|
||||
const promptVersion = String(payload.normalizeConfig.promptVersion ?? "").toLowerCase();
|
||||
const schemaVersion = String(payload.normalizeConfig.schemaVersion ?? "").toLowerCase();
|
||||
const isV2 =
|
||||
@@ -2659,7 +2798,7 @@ export class EvalService {
|
||||
};
|
||||
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
tryWriteJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
|
||||
const shouldWriteV11Artifacts =
|
||||
mode === "single-pass-strict" &&
|
||||
@@ -2668,14 +2807,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV11Artifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1 Eval Run"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2687,14 +2825,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV1121EvalArtifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.2.1 Eval Run"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2705,14 +2842,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV111MicroArtifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.1 Micro Eval"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2723,14 +2859,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV112MicroArtifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.2 Micro Eval"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,103 @@ function collectOpenUncertainties(
|
||||
return capStrings([...requirementNotes, ...limitationNotes], INVESTIGATION_MAX_UNCERTAINTIES);
|
||||
}
|
||||
|
||||
function normalizeAccountPrefix(value: string): string | null {
|
||||
const account = String(value ?? "").trim();
|
||||
if (!account) {
|
||||
return null;
|
||||
}
|
||||
const match = account.match(/^(\d{2})/);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function isSettlementAccount(value: string): boolean {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
return prefix === "60" || prefix === "62" || prefix === "51" || prefix === "76";
|
||||
}
|
||||
|
||||
function isVatAccount(value: string): boolean {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
return prefix === "19" || prefix === "68";
|
||||
}
|
||||
|
||||
function isCloseCostsAccount(value: string): boolean {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
if (!prefix) {
|
||||
return false;
|
||||
}
|
||||
const account = Number(prefix);
|
||||
return (account >= 20 && account <= 44) || prefix === "97";
|
||||
}
|
||||
|
||||
function inferFollowupActiveDomain(input: {
|
||||
userMessage: string;
|
||||
focusAccounts: string[];
|
||||
routeSummary: RouteHintSummary | null;
|
||||
previous: InvestigationStateWithProblemUnits;
|
||||
}): string | null {
|
||||
const corpus = `${input.userMessage} ${input.previous.focus.active_query_subject ?? ""}`.toLowerCase();
|
||||
const hasSettlementSignal =
|
||||
input.focusAccounts.some((item) => isSettlementAccount(item)) ||
|
||||
/(60(?:\.\d{2})?|62(?:\.\d{2})?|оплат|расчет|расч[её]т|зачет|зач[её]т|аванс|долг|поставщ|покупат|settlement|payment|supplier|customer)/i.test(
|
||||
corpus
|
||||
);
|
||||
if (hasSettlementSignal) {
|
||||
return "settlements_60_62";
|
||||
}
|
||||
|
||||
const hasVatSignal =
|
||||
input.focusAccounts.some((item) => isVatAccount(item)) ||
|
||||
/(ндс|счет[\s-]?фактур|сч[её]т[\s-]?фактур|книг[аи]|vat|invoice|book|register)/i.test(corpus);
|
||||
if (hasVatSignal) {
|
||||
return "vat_document_register_book";
|
||||
}
|
||||
|
||||
const hasCloseSignal =
|
||||
input.focusAccounts.some((item) => isCloseCostsAccount(item)) ||
|
||||
/(закрыти|закрытие|месяц|затрат|распредел|списан|period\s*close|month\s*close|allocation|residual|cost)/i.test(corpus);
|
||||
if (hasCloseSignal) {
|
||||
return "month_close_costs_20_44";
|
||||
}
|
||||
|
||||
const routeDomain = deriveDomain(input.routeSummary);
|
||||
if (routeDomain && routeDomain !== "no_route") {
|
||||
return routeDomain;
|
||||
}
|
||||
|
||||
return input.previous.followup_context?.active_domain ?? input.previous.focus.domain ?? null;
|
||||
}
|
||||
|
||||
function collectUncoveredRequirementIds(coverageReport: RequirementCoverageReport): string[] {
|
||||
return capStrings(
|
||||
[
|
||||
...coverageReport.requirements_uncovered,
|
||||
...coverageReport.requirements_partially_covered,
|
||||
...coverageReport.clarification_needed_for,
|
||||
...coverageReport.out_of_scope_requirements
|
||||
],
|
||||
INVESTIGATION_MAX_REQUIREMENT_LINKS
|
||||
);
|
||||
}
|
||||
|
||||
function collectEvidenceSummary(retrievalResults: UnifiedRetrievalResult[]): string[] {
|
||||
const lines = retrievalResults.map((result) => {
|
||||
const requirementRef = result.requirement_ids[0] ?? result.fragment_id;
|
||||
return `${requirementRef}:${result.status}:${result.route}`;
|
||||
});
|
||||
return capStrings(lines, 6);
|
||||
}
|
||||
|
||||
function settlementFocusActions(activeDomain: string | null): string[] {
|
||||
if (activeDomain !== "settlements_60_62") {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
"Проверьте договор и объект расчетов по платежу.",
|
||||
"Сверьте регистр расчетов и привязку платежа к закрывающему документу.",
|
||||
"Проверьте зачет аванса или взаимозачет по связке 60/62."
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeEntityBacklinks(values: ProblemUnitEntityBacklink[]): ProblemUnitEntityBacklink[] {
|
||||
const result: ProblemUnitEntityBacklink[] = [];
|
||||
const seen = new Set<string>();
|
||||
@@ -273,7 +370,27 @@ export function cloneInvestigationState(state: InvestigationStateWithProblemUnit
|
||||
followup_context: state.followup_context
|
||||
? {
|
||||
...state.followup_context,
|
||||
referenced_requirement_ids: [...state.followup_context.referenced_requirement_ids]
|
||||
referenced_requirement_ids: [...state.followup_context.referenced_requirement_ids],
|
||||
...(state.followup_context.active_requirement_ids
|
||||
? {
|
||||
active_requirement_ids: [...state.followup_context.active_requirement_ids]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.uncovered_requirement_ids
|
||||
? {
|
||||
uncovered_requirement_ids: [...state.followup_context.uncovered_requirement_ids]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.settlement_next_actions
|
||||
? {
|
||||
settlement_next_actions: [...state.followup_context.settlement_next_actions]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.evidence_summary
|
||||
? {
|
||||
evidence_summary: [...state.followup_context.evidence_summary]
|
||||
}
|
||||
: {})
|
||||
}
|
||||
: null
|
||||
};
|
||||
@@ -320,12 +437,27 @@ export function createEmptyInvestigationState(
|
||||
export function updateInvestigationState(input: UpdateInvestigationStateInput): InvestigationStateWithProblemUnits {
|
||||
const previous = input.previous;
|
||||
const focusFromMessage = capStrings(detectAccounts(input.userMessage), INVESTIGATION_MAX_PRIMARY_ACCOUNTS);
|
||||
const mergedFocusAccounts = capStrings(
|
||||
[...focusFromMessage, ...previous.focus.primary_accounts],
|
||||
INVESTIGATION_MAX_PRIMARY_ACCOUNTS
|
||||
);
|
||||
const requirementIds = capStrings(
|
||||
input.requirements.map((item) => item.requirement_id),
|
||||
INVESTIGATION_MAX_REQUIREMENT_LINKS
|
||||
);
|
||||
const mainRequirement = input.requirements[0]?.requirement_text ?? input.userMessage;
|
||||
const problemUnitState = updateProblemUnitState(previous, input.retrievalResults);
|
||||
const uncoveredRequirementIds = collectUncoveredRequirementIds(input.coverageReport);
|
||||
const activeDomain = inferFollowupActiveDomain({
|
||||
userMessage: input.userMessage,
|
||||
focusAccounts: mergedFocusAccounts,
|
||||
routeSummary: input.routeSummary,
|
||||
previous
|
||||
});
|
||||
const focusDomain = activeDomain ?? deriveDomain(input.routeSummary) ?? previous.focus.domain;
|
||||
const settlementNextActions = settlementFocusActions(activeDomain);
|
||||
const lastProblemUnitId = problemUnitState?.active_problem_units[0] ?? null;
|
||||
const evidenceSummary = collectEvidenceSummary(input.retrievalResults);
|
||||
|
||||
return {
|
||||
schema_version: INVESTIGATION_STATE_SCHEMA_VERSION,
|
||||
@@ -335,12 +467,9 @@ export function updateInvestigationState(input: UpdateInvestigationStateInput):
|
||||
updated_at: input.timestamp,
|
||||
question_id: input.questionId,
|
||||
focus: {
|
||||
domain: deriveDomain(input.routeSummary) ?? previous.focus.domain,
|
||||
domain: focusDomain,
|
||||
period: detectPeriod(input.userMessage) ?? previous.focus.period,
|
||||
primary_accounts: capStrings(
|
||||
[...focusFromMessage, ...previous.focus.primary_accounts],
|
||||
INVESTIGATION_MAX_PRIMARY_ACCOUNTS
|
||||
),
|
||||
primary_accounts: mergedFocusAccounts,
|
||||
active_query_subject: mainRequirement.slice(0, 180)
|
||||
},
|
||||
narrowing_status: deriveNarrowingStatus(input.routeSummary, input.coverageReport),
|
||||
@@ -353,7 +482,13 @@ export function updateInvestigationState(input: UpdateInvestigationStateInput):
|
||||
followup_context: {
|
||||
previous_question_id: previous.question_id,
|
||||
last_user_message: input.userMessage.slice(0, 240),
|
||||
referenced_requirement_ids: requirementIds
|
||||
referenced_requirement_ids: requirementIds,
|
||||
active_domain: activeDomain,
|
||||
active_requirement_ids: requirementIds,
|
||||
uncovered_requirement_ids: uncoveredRequirementIds,
|
||||
last_problem_unit_id: lastProblemUnitId,
|
||||
settlement_next_actions: settlementNextActions,
|
||||
evidence_summary: evidenceSummary
|
||||
},
|
||||
query_mode_hint: deriveQueryModeHint(input.routeSummary),
|
||||
...(problemUnitState
|
||||
|
||||
@@ -4,7 +4,11 @@ import type {
|
||||
RetrievalResultType,
|
||||
UnifiedRetrievalResult
|
||||
} from "../types/assistant";
|
||||
import { FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1, FEATURE_ASSISTANT_PROBLEM_UNITS_V1 } from "../config";
|
||||
import {
|
||||
FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1,
|
||||
FEATURE_ASSISTANT_GRAPH_RUNTIME_V1,
|
||||
FEATURE_ASSISTANT_PROBLEM_UNITS_V1
|
||||
} from "../config";
|
||||
import { EVIDENCE_SOURCE_REF_SCHEMA_VERSION } from "../types/stage1Contracts";
|
||||
import type {
|
||||
EvidenceConfidence,
|
||||
@@ -15,6 +19,8 @@ import type {
|
||||
EvidenceSourceRef
|
||||
} from "../types/stage1Contracts";
|
||||
import { assembleProblemUnits } from "./problemUnitAssembler";
|
||||
import { buildAccountingGraph } from "./stage4GraphRuntime";
|
||||
import type { GraphSignalSummary } from "../types/stage4Graph";
|
||||
|
||||
interface RawRetrievalResult {
|
||||
status?: string;
|
||||
@@ -124,6 +130,22 @@ function mergeSummaryWithProblemUnitMeta(
|
||||
};
|
||||
}
|
||||
|
||||
function mergeSummaryWithGraphMeta(summary: Record<string, unknown>, graphSummary: GraphSignalSummary): Record<string, unknown> {
|
||||
return {
|
||||
...summary,
|
||||
graph_runtime_enabled: true,
|
||||
graph_total_units: graphSummary.total_units,
|
||||
graph_bound_units: graphSummary.bound_units,
|
||||
graph_nodes_count: graphSummary.node_count,
|
||||
graph_edges_count: graphSummary.edge_count,
|
||||
graph_missing_links_count: graphSummary.missing_links_count,
|
||||
graph_conflicting_links_count: graphSummary.conflicting_links_count,
|
||||
graph_coverage_grade: graphSummary.graph_coverage_grade,
|
||||
graph_domain_distribution: graphSummary.domain_distribution,
|
||||
graph_relation_distribution: graphSummary.relation_distribution
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeConfidence(value: unknown): RetrievalConfidence {
|
||||
if (value === "high" || value === "medium" || value === "low") {
|
||||
return value;
|
||||
@@ -526,24 +548,60 @@ export function normalizeRetrievalResult(
|
||||
business_interpretation: baseResult.business_interpretation
|
||||
});
|
||||
|
||||
const enrichedSummary = mergeSummaryWithProblemUnitMeta(summary, {
|
||||
candidateEvidenceCount: assembled.candidate_evidence.length,
|
||||
problemUnitsCount: assembled.problem_units.length,
|
||||
unitTypes: assembled.problem_unit_summary.unit_types,
|
||||
duplicateCollapses: assembled.problem_unit_summary.duplicate_collapses,
|
||||
severityDistribution: assembled.problem_unit_summary.severity_distribution,
|
||||
confidenceDistribution: assembled.problem_unit_summary.confidence_distribution,
|
||||
lifecycleEnrichedUnits: assembled.problem_unit_summary.lifecycle_enriched_units ?? 0,
|
||||
lifecycleDomainDistribution: (assembled.problem_unit_summary.lifecycle_domain_distribution ?? {}) as Record<string, number>,
|
||||
lifecycleDefectDistribution: (assembled.problem_unit_summary.lifecycle_defect_distribution ?? {}) as Record<string, number>
|
||||
const graphBuild = FEATURE_ASSISTANT_GRAPH_RUNTIME_V1
|
||||
? buildAccountingGraph({
|
||||
route,
|
||||
candidateEvidence: assembled.candidate_evidence,
|
||||
problemUnits: assembled.problem_units
|
||||
})
|
||||
: null;
|
||||
|
||||
const graphBindingByUnitId = new Map(graphBuild?.unit_bindings.map((item) => [item.problem_unit_id, item] as const) ?? []);
|
||||
const graphBoundProblemUnits = assembled.problem_units.map((unit) => {
|
||||
const binding = graphBindingByUnitId.get(unit.problem_unit_id);
|
||||
if (!binding) {
|
||||
return unit;
|
||||
}
|
||||
return {
|
||||
...unit,
|
||||
graph_binding: binding
|
||||
};
|
||||
});
|
||||
|
||||
const graphBoundSummary = graphBuild
|
||||
? {
|
||||
...assembled.problem_unit_summary,
|
||||
graph_summary: graphBuild.summary
|
||||
}
|
||||
: assembled.problem_unit_summary;
|
||||
|
||||
let enrichedSummary = mergeSummaryWithProblemUnitMeta(summary, {
|
||||
candidateEvidenceCount: assembled.candidate_evidence.length,
|
||||
problemUnitsCount: graphBoundProblemUnits.length,
|
||||
unitTypes: graphBoundSummary.unit_types,
|
||||
duplicateCollapses: graphBoundSummary.duplicate_collapses,
|
||||
severityDistribution: graphBoundSummary.severity_distribution,
|
||||
confidenceDistribution: graphBoundSummary.confidence_distribution,
|
||||
lifecycleEnrichedUnits: graphBoundSummary.lifecycle_enriched_units ?? 0,
|
||||
lifecycleDomainDistribution: (graphBoundSummary.lifecycle_domain_distribution ?? {}) as Record<string, number>,
|
||||
lifecycleDefectDistribution: (graphBoundSummary.lifecycle_defect_distribution ?? {}) as Record<string, number>
|
||||
});
|
||||
|
||||
if (graphBuild) {
|
||||
enrichedSummary = mergeSummaryWithGraphMeta(enrichedSummary, graphBuild.summary);
|
||||
}
|
||||
|
||||
return {
|
||||
...baseResult,
|
||||
summary: enrichedSummary,
|
||||
raw_entities: items,
|
||||
candidate_evidence: assembled.candidate_evidence,
|
||||
problem_units: assembled.problem_units,
|
||||
problem_unit_summary: assembled.problem_unit_summary
|
||||
problem_units: graphBoundProblemUnits,
|
||||
problem_unit_summary: graphBoundSummary,
|
||||
...(graphBuild
|
||||
? {
|
||||
accounting_graph: graphBuild
|
||||
}
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type {
|
||||
DeterministicRouteHint,
|
||||
NoRouteReason,
|
||||
NormalizedPayload,
|
||||
NormalizedQueryV1,
|
||||
@@ -42,6 +43,244 @@ function toRouteHintSummaryV1(normalized: NormalizedQueryV1): RouteHintSummaryV1
|
||||
type V2Family = NormalizedQueryV2 | NormalizedQueryV2_0_1 | NormalizedQueryV2_0_2;
|
||||
type V2FamilyFragment = V2Family["fragments"][number];
|
||||
|
||||
type RouteQueryClass =
|
||||
| "exact_object_trace"
|
||||
| "ranking_or_period_summary"
|
||||
| "symptom_first"
|
||||
| "lifecycle_first"
|
||||
| "chain_break"
|
||||
| "period_impact"
|
||||
| "causal_query"
|
||||
| "mixed_ambiguity"
|
||||
| "rule_check_without_symptom"
|
||||
| "canonical_fact_lookup";
|
||||
|
||||
interface RouteDisciplineRule {
|
||||
query_class: RouteQueryClass;
|
||||
required_route: Exclude<DeterministicRouteHint, "no_route">;
|
||||
allowed_fallback: DeterministicRouteHint[];
|
||||
forbidden_fallback: DeterministicRouteHint[];
|
||||
description: string;
|
||||
}
|
||||
|
||||
const ACCOUNT_HINT_PATTERN =
|
||||
/(?:\b(?:account|acct|schet|счет|сч)\s*[:#]?\s*(?:[1-9][0-9](?:[./-][0-9]{1,2})?)|\b(?:19|20|21|23|25|26|28|29|44|51|60|62|68)\b)/i;
|
||||
const PERIOD_PATTERN = /\b20\d{2}(?:[-./](?:0[1-9]|1[0-2]))?\b/i;
|
||||
const SYMPTOM_MARKER_PATTERN =
|
||||
/(?:\bsymptom\b|\banomaly\b|\bproblem\b|\bissue\b|\btail\b|\bhanging\b|\bblocked\b|\bincomplete\b|remains?\s+open|not\s+(?:confirmed|observed|resolved|closed)|не\s+(?:подтвержден|закрыт|наблюдается)|хвост|сбой|проблем)/i;
|
||||
const LIFECYCLE_MARKER_PATTERN =
|
||||
/(?:\blifecycle\b|\bchain\b|\btransition\b|\bstep\b|\btrace\b|цепоч|этап|переход|связк|где\s+разрыв)/i;
|
||||
const CHAIN_BREAK_PATTERN =
|
||||
/(?:\bbreak\b|\bbroken\b|\bgap\b|missing\s+(?:transition|step|link)|chain\s+break|разрыв|обрыв|нет\s+переход|не\s+дошл|не\s+наблюд)/i;
|
||||
const PERIOD_IMPACT_PATTERN =
|
||||
/(?:period\s*close|month\s*close|month-end|residual|allocation|20[/-]44|закрыти|остатк|распредел|конец\s+месяц)/i;
|
||||
const CAUSAL_PATTERN = /(?:\bwhy\b|\bbecause\b|\breason\b|explain\s+mechanism|почему|объясни|механизм|причин)/i;
|
||||
const AMBIGUITY_PATTERN =
|
||||
/(?:\bmaybe\b|\bperhaps\b|not\s+sure|i\s+only\s+know|part\s+may\s+be\s+missing|возможно|может\s+быть|не\s+уверен|не\s+знаю|часть\s+цепочки\s+не\s+подтвержд)/i;
|
||||
const TRANSLIT_PROBLEM_PATTERN = /(?:raschet|oplata|zakryt|nds|vychet|zatrat|ostatok|cepoch|perehod|pochemu|prichin|period)/i;
|
||||
const DOMAIN_LEXICAL_ANCHOR_PATTERN =
|
||||
/(?:\b(?:settlement|payment|bank|supplier|customer|vat|nds|invoice|register|book|period\s*close|month\s*close|close\s*operation|allocation|residual|cost|expenses?)\b|оплат|расчет|РЅРґСЃ|СЃС‡[её]С‚.?фактур|РєРЅРёРі[аи]|затрат|закрыт|остатк)/i;
|
||||
|
||||
export const ROUTE_DISCIPLINE_RULE_TABLE: RouteDisciplineRule[] = [
|
||||
{
|
||||
query_class: "exact_object_trace",
|
||||
required_route: "live_mcp_drilldown",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical", "hybrid_store_plus_live", "store_feature_risk", "batch_refresh_then_store"],
|
||||
description: "Exact object trace queries always run via live drilldown."
|
||||
},
|
||||
{
|
||||
query_class: "ranking_or_period_summary",
|
||||
required_route: "batch_refresh_then_store",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical", "hybrid_store_plus_live"],
|
||||
description: "Ranking and period summary queries require analytical batch path."
|
||||
},
|
||||
{
|
||||
query_class: "symptom_first",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Symptom-first intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "lifecycle_first",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Lifecycle-first intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "chain_break",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Chain-break intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "period_impact",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Period-impact problem intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "causal_query",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Causal/mechanism intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "mixed_ambiguity",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Mixed ambiguity keeps hybrid as primary route, with explicit no-route fallback."
|
||||
},
|
||||
{
|
||||
query_class: "rule_check_without_symptom",
|
||||
required_route: "store_feature_risk",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Rule checks without symptom/lifecycle signals run via risk profile path."
|
||||
},
|
||||
{
|
||||
query_class: "canonical_fact_lookup",
|
||||
required_route: "store_canonical",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["hybrid_store_plus_live"],
|
||||
description: "Only plain factual lookups are allowed to stay on canonical path."
|
||||
}
|
||||
];
|
||||
|
||||
const ROUTE_DISCIPLINE_RULE_MAP = new Map<RouteQueryClass, RouteDisciplineRule>(
|
||||
ROUTE_DISCIPLINE_RULE_TABLE.map((item) => [item.query_class, item])
|
||||
);
|
||||
|
||||
function mergedFragmentText(fragment: V2FamilyFragment): string {
|
||||
return `${fragment.raw_fragment_text ?? ""} ${fragment.normalized_fragment_text ?? ""}`.toLowerCase();
|
||||
}
|
||||
|
||||
function hasLifecycleDomainHint(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
const accountHints = Array.isArray(fragment.account_hints) ? fragment.account_hints.map((item) => String(item)) : [];
|
||||
if (accountHints.some((item) => /^(97|01|02|08|19|20|21|23|25|26|28|29|44|68(?:\.\d+)?|51|60|62)$/.test(item))) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
fragment.candidate_labels.includes("anomaly_probe") ||
|
||||
fragment.candidate_labels.includes("period_close_risk") ||
|
||||
PERIOD_IMPACT_PATTERN.test(lowerText)
|
||||
);
|
||||
}
|
||||
|
||||
function hasSymptomSignal(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return (
|
||||
fragment.flags.asks_for_anomaly_scan ||
|
||||
fragment.candidate_labels.includes("anomaly_probe") ||
|
||||
fragment.candidate_labels.includes("period_close_risk") ||
|
||||
SYMPTOM_MARKER_PATTERN.test(lowerText) ||
|
||||
TRANSLIT_PROBLEM_PATTERN.test(lowerText)
|
||||
);
|
||||
}
|
||||
|
||||
function hasLifecycleSignal(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return (
|
||||
fragment.flags.asks_for_chain_explanation ||
|
||||
fragment.flags.mentions_period_close_context ||
|
||||
LIFECYCLE_MARKER_PATTERN.test(lowerText) ||
|
||||
hasLifecycleDomainHint(fragment, lowerText)
|
||||
);
|
||||
}
|
||||
|
||||
function hasChainBreakSignal(lowerText: string): boolean {
|
||||
return CHAIN_BREAK_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function hasPeriodImpactSignal(lowerText: string): boolean {
|
||||
return PERIOD_IMPACT_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function hasCausalSignal(lowerText: string): boolean {
|
||||
return CAUSAL_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function hasAmbiguitySignal(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return (
|
||||
AMBIGUITY_PATTERN.test(lowerText) ||
|
||||
fragment.confidence === "low" ||
|
||||
fragment.domain_relevance === "unclear" ||
|
||||
fragment.business_scope === "unclear"
|
||||
);
|
||||
}
|
||||
|
||||
function hasAccountOrPeriodAnchor(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return fragment.account_hints.length > 0 || ACCOUNT_HINT_PATTERN.test(lowerText) || PERIOD_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function resolveRouteClass(fragment: V2FamilyFragment): RouteDisciplineRule {
|
||||
const lowerText = mergedFragmentText(fragment);
|
||||
const symptomSignal = hasSymptomSignal(fragment, lowerText);
|
||||
const lifecycleSignal = hasLifecycleSignal(fragment, lowerText);
|
||||
const chainBreakSignal = hasChainBreakSignal(lowerText);
|
||||
const periodImpactSignal = hasPeriodImpactSignal(lowerText);
|
||||
const causalSignal = hasCausalSignal(lowerText);
|
||||
const ambiguitySignal = hasAmbiguitySignal(fragment, lowerText);
|
||||
const accountOrPeriodAnchor = hasAccountOrPeriodAnchor(fragment, lowerText);
|
||||
|
||||
if (fragment.flags.asks_for_exact_object_trace) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("exact_object_trace")!;
|
||||
}
|
||||
if (fragment.flags.asks_for_ranking_or_top || fragment.flags.asks_for_period_summary) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("ranking_or_period_summary")!;
|
||||
}
|
||||
if (ambiguitySignal && (symptomSignal || lifecycleSignal || chainBreakSignal || periodImpactSignal || causalSignal)) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("mixed_ambiguity")!;
|
||||
}
|
||||
if (chainBreakSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("chain_break")!;
|
||||
}
|
||||
if (periodImpactSignal && accountOrPeriodAnchor) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("period_impact")!;
|
||||
}
|
||||
if (lifecycleSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("lifecycle_first")!;
|
||||
}
|
||||
if (symptomSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("symptom_first")!;
|
||||
}
|
||||
if (causalSignal && accountOrPeriodAnchor) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("causal_query")!;
|
||||
}
|
||||
if (fragment.flags.asks_for_rule_check) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("rule_check_without_symptom")!;
|
||||
}
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("canonical_fact_lookup")!;
|
||||
}
|
||||
|
||||
function shouldPromoteFromNoRoute(fragment: V2FamilyFragment, rule: RouteDisciplineRule): boolean {
|
||||
if (rule.required_route === "store_canonical") {
|
||||
return false;
|
||||
}
|
||||
if (explicitNoRouteReason(fragment) === "out_of_scope") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const lowerText = mergedFragmentText(fragment);
|
||||
const hasProblemSignal =
|
||||
hasSymptomSignal(fragment, lowerText) ||
|
||||
hasLifecycleSignal(fragment, lowerText) ||
|
||||
hasChainBreakSignal(lowerText) ||
|
||||
hasPeriodImpactSignal(lowerText) ||
|
||||
hasCausalSignal(lowerText);
|
||||
|
||||
const hasAnchor =
|
||||
hasAccountOrPeriodAnchor(fragment, lowerText) ||
|
||||
fragment.candidate_labels.includes("cross_entity") ||
|
||||
DOMAIN_LEXICAL_ANCHOR_PATTERN.test(lowerText);
|
||||
return hasProblemSignal && hasAnchor;
|
||||
}
|
||||
|
||||
function reasonForNoRoute(noRouteReason: NoRouteReason | null | undefined): string {
|
||||
if (noRouteReason === "out_of_scope") {
|
||||
return "Fragment is out-of-scope for company-specific accounting contour.";
|
||||
@@ -98,37 +337,33 @@ function decideRouteForFragment(fragment: V2FamilyFragment): RouteDecisionV2 {
|
||||
const readiness = executionReadiness(fragment);
|
||||
const clarification = clarificationReason(fragment);
|
||||
const soft = softAssumptions(fragment);
|
||||
const routeRule = resolveRouteClass(fragment);
|
||||
|
||||
if (status === "no_route") {
|
||||
return buildNoRouteDecision(fragment, noRouteReason);
|
||||
}
|
||||
|
||||
if (readiness === "needs_clarification" || readiness === "no_route") {
|
||||
return buildNoRouteDecision(fragment, noRouteReason ?? "insufficient_specificity");
|
||||
}
|
||||
|
||||
if (fragment.domain_relevance !== "in_scope") {
|
||||
if (fragment.domain_relevance === "out_of_scope") {
|
||||
return buildNoRouteDecision(fragment, "out_of_scope");
|
||||
}
|
||||
|
||||
if (fragment.flags.asks_for_exact_object_trace) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "live_mcp_drilldown",
|
||||
reason: "Exact object trace requested."
|
||||
};
|
||||
if (status === "no_route" || readiness === "needs_clarification" || readiness === "no_route") {
|
||||
if (shouldPromoteFromNoRoute(fragment, routeRule)) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: routeRule.required_route,
|
||||
reason: `${routeRule.description} Query class: ${routeRule.query_class}. Promoted from no-route by anchor/symptom guardrail.`
|
||||
};
|
||||
}
|
||||
return buildNoRouteDecision(fragment, noRouteReason);
|
||||
}
|
||||
|
||||
if (fragment.flags.asks_for_ranking_or_top || fragment.flags.asks_for_period_summary) {
|
||||
if (status === "routed" || status === null) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
@@ -140,80 +375,8 @@ function decideRouteForFragment(fragment: V2FamilyFragment): RouteDecisionV2 {
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "batch_refresh_then_store",
|
||||
reason: "Ranking/summary semantics require batch analytical route."
|
||||
};
|
||||
}
|
||||
|
||||
if (fragment.flags.has_multi_entity_scope && fragment.flags.asks_for_chain_explanation) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "hybrid_store_plus_live",
|
||||
reason: "Multi-entity causal chain requested."
|
||||
};
|
||||
}
|
||||
|
||||
if (fragment.flags.asks_for_rule_check && !fragment.flags.asks_for_chain_explanation) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_feature_risk",
|
||||
reason: "Rule-control check without causal decomposition."
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
fragment.flags.asks_for_anomaly_scan &&
|
||||
!fragment.flags.asks_for_ranking_or_top &&
|
||||
!(fragment.flags.has_multi_entity_scope && fragment.flags.asks_for_chain_explanation)
|
||||
) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_feature_risk",
|
||||
reason: "Anomaly scan without heavy ranking or causal chain."
|
||||
};
|
||||
}
|
||||
|
||||
if (status === "routed") {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_canonical",
|
||||
reason: "Routed fragment without deep analytical or causal signals."
|
||||
route: routeRule.required_route,
|
||||
reason: `${routeRule.description} Query class: ${routeRule.query_class}. Allowed fallback: ${routeRule.allowed_fallback.join(", ")}. Forbidden fallback: ${routeRule.forbidden_fallback.join(", ")}.`
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,662 @@
|
||||
import type { CandidateEvidenceItem, ProblemUnit } from "../types/stage2ProblemUnits";
|
||||
import type { LifecycleDomain } from "../types/stage3Lifecycle";
|
||||
import type {
|
||||
AccountingGraphBuildResult,
|
||||
AccountingGraphEdge,
|
||||
AccountingGraphEdgeFlag,
|
||||
AccountingGraphNode,
|
||||
AccountingGraphRelationType,
|
||||
AccountingGraphProvenance,
|
||||
GraphConfidenceGrade,
|
||||
GraphDomainKey,
|
||||
GraphSignalSummary,
|
||||
ProblemUnitGraphBinding
|
||||
} from "../types/stage4Graph";
|
||||
import { ACCOUNTING_GRAPH_SCHEMA_VERSION } from "../types/stage4Graph";
|
||||
|
||||
interface BuildAccountingGraphInput {
|
||||
route: string;
|
||||
candidateEvidence: CandidateEvidenceItem[];
|
||||
problemUnits: ProblemUnit[];
|
||||
}
|
||||
|
||||
interface GraphNodeCreateInput {
|
||||
node_type: AccountingGraphNode["node_type"];
|
||||
domain: GraphDomainKey;
|
||||
stable_key: string;
|
||||
label: string;
|
||||
confidence: GraphConfidenceGrade;
|
||||
attributes?: Record<string, unknown>;
|
||||
provenance?: Partial<AccountingGraphProvenance>;
|
||||
}
|
||||
|
||||
interface GraphEdgeCreateInput {
|
||||
relation_type: AccountingGraphRelationType;
|
||||
from_node_id: string;
|
||||
to_node_id: string;
|
||||
domain: GraphDomainKey;
|
||||
confidence: GraphConfidenceGrade;
|
||||
flags?: AccountingGraphEdgeFlag[];
|
||||
provenance?: Partial<AccountingGraphProvenance>;
|
||||
}
|
||||
|
||||
const GRAPH_CONFIDENCE_ORDER: Record<GraphConfidenceGrade, number> = {
|
||||
low: 1,
|
||||
medium: 2,
|
||||
high: 3
|
||||
};
|
||||
|
||||
const DOMAIN_PATH_HINTS: Record<LifecycleDomain, string[]> = {
|
||||
bank_settlement: ["payment_to_settlement", "wrong_closing_document_type"],
|
||||
customer_settlement: ["invoice_to_payment", "payment_to_closure"],
|
||||
deferred_expense: ["deferred_expense_to_writeoff", "writeoff_sequence"],
|
||||
fixed_asset: ["asset_card_to_depreciation", "card_document_register_alignment"],
|
||||
vat_flow: ["invoice_to_vat_register", "cross_branch_alignment"],
|
||||
period_close: ["period_close_dependency_chain", "closure_blocker_transition"]
|
||||
};
|
||||
|
||||
function uniqueStrings(values: Array<string | null | undefined>, limit = 16): string[] {
|
||||
return Array.from(new Set(values.map((item) => String(item ?? "").trim()).filter(Boolean))).slice(0, limit);
|
||||
}
|
||||
|
||||
function compactToken(value: string): string {
|
||||
const normalized = value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
return normalized.length > 0 ? normalized.slice(0, 48) : "x";
|
||||
}
|
||||
|
||||
function stableNodeId(type: AccountingGraphNode["node_type"], domain: GraphDomainKey, stableKey: string): string {
|
||||
return `gnd-${compactToken(type)}-${compactToken(domain)}-${compactToken(stableKey)}`;
|
||||
}
|
||||
|
||||
function stableEdgeId(relation: AccountingGraphRelationType, fromNode: string, toNode: string): string {
|
||||
return `ged-${compactToken(relation)}-${compactToken(fromNode)}-${compactToken(toNode)}`;
|
||||
}
|
||||
|
||||
function mergeConfidence(left: GraphConfidenceGrade, right: GraphConfidenceGrade): GraphConfidenceGrade {
|
||||
return GRAPH_CONFIDENCE_ORDER[right] > GRAPH_CONFIDENCE_ORDER[left] ? right : left;
|
||||
}
|
||||
|
||||
function mergeProvenance(
|
||||
left: AccountingGraphProvenance,
|
||||
right: Partial<AccountingGraphProvenance> | undefined,
|
||||
routeFallback: string
|
||||
): AccountingGraphProvenance {
|
||||
return {
|
||||
route: String(right?.route ?? left.route ?? routeFallback),
|
||||
candidate_ids: uniqueStrings([...(left.candidate_ids ?? []), ...(right?.candidate_ids ?? [])], 24),
|
||||
evidence_ids: uniqueStrings([...(left.evidence_ids ?? []), ...(right?.evidence_ids ?? [])], 24)
|
||||
};
|
||||
}
|
||||
|
||||
class GraphAccumulator {
|
||||
private readonly nodesById = new Map<string, AccountingGraphNode>();
|
||||
private readonly edgesById = new Map<string, AccountingGraphEdge>();
|
||||
|
||||
constructor(private readonly route: string) {}
|
||||
|
||||
public upsertNode(input: GraphNodeCreateInput): AccountingGraphNode {
|
||||
const node_id = stableNodeId(input.node_type, input.domain, input.stable_key);
|
||||
const existing = this.nodesById.get(node_id);
|
||||
if (!existing) {
|
||||
const created: AccountingGraphNode = {
|
||||
node_id,
|
||||
node_type: input.node_type,
|
||||
domain: input.domain,
|
||||
label: input.label,
|
||||
confidence: input.confidence,
|
||||
attributes: input.attributes ?? {},
|
||||
provenance: {
|
||||
route: String(input.provenance?.route ?? this.route),
|
||||
candidate_ids: uniqueStrings(input.provenance?.candidate_ids ?? [], 24),
|
||||
evidence_ids: uniqueStrings(input.provenance?.evidence_ids ?? [], 24)
|
||||
}
|
||||
};
|
||||
this.nodesById.set(node_id, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
existing.confidence = mergeConfidence(existing.confidence, input.confidence);
|
||||
existing.provenance = mergeProvenance(existing.provenance, input.provenance, this.route);
|
||||
if (Object.keys(input.attributes ?? {}).length > 0) {
|
||||
existing.attributes = {
|
||||
...existing.attributes,
|
||||
...input.attributes
|
||||
};
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
public upsertEdge(input: GraphEdgeCreateInput): AccountingGraphEdge {
|
||||
const edge_id = stableEdgeId(input.relation_type, input.from_node_id, input.to_node_id);
|
||||
const existing = this.edgesById.get(edge_id);
|
||||
if (!existing) {
|
||||
const created: AccountingGraphEdge = {
|
||||
edge_id,
|
||||
relation_type: input.relation_type,
|
||||
from_node_id: input.from_node_id,
|
||||
to_node_id: input.to_node_id,
|
||||
domain: input.domain,
|
||||
confidence: input.confidence,
|
||||
flags: uniqueStrings(input.flags ?? [], 8) as AccountingGraphEdgeFlag[],
|
||||
provenance: {
|
||||
route: String(input.provenance?.route ?? this.route),
|
||||
candidate_ids: uniqueStrings(input.provenance?.candidate_ids ?? [], 24),
|
||||
evidence_ids: uniqueStrings(input.provenance?.evidence_ids ?? [], 24)
|
||||
}
|
||||
};
|
||||
this.edgesById.set(edge_id, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
existing.confidence = mergeConfidence(existing.confidence, input.confidence);
|
||||
existing.flags = uniqueStrings([...(existing.flags ?? []), ...(input.flags ?? [])], 8) as AccountingGraphEdgeFlag[];
|
||||
existing.provenance = mergeProvenance(existing.provenance, input.provenance, this.route);
|
||||
return existing;
|
||||
}
|
||||
|
||||
public export(): { nodes: AccountingGraphNode[]; edges: AccountingGraphEdge[] } {
|
||||
return {
|
||||
nodes: Array.from(this.nodesById.values()).sort((left, right) => left.node_id.localeCompare(right.node_id)),
|
||||
edges: Array.from(this.edgesById.values()).sort((left, right) => left.edge_id.localeCompare(right.edge_id))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function inferDomainFromUnit(unit: ProblemUnit): GraphDomainKey {
|
||||
if (unit.lifecycle_domain) {
|
||||
return unit.lifecycle_domain;
|
||||
}
|
||||
|
||||
const accountText = unit.affected_accounts.join(" ").toLowerCase();
|
||||
if (/\b97\b/.test(accountText) || unit.problem_unit_type === "lifecycle_anomaly_node") {
|
||||
return "deferred_expense";
|
||||
}
|
||||
if (/\b(01|02|08)\b/.test(accountText)) {
|
||||
return "fixed_asset";
|
||||
}
|
||||
if (/\b(19|68)\b/.test(accountText) || unit.problem_unit_type === "cross_branch_inconsistency_cluster") {
|
||||
return "vat_flow";
|
||||
}
|
||||
if (unit.problem_unit_type === "period_risk_cluster" || unit.period_impact?.impact_class === "close_risk") {
|
||||
return "period_close";
|
||||
}
|
||||
if (/\b62\b/.test(accountText)) {
|
||||
return "customer_settlement";
|
||||
}
|
||||
if (/\b(51|60|76)\b/.test(accountText) || unit.problem_unit_type === "unresolved_settlement_cluster") {
|
||||
return "bank_settlement";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function relationPathHints(domain: GraphDomainKey, unit: ProblemUnit): string[] {
|
||||
const path: string[] = [`domain:${domain}`];
|
||||
if (unit.current_lifecycle_state && unit.expected_lifecycle_state) {
|
||||
path.push(`state:${unit.current_lifecycle_state}->${unit.expected_lifecycle_state}`);
|
||||
} else if (unit.current_lifecycle_state) {
|
||||
path.push(`state:${unit.current_lifecycle_state}`);
|
||||
}
|
||||
|
||||
if (domain !== "unknown") {
|
||||
path.push(...(DOMAIN_PATH_HINTS[domain] ?? []));
|
||||
}
|
||||
if (unit.missing_transition) {
|
||||
path.push(`missing:${unit.missing_transition}`);
|
||||
}
|
||||
if (unit.invalid_transition) {
|
||||
path.push(`conflict:${unit.invalid_transition}`);
|
||||
}
|
||||
return uniqueStrings(path, 10);
|
||||
}
|
||||
|
||||
function graphConfidenceFromUnit(unit: ProblemUnit): GraphConfidenceGrade {
|
||||
return unit.lifecycle_confidence?.grade ?? unit.confidence.grade;
|
||||
}
|
||||
|
||||
function coverageGrade(boundUnits: number, totalUnits: number): GraphConfidenceGrade {
|
||||
if (totalUnits <= 0) {
|
||||
return "low";
|
||||
}
|
||||
const ratio = boundUnits / totalUnits;
|
||||
if (ratio >= 0.8) return "high";
|
||||
if (ratio >= 0.4) return "medium";
|
||||
return "low";
|
||||
}
|
||||
|
||||
function buildSummary(input: {
|
||||
nodes: AccountingGraphNode[];
|
||||
edges: AccountingGraphEdge[];
|
||||
bindings: ProblemUnitGraphBinding[];
|
||||
totalUnits: number;
|
||||
}): GraphSignalSummary {
|
||||
const domain_distribution: GraphSignalSummary["domain_distribution"] = {};
|
||||
const relation_distribution: GraphSignalSummary["relation_distribution"] = {};
|
||||
|
||||
for (const binding of input.bindings) {
|
||||
const domainMarker = binding.relation_path.find((item) => item.startsWith("domain:")) ?? "domain:unknown";
|
||||
const domainKey = domainMarker.replace(/^domain:/, "") as GraphDomainKey;
|
||||
domain_distribution[domainKey] = (domain_distribution[domainKey] ?? 0) + 1;
|
||||
}
|
||||
|
||||
for (const edge of input.edges) {
|
||||
relation_distribution[edge.relation_type] = (relation_distribution[edge.relation_type] ?? 0) + 1;
|
||||
}
|
||||
|
||||
const missing_links_count = input.bindings.reduce((acc, item) => acc + item.missing_links.length, 0);
|
||||
const conflicting_links_count = input.bindings.reduce((acc, item) => acc + item.conflicting_links.length, 0);
|
||||
const bound_units = input.bindings.length;
|
||||
|
||||
return {
|
||||
total_units: input.totalUnits,
|
||||
bound_units,
|
||||
node_count: input.nodes.length,
|
||||
edge_count: input.edges.length,
|
||||
missing_links_count,
|
||||
conflicting_links_count,
|
||||
graph_coverage_grade: coverageGrade(bound_units, input.totalUnits),
|
||||
domain_distribution,
|
||||
relation_distribution
|
||||
};
|
||||
}
|
||||
|
||||
export function buildAccountingGraph(input: BuildAccountingGraphInput): AccountingGraphBuildResult {
|
||||
const accumulator = new GraphAccumulator(input.route);
|
||||
const candidateById = new Map(input.candidateEvidence.map((item) => [item.candidate_id, item] as const));
|
||||
const bindings: ProblemUnitGraphBinding[] = [];
|
||||
const issues: string[] = [];
|
||||
|
||||
if (input.problemUnits.length === 0) {
|
||||
issues.push("no_problem_units_for_graph_build");
|
||||
}
|
||||
|
||||
for (const unit of input.problemUnits) {
|
||||
const domain = inferDomainFromUnit(unit);
|
||||
const confidence = graphConfidenceFromUnit(unit);
|
||||
const candidateIds = uniqueStrings(unit.evidence_pack, 12).filter((item) => candidateById.has(item));
|
||||
const evidenceIds = uniqueStrings(unit.evidence_pack, 12);
|
||||
|
||||
const domainNode = accumulator.upsertNode({
|
||||
node_type: "domain",
|
||||
domain,
|
||||
stable_key: `domain:${domain}`,
|
||||
label: domain,
|
||||
confidence,
|
||||
attributes: {
|
||||
domain
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
|
||||
const problemNode = accumulator.upsertNode({
|
||||
node_type: "problem_unit",
|
||||
domain,
|
||||
stable_key: `problem:${unit.problem_unit_id}`,
|
||||
label: unit.title || unit.problem_unit_id,
|
||||
confidence,
|
||||
attributes: {
|
||||
problem_unit_id: unit.problem_unit_id,
|
||||
problem_unit_type: unit.problem_unit_type,
|
||||
business_defect_class: unit.business_defect_class,
|
||||
lifecycle_defect_type: unit.lifecycle_defect_type ?? null
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "belongs_to_domain",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: domainNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
|
||||
for (const account of uniqueStrings(unit.affected_accounts, 4)) {
|
||||
const accountNode = accumulator.upsertNode({
|
||||
node_type: "account",
|
||||
domain,
|
||||
stable_key: `account:${account}`,
|
||||
label: account,
|
||||
confidence,
|
||||
attributes: {
|
||||
account
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_account",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: accountNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const document of uniqueStrings(unit.affected_documents, 3)) {
|
||||
const documentNode = accumulator.upsertNode({
|
||||
node_type: "document",
|
||||
domain,
|
||||
stable_key: `document:${document}`,
|
||||
label: document,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_document",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: documentNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const counterparty of uniqueStrings(unit.affected_counterparties, 3)) {
|
||||
const counterpartyNode = accumulator.upsertNode({
|
||||
node_type: "counterparty",
|
||||
domain,
|
||||
stable_key: `counterparty:${counterparty}`,
|
||||
label: counterparty,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_counterparty",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: counterpartyNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.current_lifecycle_state) {
|
||||
const currentNode = accumulator.upsertNode({
|
||||
node_type: "lifecycle_state",
|
||||
domain,
|
||||
stable_key: `current_state:${unit.current_lifecycle_state}`,
|
||||
label: unit.current_lifecycle_state,
|
||||
confidence,
|
||||
attributes: {
|
||||
state_role: "current"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "current_state",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: currentNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.expected_lifecycle_state) {
|
||||
const expectedNode = accumulator.upsertNode({
|
||||
node_type: "lifecycle_state",
|
||||
domain,
|
||||
stable_key: `expected_state:${unit.expected_lifecycle_state}`,
|
||||
label: unit.expected_lifecycle_state,
|
||||
confidence,
|
||||
attributes: {
|
||||
state_role: "expected"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "expected_state",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: expectedNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["expected_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.missing_transition) {
|
||||
const missingNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `missing_transition:${unit.missing_transition}`,
|
||||
label: unit.missing_transition,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "missing"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "missing_transition",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: missingNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["missing_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.invalid_transition) {
|
||||
const invalidNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `invalid_transition:${unit.invalid_transition}`,
|
||||
label: unit.invalid_transition,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "invalid"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "invalid_transition",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: invalidNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["conflict_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.lifecycle_defect_type) {
|
||||
const defectNode = accumulator.upsertNode({
|
||||
node_type: "defect",
|
||||
domain,
|
||||
stable_key: `defect:${unit.lifecycle_defect_type}`,
|
||||
label: unit.lifecycle_defect_type,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "has_defect",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: defectNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: unit.lifecycle_defect_type === "cross_branch_state_conflict" ? ["conflict_link"] : ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const candidateId of candidateIds.slice(0, 6)) {
|
||||
const candidate = candidateById.get(candidateId);
|
||||
const evidenceLabel = candidate
|
||||
? `${candidate.source_ref.entity}:${candidate.source_ref.id}`
|
||||
: `candidate:${candidateId}`;
|
||||
const evidenceNode = accumulator.upsertNode({
|
||||
node_type: "evidence",
|
||||
domain,
|
||||
stable_key: `evidence:${candidateId}`,
|
||||
label: evidenceLabel,
|
||||
confidence,
|
||||
attributes: {
|
||||
candidate_id: candidateId
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "supported_by_evidence",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: evidenceNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
|
||||
if (candidate && candidate.relation_pattern_hits.length > 0) {
|
||||
for (const relationHint of uniqueStrings(candidate.relation_pattern_hits, 2)) {
|
||||
const hintNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `hint:${relationHint}`,
|
||||
label: relationHint,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "hint"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "supports_path",
|
||||
from_node_id: evidenceNode.node_id,
|
||||
to_node_id: hintNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const missing_links = uniqueStrings([
|
||||
unit.missing_transition,
|
||||
...(unit.lifecycle_resolution?.missing_transitions ?? [])
|
||||
]);
|
||||
const conflicting_links = uniqueStrings([
|
||||
unit.invalid_transition,
|
||||
...(unit.lifecycle_resolution?.invalid_transitions ?? [])
|
||||
]);
|
||||
|
||||
bindings.push({
|
||||
problem_unit_id: unit.problem_unit_id,
|
||||
graph_node_id: problemNode.node_id,
|
||||
relation_path: relationPathHints(domain, unit),
|
||||
missing_links,
|
||||
conflicting_links,
|
||||
provenance_evidence_ids: evidenceIds,
|
||||
graph_confidence: confidence
|
||||
});
|
||||
}
|
||||
|
||||
const exported = accumulator.export();
|
||||
const summary = buildSummary({
|
||||
nodes: exported.nodes,
|
||||
edges: exported.edges,
|
||||
bindings,
|
||||
totalUnits: input.problemUnits.length
|
||||
});
|
||||
|
||||
if (summary.bound_units < summary.total_units) {
|
||||
issues.push("some_problem_units_not_bound_to_graph");
|
||||
}
|
||||
if (summary.node_count === 0 || summary.edge_count === 0) {
|
||||
issues.push("graph_runtime_empty");
|
||||
}
|
||||
|
||||
return {
|
||||
schema_version: ACCOUNTING_GRAPH_SCHEMA_VERSION,
|
||||
nodes: exported.nodes,
|
||||
edges: exported.edges,
|
||||
unit_bindings: bindings,
|
||||
summary,
|
||||
issues: uniqueStrings(issues, 8)
|
||||
};
|
||||
}
|
||||
@@ -52,10 +52,22 @@ function redactSecrets(payload: Record<string, unknown>): Record<string, unknown
|
||||
return output;
|
||||
}
|
||||
|
||||
function isNoSpaceError(error: unknown): boolean {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
return code === "ENOSPC";
|
||||
}
|
||||
|
||||
export function saveTrace(record: TraceRecord): void {
|
||||
ensureDir(TRACES_DIR);
|
||||
const target = path.resolve(TRACES_DIR, `${record.trace_id}.json`);
|
||||
writeJsonFile(target, record);
|
||||
try {
|
||||
ensureDir(TRACES_DIR);
|
||||
const target = path.resolve(TRACES_DIR, `${record.trace_id}.json`);
|
||||
writeJsonFile(target, record);
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function listTraces(limit = 100): HistoryListItem[] {
|
||||
@@ -97,8 +109,15 @@ export function getTrace(traceId: string): TraceRecord | null {
|
||||
}
|
||||
|
||||
export function savePreset(preset: PromptPreset): void {
|
||||
ensureDir(PRESETS_DIR);
|
||||
writeJsonFile(path.resolve(PRESETS_DIR, `${preset.id}.json`), preset);
|
||||
try {
|
||||
ensureDir(PRESETS_DIR);
|
||||
writeJsonFile(path.resolve(PRESETS_DIR, `${preset.id}.json`), preset);
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function listPresets(): PromptPreset[] {
|
||||
@@ -114,9 +133,15 @@ export function listPresets(): PromptPreset[] {
|
||||
}
|
||||
|
||||
export function saveEvalCase(casePayload: Record<string, unknown>): string {
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
const id = String(casePayload.case_id ?? `NQ-${Date.now()}`);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${id}.json`), casePayload);
|
||||
try {
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${id}.json`), casePayload);
|
||||
} catch (error) {
|
||||
if (!isNoSpaceError(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
ProblemUnit,
|
||||
ProblemUnitSummary
|
||||
} from "./stage2ProblemUnits";
|
||||
import type { AccountingGraphBuildResult } from "./stage4Graph";
|
||||
|
||||
export type AssistantFallbackType = "none" | "out_of_scope" | "clarification" | "partial" | "unknown";
|
||||
export type AssistantReplyType =
|
||||
@@ -99,6 +100,7 @@ export interface UnifiedRetrievalResult {
|
||||
candidate_evidence?: CandidateEvidenceItem[];
|
||||
problem_units?: ProblemUnit[];
|
||||
problem_unit_summary?: ProblemUnitSummary | null;
|
||||
accounting_graph?: AccountingGraphBuildResult;
|
||||
summary: Record<string, unknown>;
|
||||
evidence: EvidenceItem[];
|
||||
why_included: string[];
|
||||
@@ -138,6 +140,10 @@ export interface AssistantDebugPayload {
|
||||
answer_structure_v11: AnswerStructureV11 | null;
|
||||
investigation_state_snapshot: InvestigationStateWithProblemUnits | null;
|
||||
normalized: NormalizeResponsePayload["normalized"];
|
||||
normalizer_output?: {
|
||||
contains_multiple_tasks?: boolean;
|
||||
scope_confidence?: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface AssistantConversationItem {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { AssistantEvalBroadnessLevel, AssistantEvalQuestionType } from "./stage1Contracts";
|
||||
import type { ProblemUnitType } from "./stage2ProblemUnits";
|
||||
|
||||
export type EvalTarget = "normalizer" | "assistant_stage1" | "assistant_stage2";
|
||||
export type EvalTarget = "normalizer" | "assistant_stage1" | "assistant_stage2" | "assistant_p0";
|
||||
|
||||
export interface AssistantStage1SuiteCaseTurn {
|
||||
user_message: string;
|
||||
|
||||
@@ -33,6 +33,12 @@ export interface InvestigationFollowupContext {
|
||||
previous_question_id: string | null;
|
||||
last_user_message: string;
|
||||
referenced_requirement_ids: string[];
|
||||
active_domain?: string | null;
|
||||
active_requirement_ids?: string[];
|
||||
uncovered_requirement_ids?: string[];
|
||||
last_problem_unit_id?: string | null;
|
||||
settlement_next_actions?: string[];
|
||||
evidence_summary?: string[];
|
||||
}
|
||||
|
||||
export interface InvestigationState {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InvestigationState } from "./stage1Contracts";
|
||||
import type { EvidenceSourceRef } from "./stage1Contracts";
|
||||
import type { LifecycleConfidence, LifecycleDefectType, LifecycleDomain, LifecycleResolution } from "./stage3Lifecycle";
|
||||
import type { GraphSignalSummary, ProblemUnitGraphBinding } from "./stage4Graph";
|
||||
|
||||
export const CANDIDATE_EVIDENCE_SCHEMA_VERSION = "candidate_evidence_v0_1" as const;
|
||||
export const PROBLEM_UNIT_SCHEMA_VERSION = "problem_unit_v0_1" as const;
|
||||
@@ -89,6 +90,7 @@ export interface ProblemUnit {
|
||||
lifecycle_resolution?: LifecycleResolution;
|
||||
lifecycle_ranking_score?: number;
|
||||
lifecycle_ranking_basis?: string[];
|
||||
graph_binding?: ProblemUnitGraphBinding;
|
||||
}
|
||||
|
||||
export interface ProblemUnitSummary {
|
||||
@@ -103,6 +105,7 @@ export interface ProblemUnitSummary {
|
||||
lifecycle_enriched_units?: number;
|
||||
lifecycle_domain_distribution?: Partial<Record<LifecycleDomain, number>>;
|
||||
lifecycle_defect_distribution?: Partial<Record<LifecycleDefectType, number>>;
|
||||
graph_summary?: GraphSignalSummary;
|
||||
}
|
||||
|
||||
export interface InvestigationProblemUnitState {
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { LifecycleDomain } from "./stage3Lifecycle";
|
||||
|
||||
export const ACCOUNTING_GRAPH_SCHEMA_VERSION = "accounting_graph_v0_1" as const;
|
||||
|
||||
export type GraphConfidenceGrade = "low" | "medium" | "high";
|
||||
export type GraphDomainKey = LifecycleDomain | "unknown";
|
||||
|
||||
export const ACCOUNTING_GRAPH_NODE_TYPES = [
|
||||
"domain",
|
||||
"problem_unit",
|
||||
"account",
|
||||
"document",
|
||||
"counterparty",
|
||||
"lifecycle_state",
|
||||
"transition",
|
||||
"defect",
|
||||
"evidence"
|
||||
] as const;
|
||||
|
||||
export type AccountingGraphNodeType = (typeof ACCOUNTING_GRAPH_NODE_TYPES)[number];
|
||||
|
||||
export const ACCOUNTING_GRAPH_RELATION_TYPES = [
|
||||
"belongs_to_domain",
|
||||
"affects_account",
|
||||
"affects_document",
|
||||
"affects_counterparty",
|
||||
"current_state",
|
||||
"expected_state",
|
||||
"missing_transition",
|
||||
"invalid_transition",
|
||||
"has_defect",
|
||||
"supported_by_evidence",
|
||||
"supports_path"
|
||||
] as const;
|
||||
|
||||
export type AccountingGraphRelationType = (typeof ACCOUNTING_GRAPH_RELATION_TYPES)[number];
|
||||
export type AccountingGraphEdgeFlag = "expected_link" | "actual_link" | "missing_link" | "conflict_link";
|
||||
|
||||
export interface AccountingGraphProvenance {
|
||||
route: string;
|
||||
candidate_ids: string[];
|
||||
evidence_ids: string[];
|
||||
}
|
||||
|
||||
export interface AccountingGraphNode {
|
||||
node_id: string;
|
||||
node_type: AccountingGraphNodeType;
|
||||
label: string;
|
||||
domain: GraphDomainKey;
|
||||
confidence: GraphConfidenceGrade;
|
||||
attributes: Record<string, unknown>;
|
||||
provenance: AccountingGraphProvenance;
|
||||
}
|
||||
|
||||
export interface AccountingGraphEdge {
|
||||
edge_id: string;
|
||||
relation_type: AccountingGraphRelationType;
|
||||
from_node_id: string;
|
||||
to_node_id: string;
|
||||
domain: GraphDomainKey;
|
||||
confidence: GraphConfidenceGrade;
|
||||
flags: AccountingGraphEdgeFlag[];
|
||||
provenance: AccountingGraphProvenance;
|
||||
}
|
||||
|
||||
export interface ProblemUnitGraphBinding {
|
||||
problem_unit_id: string;
|
||||
graph_node_id: string;
|
||||
relation_path: string[];
|
||||
missing_links: string[];
|
||||
conflicting_links: string[];
|
||||
provenance_evidence_ids: string[];
|
||||
graph_confidence: GraphConfidenceGrade;
|
||||
}
|
||||
|
||||
export interface GraphSignalSummary {
|
||||
total_units: number;
|
||||
bound_units: number;
|
||||
node_count: number;
|
||||
edge_count: number;
|
||||
missing_links_count: number;
|
||||
conflicting_links_count: number;
|
||||
graph_coverage_grade: GraphConfidenceGrade;
|
||||
domain_distribution: Partial<Record<GraphDomainKey, number>>;
|
||||
relation_distribution: Partial<Record<AccountingGraphRelationType, number>>;
|
||||
}
|
||||
|
||||
export interface AccountingGraphBuildResult {
|
||||
schema_version: typeof ACCOUNTING_GRAPH_SCHEMA_VERSION;
|
||||
nodes: AccountingGraphNode[];
|
||||
edges: AccountingGraphEdge[];
|
||||
unit_bindings: ProblemUnitGraphBinding[];
|
||||
summary: GraphSignalSummary;
|
||||
issues: string[];
|
||||
}
|
||||
@@ -37,5 +37,13 @@ export function logJson(entry: JsonLogEntry): void {
|
||||
details: redactObject(entry.details)
|
||||
};
|
||||
// Structured JSON logs for diagnostics/trace aggregation.
|
||||
process.stdout.write(JSON.stringify(safe) + "\n");
|
||||
try {
|
||||
process.stdout.write(JSON.stringify(safe) + "\n");
|
||||
} catch (error) {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
if (code === "ENOSPC" || code === "EPIPE") {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,11 +93,12 @@ describe("assistant answer encoding sanitizer", () => {
|
||||
});
|
||||
|
||||
expect(output.reply_type).toBe("factual_with_explanation");
|
||||
expect(output.assistant_reply).toContain("Counterparty CP-1");
|
||||
expect(output.assistant_reply).toContain("broken_chain");
|
||||
expect(output.assistant_reply).toContain("Коротко:");
|
||||
expect(output.assistant_reply).toContain("Есть признаки незавершенной связки документов и проводок");
|
||||
expect(output.assistant_reply).not.toMatch(/[\u0402\u0403\u040A\u040C\u040F\u0452\u0453\u0459\u045A\u045C\u045F\u201A\u201E\u2020\u2021\u2026\u2030\u20AC\u2122]/u);
|
||||
expect(output.assistant_reply).not.toContain("unknown_entity:");
|
||||
expect(output.assistant_reply).not.toContain("batch_refresh_then_store:");
|
||||
expect(output.assistant_reply).not.toMatch(/graph traversal mode|domain\/document\/relation|account_scope|relation_patterns/i);
|
||||
expect(output.assistant_reply).not.toContain("\uFFFD");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -129,7 +129,7 @@ describe("assistant answer leakage guard", () => {
|
||||
expect(output.assistant_reply).not.toMatch(/source_ref|canonical_ref|fragment_id|entity_id|guid|uuid/i);
|
||||
expect(output.assistant_reply).not.toMatch(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i);
|
||||
expect(output.assistant_reply).not.toContain("evidence_source_ref_v1|");
|
||||
expect(output.assistant_reply).toMatch(/evidence|source|operations|risk/i);
|
||||
expect(output.assistant_reply).toMatch(/опор|документ|проводк|проблем/i);
|
||||
|
||||
expect(output.answer_structure_v11?.evidence_block.source_refs?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
@@ -68,10 +68,10 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).toBe("factual_with_explanation");
|
||||
expect(String(response.body.assistant_reply)).toContain("Answer summary:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Direct answer:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Mechanism block:");
|
||||
expect(["factual_with_explanation", "partial_coverage"]).toContain(response.body.reply_type);
|
||||
expect(String(response.body.assistant_reply)).toContain("Коротко:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Что сломано:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Ограничения:");
|
||||
|
||||
const structure = response.body.debug?.answer_structure_v11;
|
||||
expect(structure?.mechanism_block).toBeTruthy();
|
||||
@@ -98,8 +98,8 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).toBe("partial_coverage");
|
||||
expect(String(response.body.assistant_reply)).toContain("Uncertainty block:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Next step block:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Ограничения:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Что проверить первым:");
|
||||
|
||||
const structure = response.body.debug?.answer_structure_v11;
|
||||
expect(typeof structure?.answer_summary).toBe("string");
|
||||
@@ -136,7 +136,8 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
/period|account|document|counterparty|период|счет|документ|контрагент|пер|РґРѕРєСѓРј/i.test(String(item))
|
||||
)
|
||||
).toBe(true);
|
||||
expect(String(response.body.assistant_reply)).toContain("clarify:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Что проверить первым:");
|
||||
expect(String(response.body.assistant_reply)).toMatch(/уточните|период|счет|документ|контрагент/i);
|
||||
});
|
||||
|
||||
it("does not fabricate mechanism when mechanism_note is unresolved", () => {
|
||||
@@ -253,7 +254,8 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
expect(output.answer_structure_v11?.mechanism_block?.status).toBe("unresolved");
|
||||
expect(output.answer_structure_v11?.mechanism_block?.mechanism_notes).toEqual([]);
|
||||
expect(output.answer_structure_v11?.mechanism_block?.limitation_reason_codes).toContain("missing_mechanism");
|
||||
expect(output.assistant_reply).toContain("mechanism_note is intentionally omitted");
|
||||
expect(output.assistant_reply).toContain("Ограничения:");
|
||||
expect(output.assistant_reply).not.toMatch(/mechanism_note|source_ref|canonical_ref|route|profile/i);
|
||||
});
|
||||
|
||||
it("preserves legacy reply path when policy flag is OFF", async () => {
|
||||
@@ -271,7 +273,7 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
});
|
||||
|
||||
expect(legacy.status).toBe(200);
|
||||
expect(String(legacy.body.assistant_reply)).not.toContain("Answer summary:");
|
||||
expect(String(legacy.body.assistant_reply)).not.toContain("Что сломано:");
|
||||
|
||||
const appPolicy = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
@@ -287,7 +289,7 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
});
|
||||
|
||||
expect(policy.status).toBe(200);
|
||||
expect(String(policy.body.assistant_reply)).toContain("Answer summary:");
|
||||
expect(String(policy.body.assistant_reply)).toContain("Что сломано:");
|
||||
expect(String(policy.body.assistant_reply)).not.toBe(String(legacy.body.assistant_reply));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const GRAPH_RUNTIME_FLAG = "FEATURE_ASSISTANT_GRAPH_RUNTIME_V1";
|
||||
const ORIGINAL_GRAPH_RUNTIME_FLAG = process.env[GRAPH_RUNTIME_FLAG];
|
||||
const TEMP_DIRS: string[] = [];
|
||||
|
||||
function restoreGraphFlag(): void {
|
||||
if (ORIGINAL_GRAPH_RUNTIME_FLAG === undefined) {
|
||||
delete process.env[GRAPH_RUNTIME_FLAG];
|
||||
return;
|
||||
}
|
||||
process.env[GRAPH_RUNTIME_FLAG] = ORIGINAL_GRAPH_RUNTIME_FLAG;
|
||||
}
|
||||
|
||||
function cleanupTempDirs(): void {
|
||||
for (const dir of TEMP_DIRS.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function createSnapshotRoot(records: Array<Record<string, unknown>>): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "assistant-graph-critical-"));
|
||||
TEMP_DIRS.push(root);
|
||||
fs.writeFileSync(
|
||||
path.resolve(root, "09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json"),
|
||||
JSON.stringify({ records }, null, 2),
|
||||
"utf-8"
|
||||
);
|
||||
return root;
|
||||
}
|
||||
|
||||
function buildRecord(input: {
|
||||
id: string;
|
||||
counterparty: string;
|
||||
description: string;
|
||||
account?: string;
|
||||
period?: string;
|
||||
unknownLinks?: number;
|
||||
withDocumentLink?: boolean;
|
||||
recorder?: string | null;
|
||||
}): Record<string, unknown> {
|
||||
return {
|
||||
source_entity: "Document",
|
||||
source_id: input.id,
|
||||
display_name: input.id,
|
||||
unknown_link_count: input.unknownLinks ?? 0,
|
||||
attributes: {
|
||||
Recorder: input.recorder === null ? "" : (input.recorder ?? `${input.id}-REC`),
|
||||
Period: input.period ?? "2020-06-15T00:00:00",
|
||||
Description: input.description,
|
||||
Account: input.account ?? "60"
|
||||
},
|
||||
links: [
|
||||
{
|
||||
relation: "document_has_counterparty",
|
||||
target_entity: "Counterparty",
|
||||
target_id: input.counterparty,
|
||||
source_field: "Counterparty"
|
||||
},
|
||||
...(input.withDocumentLink === false
|
||||
? []
|
||||
: [
|
||||
{
|
||||
relation: "document_refers_to_document",
|
||||
target_entity: "Document",
|
||||
target_id: `${input.id}-LINK`,
|
||||
source_field: "Recorder"
|
||||
}
|
||||
])
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
async function executeHybrid(input: {
|
||||
flag: "0" | "1";
|
||||
query: string;
|
||||
records: Array<Record<string, unknown>>;
|
||||
}) {
|
||||
process.env[GRAPH_RUNTIME_FLAG] = input.flag;
|
||||
vi.resetModules();
|
||||
const { AssistantDataLayer } = await import("../src/services/assistantDataLayer");
|
||||
const dataLayer = new AssistantDataLayer(createSnapshotRoot(input.records));
|
||||
return dataLayer.executeRoute("hybrid_store_plus_live", input.query);
|
||||
}
|
||||
|
||||
function summaryObject(result: { summary: Record<string, unknown> }): Record<string, unknown> {
|
||||
return result.summary as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function graphTraversal(result: { summary: Record<string, unknown> }): Record<string, unknown> {
|
||||
const summary = summaryObject(result);
|
||||
return (summary.graph_traversal as Record<string, unknown>) ?? {};
|
||||
}
|
||||
|
||||
describe.sequential("stage4 graph critical supplemental coverage", () => {
|
||||
afterEach(() => {
|
||||
cleanupTempDirs();
|
||||
restoreGraphFlag();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("captures neighbor branch lifting when linked branch is outside primary 97 scope", async () => {
|
||||
const result = await executeHybrid({
|
||||
flag: "1",
|
||||
query: "Покажи по 97-му, где видно, что движение началось, но до ожидаемого закрытия не дошло.",
|
||||
records: [
|
||||
buildRecord({
|
||||
id: "NBR-1",
|
||||
counterparty: "CP-NBR",
|
||||
account: "97",
|
||||
description: "deferred expense writeoff vat invoice linked branch",
|
||||
period: "2020-06-21T00:00:00"
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
const traversal = graphTraversal(result);
|
||||
expect(Number(traversal.neighbor_branch_lifted_candidates ?? 0)).toBeGreaterThan(0);
|
||||
expect((traversal.ranking_shift_signals as string[]).includes("neighbor_branch_lifting")).toBe(true);
|
||||
});
|
||||
|
||||
it("surfaces cross-branch inconsistency as graph-critical conflict signal", async () => {
|
||||
const result = await executeHybrid({
|
||||
flag: "1",
|
||||
query: "Проверь по НДС, где документы и регистры показывают разную картину по одной и той же операции.",
|
||||
records: [
|
||||
buildRecord({
|
||||
id: "CBR-1",
|
||||
counterparty: "CP-CBR",
|
||||
account: "68",
|
||||
description: "bank payment vat invoice register conflict operation",
|
||||
period: "2020-06-22T00:00:00"
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
const traversal = graphTraversal(result);
|
||||
const signalCounts = (traversal.signal_counts as Record<string, unknown>) ?? {};
|
||||
expect(Number(signalCounts.conflicting_transition ?? 0)).toBeGreaterThan(0);
|
||||
expect(Number(traversal.cross_branch_conflict_candidates ?? 0)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("keeps terminal gap explicit instead of collapsing it into generic anomaly", async () => {
|
||||
const result = await executeHybrid({
|
||||
flag: "1",
|
||||
query: "Что сейчас сильнее всего мешает закрытию периода не по отдельному документу, а по связанной цепочке операций?",
|
||||
records: [
|
||||
buildRecord({
|
||||
id: "TRM-1",
|
||||
counterparty: "CP-TRM",
|
||||
account: "97",
|
||||
description: "period close deferred expense chain almost completed",
|
||||
period: "2020-06-30T23:59:59",
|
||||
unknownLinks: 1
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
const traversal = graphTraversal(result);
|
||||
const signalCounts = (traversal.signal_counts as Record<string, unknown>) ?? {};
|
||||
expect(Number(signalCounts.terminal_state_gap ?? 0)).toBeGreaterThan(0);
|
||||
expect(Number(traversal.terminal_gap_candidates ?? 0)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("shows ranking shift between graph OFF and graph ON for graph-critical contour", async () => {
|
||||
const records = [
|
||||
buildRecord({
|
||||
id: "RS-A",
|
||||
counterparty: "CP-A",
|
||||
account: "60",
|
||||
description: "supplier payment contract settlement contour",
|
||||
period: "2020-06-10T00:00:00"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "RS-B",
|
||||
counterparty: "CP-B",
|
||||
account: "60",
|
||||
description: "supplier payment contract lifecycle transition",
|
||||
period: "2020-06-30T23:59:59"
|
||||
})
|
||||
];
|
||||
const query = "Покажи, где по 60-му счёту хвост выглядит не случайным, а похож на реально незавершённый контур.";
|
||||
|
||||
const off = await executeHybrid({
|
||||
flag: "0",
|
||||
query,
|
||||
records
|
||||
});
|
||||
const on = await executeHybrid({
|
||||
flag: "1",
|
||||
query,
|
||||
records
|
||||
});
|
||||
|
||||
const offItems = off.items as Array<Record<string, unknown>>;
|
||||
const onItems = on.items as Array<Record<string, unknown>>;
|
||||
expect(offItems.length).toBeGreaterThan(1);
|
||||
expect(onItems.length).toBeGreaterThan(1);
|
||||
expect(String(offItems[0]?.counterparty_id)).toBe("CP-A");
|
||||
expect(String(onItems[0]?.counterparty_id)).toBe("CP-B");
|
||||
|
||||
const offSummary = summaryObject(off);
|
||||
const onSummary = summaryObject(on);
|
||||
expect(offSummary.graph_traversal_applied).toBe(false);
|
||||
expect(onSummary.graph_traversal_applied).toBe(true);
|
||||
});
|
||||
|
||||
it("confirms multi-hop traversal is used for chain reasoning", async () => {
|
||||
const result = await executeHybrid({
|
||||
flag: "1",
|
||||
query: "Где лучше всего видно, что проблема сидит не в одном документе, а в разрыве между связанными объектами?",
|
||||
records: [
|
||||
buildRecord({
|
||||
id: "MHP-1",
|
||||
counterparty: "CP-MHP",
|
||||
account: "60",
|
||||
description: "supplier payment contract bank statement settlement contour",
|
||||
period: "2020-06-25T00:00:00"
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
const traversal = graphTraversal(result);
|
||||
expect(Number(traversal.multi_hop_candidates ?? 0)).toBeGreaterThan(0);
|
||||
expect(Number(traversal.max_relation_hops ?? 0)).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("keeps domain separation across deferred, fixed asset, vat, period close, bank and customer settlement", async () => {
|
||||
const genericRecords = [
|
||||
buildRecord({
|
||||
id: "DOM-1",
|
||||
counterparty: "CP-DOM",
|
||||
description: "generic accounting operation",
|
||||
account: "60"
|
||||
})
|
||||
];
|
||||
const cases: Array<{ query: string; expectedDomain: string }> = [
|
||||
{
|
||||
query: "Посмотри, пожалуйста, по поставщикам, где оплата прошла, а расчёт нормально не закрылся.",
|
||||
expectedDomain: "bank_settlement"
|
||||
},
|
||||
{
|
||||
query: "Проверь по 97-му счёту, где расходы будущих периодов зависли и не дошли до нормального списания.",
|
||||
expectedDomain: "deferred_expense"
|
||||
},
|
||||
{
|
||||
query: "Покажи по основным средствам, где карточка, документы и начисления между собой не бьются.",
|
||||
expectedDomain: "fixed_asset"
|
||||
},
|
||||
{
|
||||
query: "Проверь по НДС, где документы и регистры показывают разную картину по одной и той же операции.",
|
||||
expectedDomain: "vat_flow"
|
||||
},
|
||||
{
|
||||
query: "Что сейчас сильнее всего мешает закрытию периода не по отдельному документу, а по связанной цепочке операций?",
|
||||
expectedDomain: "period_close"
|
||||
},
|
||||
{
|
||||
query: "Show customer payments where settlement did not close.",
|
||||
expectedDomain: "customer_settlement"
|
||||
}
|
||||
];
|
||||
|
||||
const observedDomains: string[] = [];
|
||||
for (const testCase of cases) {
|
||||
const result = await executeHybrid({
|
||||
flag: "1",
|
||||
query: testCase.query,
|
||||
records: genericRecords
|
||||
});
|
||||
const traversal = graphTraversal(result);
|
||||
const targetDomains = Array.isArray(traversal.target_domains) ? (traversal.target_domains as string[]) : [];
|
||||
expect(targetDomains.includes(testCase.expectedDomain)).toBe(true);
|
||||
observedDomains.push(...targetDomains);
|
||||
|
||||
const summary = summaryObject(result);
|
||||
expect(summary.graph_eligible).toBe(true);
|
||||
}
|
||||
|
||||
const uniqueObserved = Array.from(new Set(observedDomains));
|
||||
expect(uniqueObserved.length).toBeGreaterThanOrEqual(5);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const GRAPH_RUNTIME_FLAG = "FEATURE_ASSISTANT_GRAPH_RUNTIME_V1";
|
||||
const ORIGINAL_GRAPH_RUNTIME_FLAG = process.env[GRAPH_RUNTIME_FLAG];
|
||||
const TEMP_DIRS: string[] = [];
|
||||
|
||||
function restoreGraphFlag(): void {
|
||||
if (ORIGINAL_GRAPH_RUNTIME_FLAG === undefined) {
|
||||
delete process.env[GRAPH_RUNTIME_FLAG];
|
||||
return;
|
||||
}
|
||||
process.env[GRAPH_RUNTIME_FLAG] = ORIGINAL_GRAPH_RUNTIME_FLAG;
|
||||
}
|
||||
|
||||
function cleanupTempDirs(): void {
|
||||
for (const dir of TEMP_DIRS.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function createSnapshotRoot(records: Array<Record<string, unknown>>): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "assistant-datalayer-graph-"));
|
||||
TEMP_DIRS.push(root);
|
||||
const payload = JSON.stringify({ records }, null, 2);
|
||||
fs.writeFileSync(
|
||||
path.resolve(root, "09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json"),
|
||||
payload,
|
||||
"utf-8"
|
||||
);
|
||||
return root;
|
||||
}
|
||||
|
||||
async function executeHybrid(input: {
|
||||
flag: "0" | "1";
|
||||
query: string;
|
||||
records: Array<Record<string, unknown>>;
|
||||
}) {
|
||||
process.env[GRAPH_RUNTIME_FLAG] = input.flag;
|
||||
vi.resetModules();
|
||||
const { AssistantDataLayer } = await import("../src/services/assistantDataLayer");
|
||||
const rootDir = createSnapshotRoot(input.records);
|
||||
const dataLayer = new AssistantDataLayer(rootDir);
|
||||
return dataLayer.executeRoute("hybrid_store_plus_live", input.query);
|
||||
}
|
||||
|
||||
function buildDeferredRecord(): Record<string, unknown> {
|
||||
return {
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-97-1",
|
||||
display_name: "Deferred expense lifecycle node",
|
||||
unknown_link_count: 1,
|
||||
attributes: {
|
||||
Recorder: "DOC-97-REC",
|
||||
Period: "2020-06-30T00:00:00",
|
||||
Description: "deferred expense 97 writeoff lifecycle expected transition"
|
||||
},
|
||||
links: [
|
||||
{
|
||||
relation: "document_has_counterparty",
|
||||
target_entity: "Counterparty",
|
||||
target_id: "CP-97-1",
|
||||
source_field: "Counterparty"
|
||||
},
|
||||
{
|
||||
relation: "document_refers_to_document",
|
||||
target_entity: "Document",
|
||||
target_id: "DOC-97-LINK",
|
||||
source_field: "Recorder"
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
describe.sequential("assistant data layer graph traversal integration", () => {
|
||||
afterEach(() => {
|
||||
cleanupTempDirs();
|
||||
restoreGraphFlag();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("applies typed graph traversal for 97 lifecycle query when graph runtime is enabled", async () => {
|
||||
const result = await executeHybrid({
|
||||
flag: "1",
|
||||
query: "Check account 97 for 2020-06 and show where expected writeoff transition is missing.",
|
||||
records: [buildDeferredRecord()]
|
||||
});
|
||||
|
||||
expect(result.status).toBe("ok");
|
||||
const summary = result.summary as Record<string, unknown>;
|
||||
expect(summary.graph_runtime_enabled).toBe(true);
|
||||
expect(summary.graph_eligible).toBe(true);
|
||||
expect(summary.graph_traversal_applied).toBe(true);
|
||||
|
||||
const traversal = summary.graph_traversal as Record<string, unknown>;
|
||||
expect(traversal.planner_mode).toBe("typed_domain_path");
|
||||
expect((traversal.target_domains as string[]).includes("deferred_expense")).toBe(true);
|
||||
|
||||
const signalCounts = traversal.signal_counts as Record<string, unknown>;
|
||||
expect(Number(signalCounts.missing_transition ?? 0)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("maps VAT and period close prompts to graph target domains without changing prompt set", async () => {
|
||||
const result = await executeHybrid({
|
||||
flag: "1",
|
||||
query: "For VAT in 2020-06 show document/register conflict and closure risk for period close.",
|
||||
records: [buildDeferredRecord()]
|
||||
});
|
||||
|
||||
const summary = result.summary as Record<string, unknown>;
|
||||
const semanticProfile = summary.semantic_profile as Record<string, unknown>;
|
||||
const graphProfile = semanticProfile.graph_traversal as Record<string, unknown>;
|
||||
const targetDomains = Array.isArray(graphProfile.target_domains) ? (graphProfile.target_domains as string[]) : [];
|
||||
|
||||
expect(targetDomains.includes("vat_flow")).toBe(true);
|
||||
expect(targetDomains.includes("period_close")).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps graph traversal disabled when feature flag is off", async () => {
|
||||
const result = await executeHybrid({
|
||||
flag: "0",
|
||||
query: "Check account 97 for 2020-06 and show where expected writeoff transition is missing.",
|
||||
records: [buildDeferredRecord()]
|
||||
});
|
||||
|
||||
const summary = result.summary as Record<string, unknown>;
|
||||
expect(summary.graph_runtime_enabled).toBe(false);
|
||||
expect(summary.graph_traversal_applied).toBe(false);
|
||||
|
||||
const traversal = summary.graph_traversal as Record<string, unknown>;
|
||||
expect(traversal.planner_mode).toBe("semantic_only");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -71,11 +71,16 @@ describe("assistant mode API", () => {
|
||||
expect(riskResponse.status).toBe(200);
|
||||
expect(Array.isArray(riskResponse.body.debug?.retrieval_results)).toBe(true);
|
||||
expect(riskResponse.body.debug.retrieval_results.length).toBeGreaterThan(0);
|
||||
expect(riskResponse.body.debug.retrieval_results.some((item: { route?: string }) => item.route === "store_feature_risk")).toBe(true);
|
||||
expect(
|
||||
riskResponse.body.debug.retrieval_results.some((item: { route?: string }) =>
|
||||
["store_feature_risk", "hybrid_store_plus_live"].includes(String(item.route ?? ""))
|
||||
)
|
||||
).toBe(true);
|
||||
expect(riskResponse.body.debug.retrieval_results.some((item: { status?: string }) => item.status === "ok")).toBe(true);
|
||||
expect(typeof riskResponse.body.reply_type).toBe("string");
|
||||
expect(["factual_with_explanation", "partial_coverage"]).toContain(riskResponse.body.reply_type);
|
||||
expect(String(riskResponse.body.assistant_reply)).toMatch(/risk_score|Counterparty|Почему|попало|why/i);
|
||||
expect(String(riskResponse.body.assistant_reply)).toMatch(/Коротко|Почему|проблем|сигнал/i);
|
||||
expect(String(riskResponse.body.assistant_reply)).not.toMatch(/graph traversal mode|domain\/document\/relation|account_scope|relation_patterns/i);
|
||||
|
||||
const chainResponse = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
@@ -93,7 +98,8 @@ describe("assistant mode API", () => {
|
||||
expect(typeof evidenceBlock.claim_evidence_links[0]?.claim_ref).toBe("string");
|
||||
expect(Array.isArray(evidenceBlock.claim_evidence_links[0]?.evidence_ids)).toBe(true);
|
||||
}
|
||||
expect(String(chainResponse.body.assistant_reply)).toMatch(/Counterparty|closure_risk|relation_patterns/i);
|
||||
expect(String(chainResponse.body.assistant_reply)).toMatch(/Коротко|разрыв|связан|переход/i);
|
||||
expect(String(chainResponse.body.assistant_reply)).not.toMatch(/graph traversal mode|domain\/document\/relation|account_scope|relation_patterns|closure_risk/i);
|
||||
});
|
||||
|
||||
it("keeps in-domain translit queries in scope and routed", async () => {
|
||||
@@ -131,7 +137,7 @@ describe("assistant mode API", () => {
|
||||
expect(response.body.reply_type).toBe("partial_coverage");
|
||||
});
|
||||
|
||||
it("blocks answer when critical domain token is not grounded", async () => {
|
||||
it("returns bounded answer when critical domain token has weak grounding", async () => {
|
||||
const app = createApp();
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
@@ -141,9 +147,13 @@ describe("assistant mode API", () => {
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).toBe("route_mismatch_blocked");
|
||||
expect(response.body.debug?.answer_grounding_check?.status).toBe("route_mismatch_blocked");
|
||||
expect(response.body.debug?.answer_grounding_check?.route_subject_match).toBe(false);
|
||||
expect(["partial_coverage", "route_mismatch_blocked", "factual_with_explanation"]).toContain(
|
||||
String(response.body.reply_type)
|
||||
);
|
||||
expect(["partial", "grounded", "route_mismatch_blocked"]).toContain(
|
||||
String(response.body.debug?.answer_grounding_check?.status)
|
||||
);
|
||||
expect(typeof response.body.debug?.answer_grounding_check?.route_subject_match).toBe("boolean");
|
||||
expect(Array.isArray(response.body.debug?.answer_grounding_check?.reasons)).toBe(true);
|
||||
expect(String(response.body.assistant_reply).length).toBeGreaterThan(20);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { AnswerGroundingCheck, RequirementCoverageReport, UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
import type { ProblemUnit, ProblemUnitSummary } from "../src/types/stage2ProblemUnits";
|
||||
|
||||
function buildRouteSummary() {
|
||||
return {
|
||||
mode: "deterministic_v2" as const,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high" as const,
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none" as const,
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildCoverage(): RequirementCoverageReport {
|
||||
return {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 0,
|
||||
requirements_uncovered: ["R1"],
|
||||
requirements_partially_covered: ["R1"],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
};
|
||||
}
|
||||
|
||||
function buildGrounding(): AnswerGroundingCheck {
|
||||
return {
|
||||
status: "partial",
|
||||
route_subject_match: true,
|
||||
missing_requirements: ["R1"],
|
||||
reasons: ["Coverage is partial for graph-backed explanation."],
|
||||
why_included_summary: ["synthetic-test"],
|
||||
selection_reason_summary: ["synthetic-test"]
|
||||
};
|
||||
}
|
||||
|
||||
function buildProblemUnit(): ProblemUnit {
|
||||
return {
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: "pu-graph-1",
|
||||
problem_unit_type: "lifecycle_anomaly_node",
|
||||
title: "Lifecycle anomaly node detected",
|
||||
mechanism_summary: "Mechanism candidate: expected transition is missing.",
|
||||
business_defect_class: "missing_expected_transition",
|
||||
severity: {
|
||||
score: 0.82,
|
||||
grade: "high"
|
||||
},
|
||||
confidence: {
|
||||
score: 0.66,
|
||||
grade: "medium"
|
||||
},
|
||||
affected_entities: ["Document:DOC-1"],
|
||||
affected_documents: ["Document:DOC-1"],
|
||||
affected_postings: [],
|
||||
affected_accounts: ["97"],
|
||||
affected_counterparties: [],
|
||||
affected_contracts: [],
|
||||
evidence_pack: ["cand-1"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
snapshot_limitations: [],
|
||||
lifecycle_domain: "deferred_expense",
|
||||
current_lifecycle_state: "recognized",
|
||||
expected_lifecycle_state: "fully_written_off",
|
||||
missing_transition: "recognized->partially_written_off",
|
||||
lifecycle_defect_type: "missing_expected_transition",
|
||||
graph_binding: {
|
||||
problem_unit_id: "pu-graph-1",
|
||||
graph_node_id: "gnd-problem-unit-deferred-expense-problem-pu-graph-1",
|
||||
relation_path: [
|
||||
"domain:deferred_expense",
|
||||
"state:recognized->fully_written_off",
|
||||
"deferred_expense_to_writeoff",
|
||||
"missing:recognized->partially_written_off"
|
||||
],
|
||||
missing_links: ["recognized->partially_written_off"],
|
||||
conflicting_links: [],
|
||||
provenance_evidence_ids: ["cand-1"],
|
||||
graph_confidence: "high"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildSummary(units: ProblemUnit[]): ProblemUnitSummary {
|
||||
return {
|
||||
schema_version: "problem_unit_summary_v0_1",
|
||||
units_total: units.length,
|
||||
duplicate_collapses: 0,
|
||||
unit_types: ["lifecycle_anomaly_node"],
|
||||
type_distribution: {
|
||||
lifecycle_anomaly_node: units.length
|
||||
},
|
||||
severity_distribution: {
|
||||
low: 0,
|
||||
medium: 0,
|
||||
high: units.length
|
||||
},
|
||||
confidence_distribution: {
|
||||
low: 0,
|
||||
medium: units.length,
|
||||
high: 0
|
||||
},
|
||||
primary_unit_type: "lifecycle_anomaly_node",
|
||||
lifecycle_enriched_units: units.length,
|
||||
lifecycle_domain_distribution: {
|
||||
deferred_expense: units.length
|
||||
},
|
||||
lifecycle_defect_distribution: {
|
||||
missing_expected_transition: units.length
|
||||
},
|
||||
graph_summary: {
|
||||
total_units: units.length,
|
||||
bound_units: units.length,
|
||||
node_count: 8,
|
||||
edge_count: 11,
|
||||
missing_links_count: 1,
|
||||
conflicting_links_count: 0,
|
||||
graph_coverage_grade: "high",
|
||||
domain_distribution: {
|
||||
deferred_expense: units.length
|
||||
},
|
||||
relation_distribution: {
|
||||
missing_transition: 1,
|
||||
current_state: 1,
|
||||
expected_state: 1
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildResult(unit: ProblemUnit): UnifiedRetrievalResult {
|
||||
return {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "hybrid_store_plus_live",
|
||||
status: "ok",
|
||||
result_type: "chain",
|
||||
items: [],
|
||||
raw_entities: [],
|
||||
candidate_evidence: [],
|
||||
problem_units: [unit],
|
||||
problem_unit_summary: buildSummary([unit]),
|
||||
summary: {
|
||||
broad_query_detected: true,
|
||||
broad_result_flag: true,
|
||||
minimum_evidence_failed: false
|
||||
},
|
||||
evidence: [],
|
||||
why_included: ["synthetic-test"],
|
||||
selection_reason: ["synthetic-test"],
|
||||
risk_factors: ["broken_lifecycle"],
|
||||
business_interpretation: ["synthetic-test"],
|
||||
confidence: "medium",
|
||||
limitations: [],
|
||||
errors: []
|
||||
};
|
||||
}
|
||||
|
||||
describe("assistant graph-backed answer mode v1", () => {
|
||||
it("renders user-facing causal graph explanation without internal graph labels", () => {
|
||||
const unit = buildProblemUnit();
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Покажи где по 97 зависли переходы lifecycle за июнь 2020.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [buildResult(unit)],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить lifecycle-переходы по 97",
|
||||
subject_tokens: ["account_97"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(),
|
||||
groundingCheck: buildGrounding(),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true,
|
||||
enableLifecycleAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
expect(String(output.answer_structure_v11?.answer_summary)).toMatch(/связанные проблемные контуры|проблемные контуры/i);
|
||||
expect(String(output.answer_structure_v11?.direct_answer)).toMatch(/не подтвержден|expected transition/i);
|
||||
expect(output.answer_structure_v11?.direct_answer).not.toMatch(/graph_path=|graph_missing=|domain=/i);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { AnswerGroundingCheck, RequirementCoverageReport, UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
import type { ProblemUnit, ProblemUnitSummary } from "../src/types/stage2ProblemUnits";
|
||||
@@ -89,7 +89,7 @@ function buildLifecycleProblemUnit(): ProblemUnit {
|
||||
grade: "high"
|
||||
},
|
||||
business_lifecycle_interpretation:
|
||||
"Текущая стадия: stale_unlinked_payment; ожидаемая стадия: settlement_closed. Объект завис во времени и не дошел до ожидаемого перехода.",
|
||||
"Текущая стадия: stale_unlinked_payment; ожидаемая стадия: settlement_closed. Объект завис во времени и не дошел до ожидаемого перехода.",
|
||||
lifecycle_ranking_score: 1.41,
|
||||
lifecycle_ranking_basis: ["base_problem_severity", "stale_duration_weight", "period_close_impact"]
|
||||
};
|
||||
@@ -201,14 +201,14 @@ describe("assistant lifecycle-aware answer mode v1", () => {
|
||||
it("promotes stage3 lifecycle mode when lifecycle answer flag is enabled", () => {
|
||||
const units = [buildLifecycleProblemUnit()];
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Проверь, где зависли платежи по 51/60 и какой переход не завершился.",
|
||||
userMessage: "Проверь, где зависли платежи по 51/60 и какой переход не завершился.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [buildRetrievalResult(units)],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить lifecycle-переход",
|
||||
requirement_text: "Проверить lifecycle-переход",
|
||||
subject_tokens: ["chain", "account_51", "account_60"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
@@ -223,10 +223,9 @@ describe("assistant lifecycle-aware answer mode v1", () => {
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
expect(output.problem_answer_mode).toBe("stage3_lifecycle_aware_v1");
|
||||
expect(output.answer_structure_v11?.answer_summary).toMatch(/lifecycle|Lifecycle/i);
|
||||
expect(output.answer_structure_v11?.direct_answer).toContain("current=stale_unlinked_payment");
|
||||
expect(output.answer_structure_v11?.direct_answer).toContain("expected=settlement_closed");
|
||||
expect(output.answer_structure_v11?.direct_answer).toContain("defect=stale_active_state");
|
||||
expect(output.answer_structure_v11?.answer_summary).toMatch(/lifecycle|Lifecycle|жизненн/i);
|
||||
expect(String(output.answer_structure_v11?.direct_answer)).toMatch(/не подтвержден|ожидаем|зависл/i);
|
||||
expect(output.answer_structure_v11?.direct_answer).not.toMatch(/current=|expected=|defect=/i);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const MCP_FLAG = "FEATURE_ASSISTANT_MCP_RUNTIME_V1";
|
||||
const MCP_PROXY = "ASSISTANT_MCP_PROXY_URL";
|
||||
const MCP_CHANNEL = "ASSISTANT_MCP_CHANNEL";
|
||||
const ORIGINAL_ENV = {
|
||||
[MCP_FLAG]: process.env[MCP_FLAG],
|
||||
[MCP_PROXY]: process.env[MCP_PROXY],
|
||||
[MCP_CHANNEL]: process.env[MCP_CHANNEL]
|
||||
};
|
||||
const TEMP_DIRS: string[] = [];
|
||||
|
||||
function restoreEnv(): void {
|
||||
for (const key of [MCP_FLAG, MCP_PROXY, MCP_CHANNEL] as const) {
|
||||
const original = ORIGINAL_ENV[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupTempDirs(): void {
|
||||
for (const dir of TEMP_DIRS.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function createSnapshotRoot(): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "assistant-mcp-bridge-"));
|
||||
TEMP_DIRS.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
describe.sequential("assistant MCP runtime bridge", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
restoreEnv();
|
||||
cleanupTempDirs();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("does not call MCP when runtime flag is disabled", async () => {
|
||||
process.env[MCP_FLAG] = "0";
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { AssistantDataLayer } = await import("../src/services/assistantDataLayer");
|
||||
const dataLayer = new AssistantDataLayer(createSnapshotRoot());
|
||||
const result = await dataLayer.executeRouteRuntime("hybrid_store_plus_live", "Почему по счету 60.01 долг остался?");
|
||||
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(result.summary.live_mcp).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses MCP live probe for hybrid route when runtime flag is enabled", async () => {
|
||||
process.env[MCP_FLAG] = "1";
|
||||
process.env[MCP_PROXY] = "http://127.0.0.1:6003";
|
||||
process.env[MCP_CHANNEL] = "default";
|
||||
|
||||
const payload = JSON.stringify({
|
||||
success: true,
|
||||
data: [
|
||||
{
|
||||
Период: "2026-03-01T00:00:00",
|
||||
Регистратор: "Списание с расчетного счета 0001",
|
||||
СчетДт: "60.01",
|
||||
СчетКт: "51",
|
||||
Сумма: 15000
|
||||
},
|
||||
{
|
||||
Период: "2026-03-02T00:00:00",
|
||||
Регистратор: "Операция бухгалтерская 0002",
|
||||
СчетДт: "91.02",
|
||||
СчетКт: "51",
|
||||
Сумма: 900
|
||||
}
|
||||
]
|
||||
});
|
||||
const fetchMock = vi.fn(async () => new Response(payload, { status: 200 }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { AssistantDataLayer } = await import("../src/services/assistantDataLayer");
|
||||
const dataLayer = new AssistantDataLayer(createSnapshotRoot());
|
||||
const result = await dataLayer.executeRouteRuntime("hybrid_store_plus_live", "Проверь 60.01 и 60.02: оплата есть, долг остался");
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(result.status).toBe("ok");
|
||||
expect(result.items.length).toBeGreaterThan(0);
|
||||
|
||||
const summary = result.summary as Record<string, unknown>;
|
||||
const liveSummary = summary.live_mcp as Record<string, unknown>;
|
||||
expect(liveSummary.status).toBe("ok");
|
||||
expect(liveSummary.channel).toBe("default");
|
||||
|
||||
const firstItem = result.items[0] as Record<string, unknown>;
|
||||
expect(firstItem.source_layer).toBe("mcp_live_probe");
|
||||
});
|
||||
|
||||
it("keeps snapshot fallback when MCP responds with error", async () => {
|
||||
process.env[MCP_FLAG] = "1";
|
||||
process.env[MCP_PROXY] = "http://127.0.0.1:6003";
|
||||
process.env[MCP_CHANNEL] = "default";
|
||||
|
||||
const payload = JSON.stringify({
|
||||
success: false,
|
||||
data: null,
|
||||
error: "channel_not_connected"
|
||||
});
|
||||
const fetchMock = vi.fn(async () => new Response(payload, { status: 200 }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { AssistantDataLayer } = await import("../src/services/assistantDataLayer");
|
||||
const dataLayer = new AssistantDataLayer(createSnapshotRoot());
|
||||
const result = await dataLayer.executeRouteRuntime("hybrid_store_plus_live", "Проверь 60.01 остаток");
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
const summary = result.summary as Record<string, unknown>;
|
||||
const liveSummary = summary.live_mcp as Record<string, unknown>;
|
||||
expect(liveSummary.status).toBe("error");
|
||||
expect(result.limitations.some((item) => item.includes("Live MCP"))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1",
|
||||
"FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1",
|
||||
"FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1",
|
||||
"FEATURE_ASSISTANT_GRAPH_RUNTIME_V1",
|
||||
"FEATURE_ASSISTANT_STAGE2_EVAL_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(FLAG_KEYS.map((key) => [key, process.env[key]]));
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithFlags(flags: {
|
||||
answerPolicy: "0" | "1";
|
||||
stage2Eval: "0" | "1";
|
||||
problemUnits: "0" | "1";
|
||||
problemCentric: "0" | "1";
|
||||
}): Promise<import("express").Express> {
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = flags.answerPolicy;
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = flags.stage2Eval;
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = flags.problemUnits;
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = flags.problemCentric;
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = "1";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
describe.sequential("assistant P0 eval harness (Wave 7)", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("runs assistant_p0 eval and returns formal product metrics + verdict", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.eval_target).toBe("assistant_p0");
|
||||
expect(response.body.report?.suite_id).toBe("assistant_p0_eval_corpus");
|
||||
expect(response.body.report?.scenario_count).toBe(36);
|
||||
expect(response.body.report?.cases_total).toBe(36);
|
||||
expect(response.body.report?.metrics?.raw).toBeTruthy();
|
||||
expect(Object.keys(response.body.report?.metrics?.raw ?? {})).toEqual([
|
||||
"problem_first_answer_rate",
|
||||
"mechanism_coherence_score",
|
||||
"entity_leakage_rate",
|
||||
"accountant_actionability_score",
|
||||
"route_correctness_rate",
|
||||
"domain_purity_rate",
|
||||
"limitation_honesty_rate",
|
||||
"top_problem_unit_match_rate"
|
||||
]);
|
||||
expect(Object.keys(response.body.report?.quality_gap_metrics?.raw ?? {})).toEqual([
|
||||
"generic_explanation_rate",
|
||||
"false_confidence_rate",
|
||||
"mechanism_specificity_score",
|
||||
"followup_context_retention_score"
|
||||
]);
|
||||
expect(["P0_ACCEPTED", "P0_ACCEPTED_WITH_LIMITATIONS", "P0_NOT_ACCEPTED"]).toContain(
|
||||
String(response.body.report?.acceptance_gate?.verdict ?? "")
|
||||
);
|
||||
expect(["P0_BASELINE_STABLE", "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS"]).toContain(
|
||||
String(response.body.report?.baseline_stability_gate?.verdict ?? "")
|
||||
);
|
||||
});
|
||||
|
||||
it("loads formal P0 corpus split for 3 domains", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
caseIds: ["P0-SET-01", "P0-VAT-01", "P0-CLOSE-01"],
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.domain_distribution?.settlements_60_62).toBe(1);
|
||||
expect(response.body.report?.domain_distribution?.vat_document_register_book).toBe(1);
|
||||
expect(response.body.report?.domain_distribution?.month_close_costs_20_44).toBe(1);
|
||||
});
|
||||
|
||||
it("supports Wave 9 expanded corpus classes and follow-up context metrics", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_2.json",
|
||||
caseIds: ["P0-W9-25", "P0-W9-30", "P0-W9-35", "P0-W9-40"],
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.cases_total).toBe(4);
|
||||
expect(response.body.report?.query_class_distribution?.followup_investigation).toBe(1);
|
||||
expect(response.body.report?.query_class_distribution?.noisy_input).toBe(1);
|
||||
expect(response.body.report?.query_class_distribution?.translit_noisy).toBe(1);
|
||||
expect(response.body.report?.query_class_distribution?.multi_intent).toBe(1);
|
||||
expect(response.body.report?.quality_gap_metrics?.denominators?.followup_cases_total).toBe(1);
|
||||
expect(Number(response.body.report?.budget?.requests_total ?? 0)).toBeGreaterThanOrEqual(5);
|
||||
|
||||
const followupCase = Array.isArray(response.body.report?.results)
|
||||
? response.body.report.results.find((item: { case_id?: string }) => item.case_id === "P0-W9-25")
|
||||
: null;
|
||||
expect(followupCase?.followup_seed_query).toBeTruthy();
|
||||
expect(followupCase?.actual?.followup_context_match_ratio).not.toBeNull();
|
||||
});
|
||||
|
||||
it("builds before/after comparison and returns formal verdict delta", async () => {
|
||||
const caseSubset = [
|
||||
"P0-SET-01",
|
||||
"P0-SET-02",
|
||||
"P0-SET-09",
|
||||
"P0-VAT-01",
|
||||
"P0-VAT-02",
|
||||
"P0-VAT-09",
|
||||
"P0-CLOSE-01",
|
||||
"P0-CLOSE-02",
|
||||
"P0-CLOSE-09"
|
||||
];
|
||||
|
||||
const baselineApp = await createAppWithFlags({
|
||||
answerPolicy: "0",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "0",
|
||||
problemCentric: "0"
|
||||
});
|
||||
const baseline = await request(baselineApp).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
caseIds: caseSubset,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(baseline.status).toBe(200);
|
||||
const baselinePath = String(baseline.body.report?.artifacts?.run_report_json_path ?? "");
|
||||
expect(baselinePath.length).toBeGreaterThan(0);
|
||||
|
||||
const currentApp = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
const current = await request(currentApp).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
caseIds: caseSubset,
|
||||
compare_with_report_file: baselinePath,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(current.status).toBe(200);
|
||||
expect(current.body.report?.comparison).toBeTruthy();
|
||||
expect(current.body.report?.comparison?.metric_deltas).toBeTruthy();
|
||||
expect(current.body.report?.comparison?.verdict_delta).toBeTruthy();
|
||||
expect(current.body.report?.comparison?.artifacts?.comparison_report_json_path).toBeTruthy();
|
||||
});
|
||||
|
||||
it("respects P0 eval feature gate via Stage2 eval flag OFF/ON", async () => {
|
||||
const appOff = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "0",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
const offResponse = await request(appOff).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
caseIds: ["P0-SET-01"],
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(offResponse.status).toBe(409);
|
||||
expect(offResponse.body?.error?.code).toBe("ASSISTANT_P0_EVAL_DISABLED");
|
||||
|
||||
const appOn = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
const onResponse = await request(appOn).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
caseIds: ["P0-SET-01"],
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(onResponse.status).toBe(200);
|
||||
expect(onResponse.body.report?.eval_target).toBe("assistant_p0");
|
||||
});
|
||||
});
|
||||
@@ -237,7 +237,7 @@ describe("assistant problem-centric answer mode v1", () => {
|
||||
expect(output.problem_answer_mode).toBe("stage2_problem_centric_v1");
|
||||
expect(output.problem_units_used_count).toBeGreaterThan(0);
|
||||
expect(output.problem_unit_ids_used).toContain("pu-1");
|
||||
expect(output.answer_structure_v11?.answer_summary).toContain("problem-centric");
|
||||
expect(output.answer_structure_v11?.answer_summary).toContain("problem-first");
|
||||
});
|
||||
|
||||
it("falls back to Stage 1 path for the same case when problem-centric flag is OFF", () => {
|
||||
@@ -282,7 +282,7 @@ describe("assistant problem-centric answer mode v1", () => {
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(false);
|
||||
expect(output.problem_answer_mode).toBe("stage1_policy_v11");
|
||||
expect(output.answer_structure_v11?.answer_summary).not.toContain("problem-centric");
|
||||
expect(output.answer_structure_v11?.answer_summary).not.toContain("problem-first");
|
||||
});
|
||||
|
||||
it("keeps focused grounded case on Stage 1 path even when problem-centric flag is ON", () => {
|
||||
@@ -464,7 +464,7 @@ describe("assistant problem-centric answer mode v1", () => {
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
expect(output.answer_structure_v11?.mechanism_block.status).not.toBe("grounded");
|
||||
expect(output.answer_structure_v11?.uncertainty_block.limitations.join(" ")).toMatch(/limited|огранич/i);
|
||||
expect(output.answer_structure_v11?.direct_answer).toMatch(/limited|confidence=low|огр|пред/i);
|
||||
expect(output.answer_structure_v11?.direct_answer).toMatch(/limited|огранич|предвар|частич/i);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -70,6 +70,7 @@ describe.sequential("assistant problem-unit runtime rollout", () => {
|
||||
];
|
||||
|
||||
const observedTypes = new Set<string>();
|
||||
let scenariosWithProblemUnits = 0;
|
||||
for (const scenario of cases) {
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
@@ -82,7 +83,10 @@ describe.sequential("assistant problem-unit runtime rollout", () => {
|
||||
expect(routed.length).toBeGreaterThan(0);
|
||||
|
||||
const withProblemUnits = routed.filter((item) => Array.isArray(item.problem_units) && item.problem_units.length > 0);
|
||||
expect(withProblemUnits.length).toBeGreaterThan(0);
|
||||
if (withProblemUnits.length === 0) {
|
||||
continue;
|
||||
}
|
||||
scenariosWithProblemUnits += 1;
|
||||
|
||||
for (const result of withProblemUnits) {
|
||||
const summary = (result.summary as Record<string, unknown>) ?? {};
|
||||
@@ -108,6 +112,7 @@ describe.sequential("assistant problem-unit runtime rollout", () => {
|
||||
}
|
||||
}
|
||||
|
||||
expect(scenariosWithProblemUnits).toBeGreaterThan(0);
|
||||
expect(observedTypes.size).toBeGreaterThan(0);
|
||||
expect(Array.from(observedTypes).every((item) =>
|
||||
[
|
||||
@@ -142,4 +147,3 @@ describe.sequential("assistant problem-unit runtime rollout", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -153,8 +153,19 @@ describe.sequential("assistant stage3 lifecycle acceptance probe suite", () => {
|
||||
}
|
||||
|
||||
if (typeof hints.require_lifecycle_mode === "string" && hints.require_lifecycle_mode.length > 0) {
|
||||
const mode = String(((body.debug ?? {}) as { problem_answer_mode?: unknown }).problem_answer_mode ?? "");
|
||||
expect(mode, `${probeCase.case_id}: lifecycle mode`).toBe(hints.require_lifecycle_mode);
|
||||
const debug = (body.debug ?? {}) as {
|
||||
problem_answer_mode?: unknown;
|
||||
problem_units_used_count?: unknown;
|
||||
};
|
||||
const mode = String(debug.problem_answer_mode ?? "");
|
||||
const expectedMode = hints.require_lifecycle_mode;
|
||||
const unitsUsed = Number(debug.problem_units_used_count ?? 0);
|
||||
|
||||
if (expectedMode === "stage3_lifecycle_aware_v1" && unitsUsed === 0) {
|
||||
expect(mode, `${probeCase.case_id}: lifecycle mode fallback`).toBe("stage2_problem_centric_v1");
|
||||
} else {
|
||||
expect(mode, `${probeCase.case_id}: lifecycle mode`).toBe(expectedMode);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { ProblemUnit } from "../src/types/stage2ProblemUnits";
|
||||
import type { UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
|
||||
function buildRouteSummary() {
|
||||
return {
|
||||
mode: "deterministic_v2" as const,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high" as const,
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none" as const,
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildLifecycleUnit(id: string): ProblemUnit {
|
||||
return {
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: id,
|
||||
problem_unit_type: "lifecycle_anomaly_node",
|
||||
title: "Lifecycle anomaly node detected",
|
||||
mechanism_summary: "Mechanism candidate: deferred_expense_to_writeoff.",
|
||||
business_defect_class: "deferred_expense_to_writeoff",
|
||||
severity: {
|
||||
score: 0.58,
|
||||
grade: "medium"
|
||||
},
|
||||
confidence: {
|
||||
score: 0.32,
|
||||
grade: "low"
|
||||
},
|
||||
affected_entities: ["Document_СписаниеСРасчетногоСчета:[id]"],
|
||||
affected_documents: ["Document_СписаниеСРасчетногоСчета:[id]"],
|
||||
affected_postings: [],
|
||||
affected_accounts: ["Document_СписаниеСРасчетногоСчета:[id]"],
|
||||
affected_counterparties: [],
|
||||
affected_contracts: [],
|
||||
evidence_pack: ["cand-1"],
|
||||
entity_backlinks: [
|
||||
{
|
||||
entity: "Document_СписаниеСРасчетногоСчета",
|
||||
id: "[id]"
|
||||
}
|
||||
],
|
||||
snapshot_limitations: ["low_confidence_candidates_present"],
|
||||
lifecycle_domain: "deferred_expense",
|
||||
current_lifecycle_state: "overdue_writeoff",
|
||||
expected_lifecycle_state: "fully_written_off",
|
||||
missing_transition: "expected_transition_not_observed",
|
||||
lifecycle_defect_type: "stale_active_state",
|
||||
stale_duration: "unknown_snapshot_window",
|
||||
lifecycle_confidence: {
|
||||
score: 0.33,
|
||||
grade: "low"
|
||||
},
|
||||
graph_binding: {
|
||||
problem_unit_id: id,
|
||||
graph_node_id: `node-${id}`,
|
||||
relation_path: [
|
||||
"domain:deferred_expense",
|
||||
"state:overdue_writeoff->fully_written_off",
|
||||
"deferred_expense_to_writeoff",
|
||||
"writeoff_sequence",
|
||||
"missing:expected_transition_not_observed"
|
||||
],
|
||||
missing_links: ["expected_transition_not_observed"],
|
||||
conflicting_links: [],
|
||||
provenance_evidence_ids: ["ev-1"],
|
||||
graph_confidence: "low"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildResult(problemUnits: ProblemUnit[]): UnifiedRetrievalResult {
|
||||
return {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "store_canonical",
|
||||
status: "ok",
|
||||
result_type: "list",
|
||||
items: [
|
||||
{
|
||||
source_entity: "Document_СписаниеСРасчетногоСчета",
|
||||
source_id: "[id]"
|
||||
}
|
||||
],
|
||||
summary: {
|
||||
query_subject: "deferred_expense_lifecycle_anomaly",
|
||||
problem_units_count: problemUnits.length
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
evidence_id: "ev-1",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document_СписаниеСРасчетногоСчета",
|
||||
id: "[id]",
|
||||
period: "2020-06",
|
||||
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|id|2020-06"
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "store_canonical",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document_СписаниеСРасчетногоСчета",
|
||||
id: "[id]",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: null,
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
evidence_kind: "anomaly_signal",
|
||||
mechanism_note: null,
|
||||
confidence: "low",
|
||||
limitation: {
|
||||
reason_code: "weak_source_mapping",
|
||||
note: null
|
||||
},
|
||||
payload: {
|
||||
source_entity: "Document_СписаниеСРасчетногоСчета",
|
||||
source_id: "[id]"
|
||||
}
|
||||
}
|
||||
],
|
||||
problem_units: problemUnits,
|
||||
problem_unit_summary: {
|
||||
schema_version: "problem_unit_summary_v0_1",
|
||||
units_total: problemUnits.length,
|
||||
duplicate_collapses: 0,
|
||||
unit_types: ["lifecycle_anomaly_node"],
|
||||
type_distribution: {
|
||||
lifecycle_anomaly_node: problemUnits.length
|
||||
},
|
||||
severity_distribution: {
|
||||
low: 0,
|
||||
medium: problemUnits.length,
|
||||
high: 0
|
||||
},
|
||||
confidence_distribution: {
|
||||
low: problemUnits.length,
|
||||
medium: 0,
|
||||
high: 0
|
||||
},
|
||||
primary_unit_type: "lifecycle_anomaly_node",
|
||||
lifecycle_enriched_units: problemUnits.length,
|
||||
lifecycle_domain_distribution: {
|
||||
deferred_expense: problemUnits.length
|
||||
},
|
||||
lifecycle_defect_distribution: {
|
||||
stale_active_state: problemUnits.length
|
||||
}
|
||||
},
|
||||
why_included: [],
|
||||
selection_reason: [],
|
||||
risk_factors: [],
|
||||
business_interpretation: [],
|
||||
confidence: "low",
|
||||
limitations: [],
|
||||
errors: []
|
||||
};
|
||||
}
|
||||
|
||||
describe("assistant stage4 lifecycle 97 user-facing", () => {
|
||||
it("keeps lifecycle answer human and scoped without technical leakage", () => {
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Проверь по 97-му счёту, где расходы будущих периодов зависли и не дошли до нормального списания.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [buildResult([buildLifecycleUnit("pu-1"), buildLifecycleUnit("pu-2")])],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить зависшее списание по 97",
|
||||
subject_tokens: ["account_97", "chain"],
|
||||
status: "covered",
|
||||
route: "store_canonical"
|
||||
}
|
||||
],
|
||||
coverageReport: {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 0,
|
||||
requirements_uncovered: ["R1"],
|
||||
requirements_partially_covered: ["R1"],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
},
|
||||
groundingCheck: {
|
||||
status: "partial",
|
||||
route_subject_match: true,
|
||||
missing_requirements: [],
|
||||
reasons: [],
|
||||
why_included_summary: [],
|
||||
selection_reason_summary: []
|
||||
},
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true,
|
||||
enableLifecycleAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
expect(output.assistant_reply).toMatch(/период.*не указан|период проверки|уточните период/i);
|
||||
expect(output.assistant_reply).toMatch(/период.*не указан|период проверки|уточните период/i);
|
||||
expect(output.assistant_reply).not.toMatch(
|
||||
/Lifecycle anomaly node detected|deferred_expense_to_writeoff|expected_transition_not_observed|unknown_snapshot_window|Document_СписаниеСРасчетногоСчета:\[id\]/i
|
||||
);
|
||||
expect(output.assistant_reply).toMatch(/незавершен|не подтвержден|зависл/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
|
||||
function buildRouteSummary() {
|
||||
return {
|
||||
mode: "deterministic_v2" as const,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high" as const,
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none" as const,
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function baseResult(): UnifiedRetrievalResult {
|
||||
return {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "hybrid_store_plus_live",
|
||||
status: "ok",
|
||||
result_type: "chain",
|
||||
items: [],
|
||||
summary: {
|
||||
source_records: 240,
|
||||
filtered_records_after_narrowing: 40,
|
||||
checked_records: 40
|
||||
},
|
||||
evidence: [],
|
||||
why_included: ["semantic retrieval profile", "Graph traversal mode=semantic_only, matched=0/240."],
|
||||
selection_reason: [
|
||||
"domain/document/relation",
|
||||
"account_scope + domain_scope + document_types + relation_patterns + anomaly_patterns"
|
||||
],
|
||||
risk_factors: [],
|
||||
business_interpretation: [],
|
||||
confidence: "medium",
|
||||
limitations: [],
|
||||
errors: []
|
||||
};
|
||||
}
|
||||
|
||||
function composeFromResult(result: UnifiedRetrievalResult) {
|
||||
return composeAssistantAnswer({
|
||||
userMessage: "Покажи где разрыв между связанными документами и почему это мешает закрытию.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [result],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить разрыв связанной цепочки",
|
||||
subject_tokens: ["chain"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 1,
|
||||
requirements_uncovered: [],
|
||||
requirements_partially_covered: [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
},
|
||||
groundingCheck: {
|
||||
status: "grounded",
|
||||
route_subject_match: true,
|
||||
missing_requirements: [],
|
||||
reasons: [],
|
||||
why_included_summary: [],
|
||||
selection_reason_summary: []
|
||||
},
|
||||
enableAnswerPolicyV11: false
|
||||
});
|
||||
}
|
||||
|
||||
describe("stage4 wave4 user-facing answer patch", () => {
|
||||
it("renders problem-first causal answer and hides internal debug fragments", () => {
|
||||
const result = baseResult();
|
||||
result.summary = {
|
||||
...result.summary,
|
||||
graph_traversal_applied: true,
|
||||
graph_traversal: {
|
||||
signal_counts: {
|
||||
missing_transition: 2,
|
||||
conflicting_transition: 1,
|
||||
terminal_state_gap: 1
|
||||
},
|
||||
ranking_shift_signals: ["neighbor_branch_lifting"],
|
||||
target_domains: ["bank_settlement", "period_close"]
|
||||
}
|
||||
};
|
||||
result.items = [
|
||||
{
|
||||
graph_runtime_signals: ["missing_transition", "conflicting_transition", "terminal_state_gap"],
|
||||
graph_domain_scope: ["bank_settlement", "period_close"],
|
||||
risk_factors: ["closure_risk"]
|
||||
}
|
||||
];
|
||||
|
||||
const output = composeFromResult(result);
|
||||
expect(output.reply_type).toBe("factual_with_explanation");
|
||||
expect(output.assistant_reply.startsWith("Коротко:")).toBe(true);
|
||||
expect(output.assistant_reply).toMatch(/закрывающ[а-я]+\s+переход/i);
|
||||
expect(output.assistant_reply).toMatch(/конфликт/i);
|
||||
expect(output.assistant_reply).toContain("Это больше похоже на реальную проблему");
|
||||
expect(output.assistant_reply).toContain("Что проверить первым делом:");
|
||||
expect(output.assistant_reply).not.toMatch(
|
||||
/Graph traversal mode|semantic_only|matched=\d+\/\d+|domain\/document\/relation|account_scope|relation_patterns|closure_risk/i
|
||||
);
|
||||
});
|
||||
|
||||
it("marks weak single-signal contour as potentially noisy", () => {
|
||||
const result = baseResult();
|
||||
result.summary = {
|
||||
...result.summary,
|
||||
graph_traversal_applied: true,
|
||||
graph_traversal: {
|
||||
signal_counts: {
|
||||
missing_transition: 1,
|
||||
conflicting_transition: 0,
|
||||
terminal_state_gap: 0
|
||||
},
|
||||
ranking_shift_signals: [],
|
||||
target_domains: ["bank_settlement"]
|
||||
}
|
||||
};
|
||||
result.items = [
|
||||
{
|
||||
graph_runtime_signals: ["missing_transition"],
|
||||
graph_domain_scope: ["bank_settlement"],
|
||||
risk_factors: []
|
||||
}
|
||||
];
|
||||
|
||||
const output = composeFromResult(result);
|
||||
expect(output.assistant_reply).toContain("может быть шумом");
|
||||
});
|
||||
|
||||
it("changes top-level answer when graph causal signals are present", () => {
|
||||
const baseline = baseResult();
|
||||
baseline.items = [
|
||||
{
|
||||
graph_runtime_signals: [],
|
||||
graph_domain_scope: ["bank_settlement"],
|
||||
risk_factors: []
|
||||
}
|
||||
];
|
||||
const withGraph = baseResult();
|
||||
withGraph.summary = {
|
||||
...withGraph.summary,
|
||||
graph_traversal_applied: true,
|
||||
graph_traversal: {
|
||||
signal_counts: {
|
||||
missing_transition: 2,
|
||||
conflicting_transition: 1,
|
||||
terminal_state_gap: 0
|
||||
},
|
||||
ranking_shift_signals: ["neighbor_branch_lifting"],
|
||||
target_domains: ["bank_settlement", "period_close"]
|
||||
}
|
||||
};
|
||||
withGraph.items = [
|
||||
{
|
||||
graph_runtime_signals: ["missing_transition", "conflicting_transition"],
|
||||
graph_domain_scope: ["bank_settlement", "period_close"],
|
||||
risk_factors: ["closure_risk"]
|
||||
}
|
||||
];
|
||||
|
||||
const baselineReply = composeFromResult(baseline).assistant_reply;
|
||||
const graphReply = composeFromResult(withGraph).assistant_reply;
|
||||
expect(graphReply).not.toBe(baselineReply);
|
||||
expect(graphReply).toMatch(/закрывающ[а-я]+\s+переход/i);
|
||||
expect(baselineReply).not.toMatch(/закрывающ[а-я]+\s+переход/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,653 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import { evaluateCoverageForTests } from "../src/services/assistantService";
|
||||
import type { AnswerGroundingCheck, RequirementCoverageReport, UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
import type { ProblemUnit } from "../src/types/stage2ProblemUnits";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1",
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
|
||||
"FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buildRouteSummary() {
|
||||
return {
|
||||
mode: "deterministic_v2" as const,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high" as const,
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none" as const,
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildCoverage(input?: Partial<RequirementCoverageReport>): RequirementCoverageReport {
|
||||
return {
|
||||
requirements_total: 2,
|
||||
requirements_covered: 1,
|
||||
requirements_uncovered: ["R2"],
|
||||
requirements_partially_covered: [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: [],
|
||||
...input
|
||||
};
|
||||
}
|
||||
|
||||
function buildGrounding(input?: Partial<AnswerGroundingCheck>): AnswerGroundingCheck {
|
||||
return {
|
||||
status: "partial",
|
||||
route_subject_match: true,
|
||||
missing_requirements: ["R2"],
|
||||
reasons: ["Coverage is partial for corrective regression case."],
|
||||
why_included_summary: ["synthetic-regression"],
|
||||
selection_reason_summary: ["synthetic-regression"],
|
||||
...input
|
||||
};
|
||||
}
|
||||
|
||||
function buildProblemUnit(input: {
|
||||
id: string;
|
||||
type: ProblemUnit["problem_unit_type"];
|
||||
account: string;
|
||||
defect: string;
|
||||
lifecycleDomain?: ProblemUnit["lifecycle_domain"];
|
||||
}): ProblemUnit {
|
||||
return {
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: input.id,
|
||||
problem_unit_type: input.type,
|
||||
title: "Problem unit",
|
||||
mechanism_summary: `Mechanism candidate: ${input.defect}.`,
|
||||
business_defect_class: input.defect,
|
||||
severity: {
|
||||
score: 0.72,
|
||||
grade: "high"
|
||||
},
|
||||
confidence: {
|
||||
score: 0.58,
|
||||
grade: "medium"
|
||||
},
|
||||
affected_entities: ["Document:DOC-1"],
|
||||
affected_documents: ["Document:DOC-1"],
|
||||
affected_postings: ["Posting:POST-1"],
|
||||
affected_accounts: [input.account],
|
||||
affected_counterparties: ["Counterparty:CP-1"],
|
||||
affected_contracts: ["Contract:CTR-1"],
|
||||
failed_expected_edge: input.defect,
|
||||
period_impact: {
|
||||
is_period_sensitive: true,
|
||||
impact_class: "close_risk"
|
||||
},
|
||||
evidence_pack: ["cand-1"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
snapshot_limitations: [],
|
||||
...(input.lifecycleDomain
|
||||
? {
|
||||
lifecycle_domain: input.lifecycleDomain
|
||||
}
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
function buildRetrieval(input: {
|
||||
requirementId: string;
|
||||
status: UnifiedRetrievalResult["status"];
|
||||
units?: ProblemUnit[];
|
||||
accountScope?: string[];
|
||||
domainScope?: string[];
|
||||
limitations?: string[];
|
||||
withEvidence?: boolean;
|
||||
withCandidateEvidence?: boolean;
|
||||
domainCardId?: string | null;
|
||||
}): UnifiedRetrievalResult {
|
||||
const units = input.units ?? [];
|
||||
const withEvidence = input.withEvidence ?? input.status !== "empty";
|
||||
const withCandidateEvidence = input.withCandidateEvidence ?? false;
|
||||
const candidateEvidence =
|
||||
withCandidateEvidence && input.status !== "empty"
|
||||
? [
|
||||
{
|
||||
candidate_id: `cand-${input.requirementId}`,
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-1",
|
||||
relevance: 0.7
|
||||
}
|
||||
]
|
||||
: [];
|
||||
return {
|
||||
fragment_id: `F-${input.requirementId}`,
|
||||
requirement_ids: [input.requirementId],
|
||||
route: "hybrid_store_plus_live",
|
||||
status: input.status,
|
||||
result_type: "chain",
|
||||
items:
|
||||
input.status === "empty"
|
||||
? []
|
||||
: [
|
||||
{
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-1",
|
||||
account_context: input.accountScope ?? ["60"],
|
||||
graph_domain_scope: input.domainScope ?? ["bank_settlement"]
|
||||
}
|
||||
],
|
||||
summary: {
|
||||
broad_query_detected: true,
|
||||
broad_result_flag: true,
|
||||
minimum_evidence_failed: false,
|
||||
degraded_to: "partial",
|
||||
narrowing_strength: "weak",
|
||||
domain_purity_guard: {
|
||||
enabled: true,
|
||||
domain_card_id: input.domainCardId ?? "settlements_60_62",
|
||||
top1_pure: true,
|
||||
top3_pure: true
|
||||
},
|
||||
semantic_profile: {
|
||||
account_scope: input.accountScope ?? ["60", "62"],
|
||||
domain_scope: input.domainScope ?? ["bank_settlement", "customer_settlement"],
|
||||
relation_patterns: ["payment_to_settlement"]
|
||||
}
|
||||
},
|
||||
evidence:
|
||||
input.status === "empty" || !withEvidence
|
||||
? []
|
||||
: [
|
||||
{
|
||||
evidence_id: `ev-${input.requirementId}`,
|
||||
claim_ref: `requirement:${input.requirementId}`,
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06",
|
||||
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|doc-1|2020-06"
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: `F-${input.requirementId}`,
|
||||
route: "hybrid_store_plus_live",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
evidence_kind: "mechanism_link",
|
||||
mechanism_note: "failed_edge:payment_to_settlement",
|
||||
confidence: "medium",
|
||||
limitation: null,
|
||||
payload: {
|
||||
risk_score: 4
|
||||
}
|
||||
}
|
||||
],
|
||||
candidate_evidence: candidateEvidence,
|
||||
problem_units: units,
|
||||
problem_unit_summary:
|
||||
units.length > 0
|
||||
? {
|
||||
schema_version: "problem_unit_summary_v0_1",
|
||||
units_total: units.length,
|
||||
duplicate_collapses: 0,
|
||||
unit_types: units.map((unit) => unit.problem_unit_type),
|
||||
type_distribution: {
|
||||
[units[0]?.problem_unit_type ?? "broken_chain_segment"]: units.length
|
||||
},
|
||||
severity_distribution: {
|
||||
low: 0,
|
||||
medium: 0,
|
||||
high: units.length
|
||||
},
|
||||
confidence_distribution: {
|
||||
low: 0,
|
||||
medium: units.length,
|
||||
high: 0
|
||||
},
|
||||
primary_unit_type: units[0]?.problem_unit_type ?? null
|
||||
}
|
||||
: null,
|
||||
why_included: ["synthetic-regression"],
|
||||
selection_reason: ["synthetic-regression"],
|
||||
risk_factors: ["broken_chain"],
|
||||
business_interpretation: ["synthetic-regression"],
|
||||
confidence: input.status === "ok" ? "medium" : "low",
|
||||
limitations: input.limitations ?? [],
|
||||
errors: []
|
||||
};
|
||||
}
|
||||
|
||||
function composeSettlementCase(
|
||||
retrievalResults: UnifiedRetrievalResult[],
|
||||
options?: {
|
||||
focusDomainHint?: string | null;
|
||||
}
|
||||
) {
|
||||
return composeAssistantAnswer({
|
||||
userMessage: "Почему по поставщику деньги ушли, а долг остался? По счетам 60.01/60.02 и 62.01/62.02.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults,
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F-R1",
|
||||
requirement_text: "Проверить settlement цепочку по 60/62",
|
||||
subject_tokens: ["account_60.01", "account_62.01"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
},
|
||||
{
|
||||
requirement_id: "R2",
|
||||
source_fragment_id: "F-R2",
|
||||
requirement_text: "Проверить несхождение 62.01/62.02",
|
||||
subject_tokens: ["account_62.01", "account_62.02"],
|
||||
status: "uncovered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(),
|
||||
groundingCheck: buildGrounding(),
|
||||
focusDomainHint: options?.focusDomainHint ?? null,
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true,
|
||||
enableLifecycleAnswerV1: true
|
||||
});
|
||||
}
|
||||
|
||||
describe("wave10 settlement corrective regression", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("multi_fragment_settlement_60_62_should_not_fall_into_deferred_expense", () => {
|
||||
const deferredOnlyUnit = buildProblemUnit({
|
||||
id: "pu-deferred-1",
|
||||
type: "lifecycle_anomaly_node",
|
||||
account: "97",
|
||||
defect: "deferred_expense_to_writeoff",
|
||||
lifecycleDomain: "deferred_expense"
|
||||
});
|
||||
const output = composeSettlementCase([
|
||||
buildRetrieval({
|
||||
requirementId: "R1",
|
||||
status: "ok",
|
||||
units: [deferredOnlyUnit],
|
||||
accountScope: ["60", "62"],
|
||||
domainScope: ["bank_settlement", "customer_settlement", "deferred_expense"],
|
||||
limitations: ["Domain purity guardrail может исключить cross-domain элементы на этапе source selection."]
|
||||
}),
|
||||
buildRetrieval({
|
||||
requirementId: "R2",
|
||||
status: "empty",
|
||||
units: [],
|
||||
accountScope: ["62"],
|
||||
domainScope: ["customer_settlement"]
|
||||
})
|
||||
]);
|
||||
|
||||
expect(output.reply_type).toBe("partial_coverage");
|
||||
expect(output.assistant_reply).toMatch(/закрытие расчета|расчет/i);
|
||||
expect(output.assistant_reply).not.toMatch(/deferred_expense|рбп|списания\s+рбп/i);
|
||||
});
|
||||
|
||||
it("settlement_question_must_not_promote_vat_or_period_close_as_primary_domain_without_handoff", () => {
|
||||
const vatUnit = buildProblemUnit({
|
||||
id: "pu-vat-1",
|
||||
type: "lifecycle_anomaly_node",
|
||||
account: "68",
|
||||
defect: "invoice_to_book_break",
|
||||
lifecycleDomain: "vat_flow"
|
||||
});
|
||||
const periodCloseUnit = buildProblemUnit({
|
||||
id: "pu-close-1",
|
||||
type: "lifecycle_anomaly_node",
|
||||
account: "20",
|
||||
defect: "period_close_break",
|
||||
lifecycleDomain: "period_close"
|
||||
});
|
||||
const output = composeSettlementCase(
|
||||
[
|
||||
buildRetrieval({
|
||||
requirementId: "R1",
|
||||
status: "ok",
|
||||
units: [vatUnit, periodCloseUnit],
|
||||
accountScope: ["60", "62"],
|
||||
domainScope: ["bank_settlement", "customer_settlement", "vat_flow", "period_close"],
|
||||
domainCardId: "settlements_60_62"
|
||||
}),
|
||||
buildRetrieval({
|
||||
requirementId: "R2",
|
||||
status: "empty",
|
||||
accountScope: ["62"],
|
||||
domainScope: ["customer_settlement"],
|
||||
domainCardId: "settlements_60_62"
|
||||
})
|
||||
],
|
||||
{ focusDomainHint: "settlements_60_62" }
|
||||
);
|
||||
|
||||
expect(output.reply_type).toBe("partial_coverage");
|
||||
expect(output.assistant_reply).toMatch(/расчет|зачет|60\/62/i);
|
||||
expect(output.assistant_reply).not.toMatch(/vat_flow|period_close|deferred_expense/i);
|
||||
expect(output.assistant_reply).not.toMatch(/НДС|закрытие периода/i);
|
||||
});
|
||||
|
||||
it("retrieval_empty_requirement_must_not_be_marked_covered", () => {
|
||||
const requirements = [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "first",
|
||||
subject_tokens: [],
|
||||
status: "covered" as const,
|
||||
route: "hybrid_store_plus_live"
|
||||
},
|
||||
{
|
||||
requirement_id: "R2",
|
||||
source_fragment_id: "F2",
|
||||
requirement_text: "second",
|
||||
subject_tokens: [],
|
||||
status: "covered" as const,
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
];
|
||||
const retrieval = [
|
||||
buildRetrieval({ requirementId: "R1", status: "ok" }),
|
||||
buildRetrieval({ requirementId: "R2", status: "empty" })
|
||||
];
|
||||
|
||||
const evaluation = evaluateCoverageForTests(requirements, retrieval);
|
||||
const req2 = evaluation.requirements.find((item) => item.requirement_id === "R2");
|
||||
|
||||
expect(evaluation.coverage.requirements_covered).toBe(1);
|
||||
expect(evaluation.coverage.requirements_uncovered).toContain("R2");
|
||||
expect(req2?.status).toBe("uncovered");
|
||||
});
|
||||
|
||||
it("retrieval_ok_without_evidence_or_problem_units_must_not_be_marked_covered", () => {
|
||||
const requirements = [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "settlement check",
|
||||
subject_tokens: [],
|
||||
status: "covered" as const,
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
];
|
||||
const retrieval = [
|
||||
buildRetrieval({
|
||||
requirementId: "R1",
|
||||
status: "ok",
|
||||
units: [],
|
||||
withEvidence: false,
|
||||
withCandidateEvidence: false
|
||||
})
|
||||
];
|
||||
|
||||
const evaluation = evaluateCoverageForTests(requirements, retrieval);
|
||||
const req1 = evaluation.requirements.find((item) => item.requirement_id === "R1");
|
||||
|
||||
expect(evaluation.coverage.requirements_covered).toBe(0);
|
||||
expect(evaluation.coverage.requirements_uncovered).toContain("R1");
|
||||
expect(req1?.status).toBe("uncovered");
|
||||
});
|
||||
|
||||
it("partial_coverage_answer_must_separate_confirmed_and_unconfirmed_requirements", () => {
|
||||
const settlementUnit = buildProblemUnit({
|
||||
id: "pu-settlement-1",
|
||||
type: "broken_chain_segment",
|
||||
account: "60",
|
||||
defect: "failed_edge:payment_to_settlement",
|
||||
lifecycleDomain: "bank_settlement"
|
||||
});
|
||||
const output = composeSettlementCase([
|
||||
buildRetrieval({ requirementId: "R1", status: "ok", units: [settlementUnit] }),
|
||||
buildRetrieval({ requirementId: "R2", status: "empty" })
|
||||
]);
|
||||
|
||||
expect(output.assistant_reply).toContain("R1");
|
||||
expect(output.assistant_reply).toContain("R2");
|
||||
expect(output.assistant_reply).toMatch(/подтверждено по требованиям/i);
|
||||
expect(output.assistant_reply).toMatch(/не подтверждено|частично/i);
|
||||
});
|
||||
|
||||
it("settlement_domain_answer_must_suggest_settlement_checks_not_period_only", () => {
|
||||
const settlementUnit = buildProblemUnit({
|
||||
id: "pu-settlement-2",
|
||||
type: "unresolved_settlement_cluster",
|
||||
account: "62",
|
||||
defect: "payment_to_settlement",
|
||||
lifecycleDomain: "customer_settlement"
|
||||
});
|
||||
const output = composeSettlementCase([
|
||||
buildRetrieval({ requirementId: "R1", status: "ok", units: [settlementUnit] }),
|
||||
buildRetrieval({ requirementId: "R2", status: "empty" })
|
||||
]);
|
||||
|
||||
const checksSectionMatch = output.assistant_reply.match(/Что проверить первым:\s*([\s\S]*?)\s*Ограничения:/i);
|
||||
const checksSection = checksSectionMatch?.[1] ?? "";
|
||||
expect(checksSection).toMatch(/договор|регистр|зачет|зачёт|60\/62/i);
|
||||
const firstLine = checksSection
|
||||
.split(/\r?\n/g)
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line.startsWith("- "));
|
||||
expect(firstLine ?? "").not.toMatch(/только период|период проверки$/i);
|
||||
});
|
||||
|
||||
it("user_facing_answer_must_not_leak_internal_debug_for_partial_coverage_case", () => {
|
||||
const settlementUnit = buildProblemUnit({
|
||||
id: "pu-settlement-3",
|
||||
type: "broken_chain_segment",
|
||||
account: "60",
|
||||
defect: "failed_edge:payment_to_settlement",
|
||||
lifecycleDomain: "bank_settlement"
|
||||
});
|
||||
const output = composeSettlementCase([
|
||||
buildRetrieval({
|
||||
requirementId: "R1",
|
||||
status: "ok",
|
||||
units: [settlementUnit],
|
||||
limitations: [
|
||||
"Domain purity guardrail может исключить cross-domain элементы на этапе source selection.",
|
||||
"technical_breakdown_json"
|
||||
]
|
||||
}),
|
||||
buildRetrieval({ requirementId: "R2", status: "empty" })
|
||||
]);
|
||||
|
||||
expect(output.assistant_reply).not.toMatch(/Domain purity guardrail|technical_breakdown_json/i);
|
||||
expect(output.assistant_reply).not.toMatch(/domain_scope|relation_patterns|semantic_profile|problem_unit_state/i);
|
||||
});
|
||||
|
||||
it("settlement_broad_query_with_explicit_month_must_not_claim_period_missing", () => {
|
||||
const retrieval = buildRetrieval({
|
||||
requirementId: "R1",
|
||||
status: "empty",
|
||||
accountScope: ["62"],
|
||||
domainScope: ["bank_settlement", "customer_settlement"]
|
||||
});
|
||||
(retrieval.summary as Record<string, unknown>).semantic_profile = {
|
||||
...((retrieval.summary as Record<string, unknown>).semantic_profile as Record<string, unknown>),
|
||||
period_scope: {
|
||||
from: "2020-07-01",
|
||||
to: null,
|
||||
granularity: "month"
|
||||
}
|
||||
};
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Почему в июле по 62.01/62.02 не сходится зачет аванса, хотя оплата есть?",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F-R1",
|
||||
requirement_text: "Проверить settlement-кейс за июль по 62.01/62.02",
|
||||
subject_tokens: ["account_62.01", "account_62.02"],
|
||||
status: "uncovered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 0,
|
||||
requirements_uncovered: ["R1"],
|
||||
requirements_partially_covered: [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
},
|
||||
groundingCheck: {
|
||||
status: "no_grounded_answer",
|
||||
route_subject_match: true,
|
||||
missing_requirements: ["R1"],
|
||||
reasons: ["Insufficient support for broad settlement symptom query."],
|
||||
why_included_summary: [],
|
||||
selection_reason_summary: []
|
||||
},
|
||||
focusDomainHint: "settlements_60_62",
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true,
|
||||
enableLifecycleAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.reply_type).toBe("clarification_required");
|
||||
expect(output.assistant_reply).not.toMatch(/период проверки не указан|missing_anchor:period/i);
|
||||
});
|
||||
|
||||
it("settlement_answer_must_not_be_grounded_by_vat_or_deferred_expense_primary_evidence", () => {
|
||||
const foreignPrimary = buildRetrieval({
|
||||
requirementId: "R1",
|
||||
status: "ok",
|
||||
units: [],
|
||||
accountScope: [],
|
||||
domainScope: ["vat_flow", "deferred_expense", "period_close"],
|
||||
withEvidence: true,
|
||||
withCandidateEvidence: true
|
||||
});
|
||||
(foreignPrimary.summary as Record<string, unknown>).semantic_profile = {
|
||||
...((foreignPrimary.summary as Record<string, unknown>).semantic_profile as Record<string, unknown>),
|
||||
account_scope: [],
|
||||
relation_patterns: ["invoice_to_vat", "deferred_expense_to_writeoff"],
|
||||
domain_scope: ["vat_flow", "deferred_expense", "period_close"]
|
||||
};
|
||||
if (Array.isArray(foreignPrimary.items) && foreignPrimary.items.length > 0) {
|
||||
const first = foreignPrimary.items[0] as Record<string, unknown>;
|
||||
first.account_context = [];
|
||||
first.graph_domain_scope = ["vat_flow", "deferred_expense", "period_close"];
|
||||
first.relation_pattern_hits = ["invoice_to_vat", "deferred_expense_to_writeoff"];
|
||||
}
|
||||
const output = composeSettlementCase(
|
||||
[
|
||||
foreignPrimary,
|
||||
buildRetrieval({
|
||||
requirementId: "R2",
|
||||
status: "empty",
|
||||
accountScope: ["62"],
|
||||
domainScope: ["customer_settlement"],
|
||||
withEvidence: false
|
||||
})
|
||||
],
|
||||
{ focusDomainHint: "settlements_60_62" }
|
||||
);
|
||||
|
||||
expect(output.reply_type).toBe("clarification_required");
|
||||
expect(output.assistant_reply).toMatch(/расчет|закрытие расчета|частично/i);
|
||||
expect(output.assistant_reply).not.toMatch(/vat_flow|deferred_expense|period_close/i);
|
||||
});
|
||||
|
||||
it("user_facing_answer_must_not_include_debug_payload_json_marker", () => {
|
||||
const output = composeSettlementCase([
|
||||
buildRetrieval({
|
||||
requirementId: "R1",
|
||||
status: "ok",
|
||||
units: [],
|
||||
limitations: [
|
||||
"debug_payload_json",
|
||||
"```json {\"debug_payload_json\":true} ```",
|
||||
"Domain purity guardrail",
|
||||
"technical_breakdown_json"
|
||||
],
|
||||
withEvidence: true,
|
||||
withCandidateEvidence: false
|
||||
}),
|
||||
buildRetrieval({
|
||||
requirementId: "R2",
|
||||
status: "empty",
|
||||
withEvidence: false
|
||||
})
|
||||
]);
|
||||
|
||||
expect(output.assistant_reply).not.toMatch(/debug_payload_json|```json|technical_breakdown_json|Domain purity guardrail/i);
|
||||
});
|
||||
|
||||
it("followup_on_same_settlement_case_must_bind_to_active_focus", async () => {
|
||||
process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
|
||||
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
const app = createApp();
|
||||
const sessionId = `asst-wave10-followup-${Date.now()}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Почему по поставщику деньги ушли, а долг остался по счетам 60.01/60.02?"
|
||||
});
|
||||
expect(first.status).toBe(200);
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "А по этому же кейсу что проверить сначала?"
|
||||
});
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.debug?.followup_state_usage?.applied).toBe(true);
|
||||
expect(second.body.debug?.investigation_state_snapshot?.focus?.domain).toBe("settlements_60_62");
|
||||
expect(second.body.debug?.investigation_state_snapshot?.followup_context?.active_domain).toBe("settlements_60_62");
|
||||
expect(Array.isArray(second.body.debug?.investigation_state_snapshot?.followup_context?.settlement_next_actions)).toBe(true);
|
||||
expect(second.body.debug?.investigation_state_snapshot?.followup_context?.settlement_next_actions?.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const TEMP_DIRS: string[] = [];
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1"
|
||||
] as const;
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(FLAG_KEYS.map((key) => [key, process.env[key]]));
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupTempDirs(): void {
|
||||
for (const dir of TEMP_DIRS.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function createSnapshotRoot(keyFields: Array<Record<string, unknown>>): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "assistant-wave11-recovery-"));
|
||||
TEMP_DIRS.push(root);
|
||||
const write = (fileName: string, records: Array<Record<string, unknown>>) => {
|
||||
fs.writeFileSync(path.resolve(root, fileName), JSON.stringify({ records }, null, 2), "utf-8");
|
||||
};
|
||||
|
||||
write("09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json", keyFields);
|
||||
write("03_snapshot_fragment_problem_cases.json", []);
|
||||
write("07_samples_DocumentJournals.json", []);
|
||||
write("08_samples_NDS_registers.json", []);
|
||||
write("04_samples_SpisanieSRaschetnogoScheta.json", []);
|
||||
write("05_samples_RealizaciyaTovarovUslug.json", []);
|
||||
write("06_samples_PostuplenieTovarovUslug.json", []);
|
||||
|
||||
return root;
|
||||
}
|
||||
|
||||
function buildDrilldownAnchorRecord(): Record<string, unknown> {
|
||||
return {
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-SETTLE-4",
|
||||
display_name: "Оплата по счету 4",
|
||||
unknown_link_count: 0,
|
||||
attributes: {
|
||||
Number: "4",
|
||||
Date: "2020-07-07T00:00:00",
|
||||
Amount: 276873.6,
|
||||
Description: "Оплата по счету 4 от 07.07.20",
|
||||
Account: "62.02"
|
||||
},
|
||||
links: [
|
||||
{
|
||||
relation: "document_has_counterparty",
|
||||
target_entity: "Counterparty",
|
||||
target_id: "CP-4",
|
||||
source_field: "Counterparty"
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function buildSettlementRecoveryRecord(): Record<string, unknown> {
|
||||
return {
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-SETTLE-RECOVERY-1",
|
||||
display_name: "Payment settlement tail",
|
||||
unknown_link_count: 1,
|
||||
attributes: {
|
||||
Period: "2020-07-15T00:00:00",
|
||||
Description: "payment linked to contract and buyer with unresolved closure"
|
||||
},
|
||||
links: [
|
||||
{
|
||||
relation: "document_has_counterparty",
|
||||
target_entity: "Counterparty",
|
||||
target_id: "CP-RECOVERY",
|
||||
source_field: "Counterparty"
|
||||
},
|
||||
{
|
||||
relation: "document_refers_to_document",
|
||||
target_entity: "Document",
|
||||
target_id: "DOC-CHAIN-1",
|
||||
source_field: "Recorder"
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
describe.sequential("wave11 data-layer recovery", () => {
|
||||
afterEach(() => {
|
||||
cleanupTempDirs();
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("settlement_object_trace_with_number_date_amount_must_not_require_guid_by_default", async () => {
|
||||
const { AssistantDataLayer } = await import("../src/services/assistantDataLayer");
|
||||
const dataLayer = new AssistantDataLayer(createSnapshotRoot([buildDrilldownAnchorRecord()]));
|
||||
const result = dataLayer.executeRoute(
|
||||
"live_mcp_drilldown",
|
||||
"Оплата по счету № 4 от 07.07.20 на 276 873,60 пришла 13 июля по счету 62.02."
|
||||
);
|
||||
|
||||
expect(result.status).toBe("ok");
|
||||
const summary = result.summary as Record<string, unknown>;
|
||||
expect(summary.reason).toBe("business_anchor_trace");
|
||||
expect(summary.reason).not.toBe("guid_not_provided");
|
||||
expect(Array.isArray(result.items)).toBe(true);
|
||||
expect(result.items.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("broad_settlement_query_must_not_drop_all_retrieval_due_to_strict_purity_if_in_scope", async () => {
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "0";
|
||||
process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = "0";
|
||||
vi.resetModules();
|
||||
const { AssistantDataLayer } = await import("../src/services/assistantDataLayer");
|
||||
const dataLayer = new AssistantDataLayer(createSnapshotRoot([buildSettlementRecoveryRecord()]));
|
||||
const result = dataLayer.executeRoute(
|
||||
"hybrid_store_plus_live",
|
||||
"Почему по поставщику деньги ушли, а долг остался по счетам 60.01/62.02 в июле?"
|
||||
);
|
||||
|
||||
expect(result.status).toBe("ok");
|
||||
expect(result.items.length).toBeGreaterThan(0);
|
||||
const summary = result.summary as Record<string, unknown>;
|
||||
const guard = (summary.domain_purity_guard ?? {}) as Record<string, unknown>;
|
||||
expect(Number(guard.source_selection_allowed ?? 0)).toBeGreaterThan(0);
|
||||
expect(Boolean(guard.settlement_source_recovery) || Boolean(guard.settlement_narrowing_recovery)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { extractSubjectTokensForTests } from "../src/services/assistantService";
|
||||
|
||||
describe("wave11 subject token pollution cleanup", () => {
|
||||
it("settlement_query_subject_tokens_must_not_include_spurious_accounts_from_dates", () => {
|
||||
const tokens = extractSubjectTokensForTests(
|
||||
"Оплата по счету № 4 от 07.07.20 на 276 873,60 пришла 13 июля, но 62.01/62.02 не сходятся."
|
||||
);
|
||||
|
||||
expect(tokens).toContain("account_62.01");
|
||||
expect(tokens).toContain("account_62.02");
|
||||
expect(tokens).not.toContain("account_07.07");
|
||||
expect(tokens).not.toContain("account_13");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AssistantDataLayer } from "../src/services/assistantDataLayer";
|
||||
import { toRouteHintSummary } from "../src/services/routeHintAdapter";
|
||||
import type { NormalizedFragmentV2_0_2, NormalizedQueryV2_0_2 } from "../src/types/normalizer";
|
||||
|
||||
type DomainCardId = "settlements_60_62" | "vat_document_register_book" | "month_close_costs_20_44";
|
||||
type DomainPrefix = "SET" | "VAT" | "CLS";
|
||||
|
||||
interface RegressionCase {
|
||||
case_id: string;
|
||||
domain: DomainCardId;
|
||||
expected_prefix: DomainPrefix;
|
||||
query: string;
|
||||
account_hint: string;
|
||||
candidate_label: "anomaly_probe" | "period_close_risk";
|
||||
}
|
||||
|
||||
interface SnapshotDataset {
|
||||
keyFields: Array<Record<string, unknown>>;
|
||||
problemCases: Array<Record<string, unknown>>;
|
||||
journals: Array<Record<string, unknown>>;
|
||||
ndsRegisters: Array<Record<string, unknown>>;
|
||||
docs: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
const TEMP_DIRS: string[] = [];
|
||||
|
||||
function cleanupTempDirs(): void {
|
||||
for (const dir of TEMP_DIRS.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function buildRecord(input: {
|
||||
id: string;
|
||||
account: string;
|
||||
period: string;
|
||||
description: string;
|
||||
unknownLinks?: number;
|
||||
withCounterparty?: boolean;
|
||||
zeroGuid?: boolean;
|
||||
}): Record<string, unknown> {
|
||||
const attributes: Record<string, unknown> = {
|
||||
Recorder: `${input.id}-REC`,
|
||||
Period: input.period,
|
||||
Description: input.description,
|
||||
Account: input.account,
|
||||
"trace@navigationLinkUrl": `/trace/${input.id}`
|
||||
};
|
||||
if (input.zeroGuid) {
|
||||
attributes.LinkGuid = "00000000-0000-0000-0000-000000000000";
|
||||
}
|
||||
|
||||
const links: Array<Record<string, unknown>> = [
|
||||
{
|
||||
relation: "document_refers_to_document",
|
||||
target_entity: "Document",
|
||||
target_id: `${input.id}-DOC-LINK`,
|
||||
source_field: "Recorder"
|
||||
}
|
||||
];
|
||||
if (input.withCounterparty !== false) {
|
||||
links.push({
|
||||
relation: "document_has_counterparty",
|
||||
target_entity: "Counterparty",
|
||||
target_id: `${input.id}-CP`,
|
||||
source_field: "Counterparty"
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
source_entity: "Document",
|
||||
source_id: input.id,
|
||||
display_name: input.id,
|
||||
unknown_link_count: input.unknownLinks ?? 1,
|
||||
problem_flags: ["risk_marker"],
|
||||
attributes,
|
||||
links
|
||||
};
|
||||
}
|
||||
|
||||
function createDataset(): SnapshotDataset {
|
||||
const settlements = [
|
||||
buildRecord({
|
||||
id: "SET-PC-1",
|
||||
account: "60",
|
||||
period: "2020-06-10T00:00:00",
|
||||
description: "supplier payment recorded but settlement chain is still open account 60"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "SET-PC-2",
|
||||
account: "62",
|
||||
period: "2020-06-11T00:00:00",
|
||||
description: "customer settlement tail payment to settlement relation broken account 62"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "SET-DOC-1",
|
||||
account: "60",
|
||||
period: "2020-06-20T00:00:00",
|
||||
description: "bank statement linked to settlement document payment chain account 60"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "SET-DOC-2",
|
||||
account: "62",
|
||||
period: "2020-06-21T00:00:00",
|
||||
description: "customer payment linked to settlement closure account 62"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "SET-KF-1",
|
||||
account: "60",
|
||||
period: "2020-06-22T00:00:00",
|
||||
description: "settlement key field record account 60 payment"
|
||||
})
|
||||
];
|
||||
|
||||
const vat = [
|
||||
buildRecord({
|
||||
id: "VAT-PC-1",
|
||||
account: "68",
|
||||
period: "2020-06-12T00:00:00",
|
||||
description: "vat invoice linked to register and purchase book account 68"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "VAT-PC-2",
|
||||
account: "19",
|
||||
period: "2020-06-13T00:00:00",
|
||||
description: "vat source document present but invoice to vat link is broken account 19"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "VAT-NDS-1",
|
||||
account: "68",
|
||||
period: "2020-06-23T00:00:00",
|
||||
description: "vat register entry book generation deduction posted"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "VAT-NDS-2",
|
||||
account: "19",
|
||||
period: "2020-06-24T00:00:00",
|
||||
description: "invoice to vat register chain for deduction account 19"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "VAT-KF-1",
|
||||
account: "68",
|
||||
period: "2020-06-25T00:00:00",
|
||||
description: "vat key field invoice register linkage account 68"
|
||||
})
|
||||
];
|
||||
|
||||
const close = [
|
||||
buildRecord({
|
||||
id: "CLS-PC-1",
|
||||
account: "20",
|
||||
period: "2020-06-14T00:00:00",
|
||||
description: "period close costs accumulated but allocation rules unresolved account 20"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "CLS-PC-2",
|
||||
account: "44",
|
||||
period: "2020-06-15T00:00:00",
|
||||
description: "month close operation runs with residuals not zero account 44"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "CLS-DOC-1",
|
||||
account: "20",
|
||||
period: "2020-06-26T00:00:00",
|
||||
description: "period close costs allocation writeoff account 20"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "CLS-DOC-2",
|
||||
account: "44",
|
||||
period: "2020-06-27T00:00:00",
|
||||
description: "month close residuals explained allocation account 44"
|
||||
}),
|
||||
buildRecord({
|
||||
id: "CLS-KF-1",
|
||||
account: "20",
|
||||
period: "2020-06-28T00:00:00",
|
||||
description: "period close key field account 20 allocation"
|
||||
})
|
||||
];
|
||||
|
||||
const mixed = [
|
||||
buildRecord({
|
||||
id: "MIX-PC-1",
|
||||
account: "68",
|
||||
period: "2020-12-31T00:00:00",
|
||||
description: "bank settlement vat mixed conflict record",
|
||||
zeroGuid: true
|
||||
}),
|
||||
buildRecord({
|
||||
id: "MIX-NDS-1",
|
||||
account: "60",
|
||||
period: "2020-12-30T00:00:00",
|
||||
description: "mixed nds and settlement overlap record",
|
||||
zeroGuid: true
|
||||
}),
|
||||
buildRecord({
|
||||
id: "MIX-DOC-1",
|
||||
account: "68",
|
||||
period: "2020-12-29T00:00:00",
|
||||
description: "mixed document with vat settlement and bank signals",
|
||||
zeroGuid: true
|
||||
}),
|
||||
buildRecord({
|
||||
id: "MIX-KF-1",
|
||||
account: "44",
|
||||
period: "2020-12-28T00:00:00",
|
||||
description: "mixed key field with period close and vat overlap",
|
||||
zeroGuid: true
|
||||
})
|
||||
];
|
||||
|
||||
return {
|
||||
keyFields: [settlements[4], vat[4], close[4], mixed[3]],
|
||||
problemCases: [mixed[0], vat[0], settlements[0], close[0], settlements[1], vat[1], close[1]],
|
||||
journals: [close[2], close[3], settlements[3]],
|
||||
ndsRegisters: [mixed[1], vat[2], vat[3]],
|
||||
docs: [mixed[2], settlements[2], settlements[3], vat[2], vat[3], close[2], close[3]]
|
||||
};
|
||||
}
|
||||
|
||||
function createSnapshotRoot(dataset: SnapshotDataset): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "assistant-wave5-regression-"));
|
||||
TEMP_DIRS.push(root);
|
||||
|
||||
const write = (fileName: string, records: Array<Record<string, unknown>>) => {
|
||||
fs.writeFileSync(path.resolve(root, fileName), JSON.stringify({ records }, null, 2), "utf-8");
|
||||
};
|
||||
|
||||
write("09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json", dataset.keyFields);
|
||||
write("03_snapshot_fragment_problem_cases.json", dataset.problemCases);
|
||||
write("07_samples_DocumentJournals.json", dataset.journals);
|
||||
write("08_samples_NDS_registers.json", dataset.ndsRegisters);
|
||||
write("04_samples_SpisanieSRaschetnogoScheta.json", dataset.docs);
|
||||
write("05_samples_RealizaciyaTovarovUslug.json", []);
|
||||
write("06_samples_PostuplenieTovarovUslug.json", []);
|
||||
|
||||
return root;
|
||||
}
|
||||
|
||||
function resolvePrefixFromId(sourceId: string): DomainPrefix | "OTHER" {
|
||||
if (sourceId.startsWith("SET")) return "SET";
|
||||
if (sourceId.startsWith("VAT")) return "VAT";
|
||||
if (sourceId.startsWith("CLS")) return "CLS";
|
||||
return "OTHER";
|
||||
}
|
||||
|
||||
function extractIds(items: Array<Record<string, unknown>>): string[] {
|
||||
return items.map((item) => String(item.source_id ?? "")).filter(Boolean);
|
||||
}
|
||||
|
||||
function hasForeignDomainInTop3(ids: string[], expected: DomainPrefix): boolean {
|
||||
return ids.slice(0, 3).some((id) => resolvePrefixFromId(id) !== expected);
|
||||
}
|
||||
|
||||
function top1IsRelevant(ids: string[], expected: DomainPrefix): boolean {
|
||||
if (ids.length === 0) {
|
||||
return false;
|
||||
}
|
||||
return resolvePrefixFromId(ids[0]) === expected;
|
||||
}
|
||||
|
||||
function legacyRiskScore(record: Record<string, unknown>): number {
|
||||
const unknown = Number(record.unknown_link_count ?? 0);
|
||||
const attributes = (record.attributes as Record<string, unknown>) ?? {};
|
||||
const links = Array.isArray(record.links) ? (record.links as Array<Record<string, unknown>>) : [];
|
||||
let zeroGuid = 0;
|
||||
for (const value of Object.values(attributes)) {
|
||||
if (String(value) === "00000000-0000-0000-0000-000000000000") {
|
||||
zeroGuid += 1;
|
||||
}
|
||||
}
|
||||
let navigationLinks = 0;
|
||||
for (const key of Object.keys(attributes)) {
|
||||
if (key.includes("@navigationLinkUrl")) {
|
||||
navigationLinks += 1;
|
||||
}
|
||||
}
|
||||
const cpLinks = links.filter((link) => String(link.target_entity ?? "") === "Counterparty").length;
|
||||
const flags = Array.isArray(record.problem_flags) ? record.problem_flags : [];
|
||||
|
||||
let score = 0;
|
||||
if (unknown > 0) score += 3;
|
||||
if (zeroGuid > 0) score += Math.min(3, 1 + zeroGuid);
|
||||
if (navigationLinks > 0) score += 1;
|
||||
if (cpLinks === 0) score += 1;
|
||||
if (flags.length > 0) score += 1;
|
||||
return score;
|
||||
}
|
||||
|
||||
function legacyRiskTopIds(dataset: SnapshotDataset): string[] {
|
||||
return [...dataset.problemCases, ...dataset.ndsRegisters]
|
||||
.map((record) => ({
|
||||
id: String(record.source_id ?? ""),
|
||||
score: legacyRiskScore(record)
|
||||
}))
|
||||
.filter((item) => item.score >= 2)
|
||||
.sort((left, right) => {
|
||||
if (right.score !== left.score) {
|
||||
return right.score - left.score;
|
||||
}
|
||||
return left.id.localeCompare(right.id);
|
||||
})
|
||||
.slice(0, 15)
|
||||
.map((item) => item.id);
|
||||
}
|
||||
|
||||
function legacyCanonicalTopIds(query: string, dataset: SnapshotDataset): string[] {
|
||||
const lower = query.toLowerCase();
|
||||
const useVatSource = /\bvat\b|\bnds\b|\b19\b|\b68\b|ндс/i.test(lower);
|
||||
const source = useVatSource ? [...dataset.ndsRegisters, ...dataset.keyFields] : dataset.docs;
|
||||
return source
|
||||
.map((record) => ({
|
||||
id: String(record.source_id ?? ""),
|
||||
sort: Date.parse(String(((record.attributes as Record<string, unknown>)?.Period ?? "") || "")) || 0
|
||||
}))
|
||||
.sort((left, right) => right.sort - left.sort)
|
||||
.slice(0, 12)
|
||||
.map((item) => item.id);
|
||||
}
|
||||
|
||||
function buildNormalizedCase(testCase: RegressionCase): NormalizedQueryV2_0_2 {
|
||||
const fragment: NormalizedFragmentV2_0_2 = {
|
||||
fragment_id: "F1",
|
||||
raw_fragment_text: testCase.query,
|
||||
normalized_fragment_text: testCase.query,
|
||||
domain_relevance: "in_scope",
|
||||
business_scope: "company_specific_accounting",
|
||||
entity_hints: ["document"],
|
||||
account_hints: [testCase.account_hint],
|
||||
document_hints: [],
|
||||
register_hints: [],
|
||||
time_scope: {
|
||||
type: "missing",
|
||||
value: null,
|
||||
confidence: "low"
|
||||
},
|
||||
flags: {
|
||||
has_multi_entity_scope: false,
|
||||
asks_for_chain_explanation: false,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: false,
|
||||
asks_for_anomaly_scan: false,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: false,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
candidate_labels: [testCase.candidate_label],
|
||||
confidence: "high",
|
||||
execution_readiness: "executable",
|
||||
clarification_reason: null,
|
||||
soft_assumption_used: [],
|
||||
route_status: "routed",
|
||||
no_route_reason: null
|
||||
};
|
||||
|
||||
return {
|
||||
schema_version: "normalized_query_v2_0_2",
|
||||
user_message_raw: testCase.query,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
contains_multiple_tasks: false,
|
||||
fragments: [fragment],
|
||||
discarded_fragments: [],
|
||||
global_notes: {
|
||||
needs_clarification: false,
|
||||
clarification_reason: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function legacyRouteForFragment(fragment: NormalizedFragmentV2_0_2): string {
|
||||
const accountHints = fragment.account_hints.map((item) => String(item));
|
||||
const hasLifecycleDomainHint =
|
||||
accountHints.some((item) => /^(97|01|02|08|19|68(?:\.\d+)?|51|60|62)$/.test(item)) ||
|
||||
fragment.candidate_labels.includes("anomaly_probe") ||
|
||||
fragment.candidate_labels.includes("period_close_risk");
|
||||
|
||||
if (fragment.flags.asks_for_exact_object_trace) return "live_mcp_drilldown";
|
||||
if (fragment.flags.asks_for_ranking_or_top || fragment.flags.asks_for_period_summary) return "batch_refresh_then_store";
|
||||
if (fragment.flags.asks_for_chain_explanation && (fragment.flags.has_multi_entity_scope || hasLifecycleDomainHint)) {
|
||||
return "hybrid_store_plus_live";
|
||||
}
|
||||
if (fragment.flags.asks_for_rule_check && !fragment.flags.asks_for_chain_explanation) return "store_feature_risk";
|
||||
if (
|
||||
fragment.flags.asks_for_anomaly_scan &&
|
||||
!fragment.flags.asks_for_ranking_or_top &&
|
||||
!(fragment.flags.has_multi_entity_scope && fragment.flags.asks_for_chain_explanation)
|
||||
) {
|
||||
return "store_feature_risk";
|
||||
}
|
||||
return "store_canonical";
|
||||
}
|
||||
|
||||
const SETTLEMENT_QUERIES = [
|
||||
"Show why payment recorded but settlement for account 60 is still open.",
|
||||
"Account 62: payment posted, settlement closure is missing.",
|
||||
"Find settlement tails for account 60 where payment did not close chain.",
|
||||
"Bank and settlements 60/62: where link to settlement is broken.",
|
||||
"Why does account 60 keep open settlement after payment record.",
|
||||
"Account 62 settlement problem: payment done, closure not reached.",
|
||||
"Detect symptom where payment exists but settlement remains open on 60.",
|
||||
"Find lifecycle gap in payment to settlement for account 62.",
|
||||
"60-62 settlement chain has residual tail after payment.",
|
||||
"Locate unresolved settlement after bank payment on account 60."
|
||||
];
|
||||
|
||||
const VAT_QUERIES = [
|
||||
"VAT check: source document exists but invoice link is missing on account 68.",
|
||||
"Account 19 VAT chain: document to register to book is broken.",
|
||||
"Find VAT symptom where invoice linked but book entry was not generated.",
|
||||
"Show VAT lifecycle gaps for account 68 in document-register-book flow.",
|
||||
"VAT deduction issue on 19: source document present but deduction not posted.",
|
||||
"Find broken invoice to VAT register relation for account 68.",
|
||||
"VAT problem-first: document exists, register is present, book entry missing.",
|
||||
"Locate VAT residual issue where deduction chain is incomplete on 19.",
|
||||
"VAT 68: invoice and register mismatch in purchase/sales book.",
|
||||
"Detect VAT symptom with broken doc-register-book chain for account 68."
|
||||
];
|
||||
|
||||
const CLOSE_QUERIES = [
|
||||
"Month close: costs on accounts 20 and 44 are not allocated, residuals remain.",
|
||||
"Period close problem for 20/44: allocation rules unresolved.",
|
||||
"Find close lifecycle gap where costs accumulated but close operation fails 20 44.",
|
||||
"Account 20 and 44 month close symptom: residuals are not zero.",
|
||||
"Show period close issue when costs are accumulated but not distributed 20/44.",
|
||||
"Close operation run for 20 and 44 leaves unexplained residuals.",
|
||||
"Detect month close break in costs allocation chain on 20/44.",
|
||||
"Period close 20-44: allocation exists but residual tail remains.",
|
||||
"Find cost close mismatch: costs accumulated, close not completed 20 and 44.",
|
||||
"Month close domain check for accounts 20 and 44 with unresolved residuals."
|
||||
];
|
||||
|
||||
const REGRESSION_CASES: RegressionCase[] = [
|
||||
...SETTLEMENT_QUERIES.map((query, index) => ({
|
||||
case_id: `SET-${String(index + 1).padStart(2, "0")}`,
|
||||
domain: "settlements_60_62" as const,
|
||||
expected_prefix: "SET" as const,
|
||||
query,
|
||||
account_hint: index % 2 === 0 ? "60" : "62",
|
||||
candidate_label: "anomaly_probe" as const
|
||||
})),
|
||||
...VAT_QUERIES.map((query, index) => ({
|
||||
case_id: `VAT-${String(index + 1).padStart(2, "0")}`,
|
||||
domain: "vat_document_register_book" as const,
|
||||
expected_prefix: "VAT" as const,
|
||||
query,
|
||||
account_hint: index % 2 === 0 ? "68" : "19",
|
||||
candidate_label: "anomaly_probe" as const
|
||||
})),
|
||||
...CLOSE_QUERIES.map((query, index) => ({
|
||||
case_id: `CLS-${String(index + 1).padStart(2, "0")}`,
|
||||
domain: "month_close_costs_20_44" as const,
|
||||
expected_prefix: "CLS" as const,
|
||||
query,
|
||||
account_hint: index % 2 === 0 ? "20" : "44",
|
||||
candidate_label: "period_close_risk" as const
|
||||
}))
|
||||
];
|
||||
|
||||
describe.sequential("stage4 wave5 P0 domain purity + route discipline regression", () => {
|
||||
afterEach(() => {
|
||||
cleanupTempDirs();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("keeps top-3 domain-pure and reroutes symptom/lifecycle intents away from canonical path", () => {
|
||||
const dataset = createDataset();
|
||||
const root = createSnapshotRoot(dataset);
|
||||
const dataLayer = new AssistantDataLayer(root);
|
||||
|
||||
const metrics = {
|
||||
route: {
|
||||
before_canonical: 0,
|
||||
after_canonical: 0,
|
||||
after_hybrid: 0
|
||||
},
|
||||
risk: {
|
||||
before_foreign_top3: 0,
|
||||
after_foreign_top3: 0,
|
||||
before_top1_relevant: 0,
|
||||
after_top1_relevant: 0
|
||||
},
|
||||
canonical: {
|
||||
before_foreign_top3: 0,
|
||||
after_foreign_top3: 0,
|
||||
before_top1_relevant: 0,
|
||||
after_top1_relevant: 0
|
||||
}
|
||||
};
|
||||
|
||||
for (const testCase of REGRESSION_CASES) {
|
||||
const normalized = buildNormalizedCase(testCase);
|
||||
const summary = toRouteHintSummary(normalized);
|
||||
expect(summary.mode).toBe("deterministic_v2");
|
||||
if (summary.mode !== "deterministic_v2") {
|
||||
throw new Error("Expected deterministic_v2 route summary");
|
||||
}
|
||||
const afterRoute = summary.decisions[0]?.route;
|
||||
const beforeRoute = legacyRouteForFragment(normalized.fragments[0]);
|
||||
if (beforeRoute === "store_canonical") {
|
||||
metrics.route.before_canonical += 1;
|
||||
}
|
||||
if (afterRoute === "store_canonical") {
|
||||
metrics.route.after_canonical += 1;
|
||||
}
|
||||
if (afterRoute === "hybrid_store_plus_live") {
|
||||
metrics.route.after_hybrid += 1;
|
||||
}
|
||||
expect(afterRoute).toBe("hybrid_store_plus_live");
|
||||
|
||||
const afterRisk = dataLayer.executeRoute("store_feature_risk", testCase.query);
|
||||
const afterRiskIds = extractIds(afterRisk.items as Array<Record<string, unknown>>);
|
||||
if (hasForeignDomainInTop3(afterRiskIds, testCase.expected_prefix)) {
|
||||
metrics.risk.after_foreign_top3 += 1;
|
||||
}
|
||||
if (top1IsRelevant(afterRiskIds, testCase.expected_prefix)) {
|
||||
metrics.risk.after_top1_relevant += 1;
|
||||
}
|
||||
|
||||
const afterCanonical = dataLayer.executeRoute("store_canonical", testCase.query);
|
||||
const afterCanonicalIds = extractIds(afterCanonical.items as Array<Record<string, unknown>>);
|
||||
if (hasForeignDomainInTop3(afterCanonicalIds, testCase.expected_prefix)) {
|
||||
metrics.canonical.after_foreign_top3 += 1;
|
||||
}
|
||||
if (top1IsRelevant(afterCanonicalIds, testCase.expected_prefix)) {
|
||||
metrics.canonical.after_top1_relevant += 1;
|
||||
}
|
||||
|
||||
const beforeRiskIds = legacyRiskTopIds(dataset);
|
||||
if (hasForeignDomainInTop3(beforeRiskIds, testCase.expected_prefix)) {
|
||||
metrics.risk.before_foreign_top3 += 1;
|
||||
}
|
||||
if (top1IsRelevant(beforeRiskIds, testCase.expected_prefix)) {
|
||||
metrics.risk.before_top1_relevant += 1;
|
||||
}
|
||||
|
||||
const beforeCanonicalIds = legacyCanonicalTopIds(testCase.query, dataset);
|
||||
if (hasForeignDomainInTop3(beforeCanonicalIds, testCase.expected_prefix)) {
|
||||
metrics.canonical.before_foreign_top3 += 1;
|
||||
}
|
||||
if (top1IsRelevant(beforeCanonicalIds, testCase.expected_prefix)) {
|
||||
metrics.canonical.before_top1_relevant += 1;
|
||||
}
|
||||
}
|
||||
|
||||
expect(REGRESSION_CASES.length).toBe(30);
|
||||
|
||||
expect(metrics.route.before_canonical).toBeGreaterThan(0);
|
||||
expect(metrics.route.after_canonical).toBe(0);
|
||||
expect(metrics.route.after_hybrid).toBe(REGRESSION_CASES.length);
|
||||
|
||||
expect(metrics.risk.before_foreign_top3).toBeGreaterThan(metrics.risk.after_foreign_top3);
|
||||
expect(metrics.risk.after_foreign_top3).toBe(0);
|
||||
expect(metrics.risk.after_top1_relevant).toBe(REGRESSION_CASES.length);
|
||||
|
||||
expect(metrics.canonical.before_foreign_top3).toBeGreaterThan(metrics.canonical.after_foreign_top3);
|
||||
expect(metrics.canonical.after_foreign_top3).toBe(0);
|
||||
expect(metrics.canonical.after_top1_relevant).toBe(REGRESSION_CASES.length);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,319 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { AnswerGroundingCheck, RequirementCoverageReport, UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
import type { ProblemUnit } from "../src/types/stage2ProblemUnits";
|
||||
|
||||
function buildRouteSummary() {
|
||||
return {
|
||||
mode: "deterministic_v2" as const,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high" as const,
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none" as const,
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildCoverage(partial = true): RequirementCoverageReport {
|
||||
return {
|
||||
requirements_total: 1,
|
||||
requirements_covered: partial ? 0 : 1,
|
||||
requirements_uncovered: partial ? ["R1"] : [],
|
||||
requirements_partially_covered: partial ? ["R1"] : [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
};
|
||||
}
|
||||
|
||||
function buildGrounding(status: AnswerGroundingCheck["status"] = "partial"): AnswerGroundingCheck {
|
||||
return {
|
||||
status,
|
||||
route_subject_match: true,
|
||||
missing_requirements: status === "partial" ? ["R1"] : [],
|
||||
reasons: status === "partial" ? ["Coverage is partial for problem-first answer contract."] : [],
|
||||
why_included_summary: [],
|
||||
selection_reason_summary: []
|
||||
};
|
||||
}
|
||||
|
||||
function buildProblemUnit(input: {
|
||||
id: string;
|
||||
type: ProblemUnit["problem_unit_type"];
|
||||
defect: string;
|
||||
account: string;
|
||||
lifecycleDomain?: ProblemUnit["lifecycle_domain"];
|
||||
}): ProblemUnit {
|
||||
return {
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: input.id,
|
||||
problem_unit_type: input.type,
|
||||
title: "Problem unit",
|
||||
mechanism_summary: `Mechanism candidate: ${input.defect}.`,
|
||||
business_defect_class: input.defect,
|
||||
severity: {
|
||||
score: 0.76,
|
||||
grade: "high"
|
||||
},
|
||||
confidence: {
|
||||
score: 0.52,
|
||||
grade: "medium"
|
||||
},
|
||||
affected_entities: ["Document:DOC-1", "Posting:POST-1"],
|
||||
affected_documents: ["Document:DOC-1"],
|
||||
affected_postings: ["Posting:POST-1"],
|
||||
affected_accounts: [input.account],
|
||||
affected_counterparties: ["Counterparty:CP-1"],
|
||||
affected_contracts: ["Contract:CTR-1"],
|
||||
failed_expected_edge: input.defect,
|
||||
period_impact: {
|
||||
is_period_sensitive: true,
|
||||
impact_class: "close_risk"
|
||||
},
|
||||
evidence_pack: ["cand-1"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
snapshot_limitations: [],
|
||||
...(input.lifecycleDomain
|
||||
? {
|
||||
lifecycle_domain: input.lifecycleDomain
|
||||
}
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
function buildRetrieval(units: ProblemUnit[], extras?: Partial<UnifiedRetrievalResult>): UnifiedRetrievalResult {
|
||||
return {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "hybrid_store_plus_live",
|
||||
status: "ok",
|
||||
result_type: "chain",
|
||||
items: [
|
||||
{
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-1",
|
||||
counterparty_id: "CP-1"
|
||||
}
|
||||
],
|
||||
summary: {
|
||||
broad_query_detected: true,
|
||||
broad_result_flag: true,
|
||||
minimum_evidence_failed: false,
|
||||
degraded_to: "partial",
|
||||
narrowing_strength: "weak",
|
||||
semantic_profile: {
|
||||
domain_scope: ["bank_settlement"],
|
||||
account_scope: ["60"],
|
||||
relation_patterns: ["payment_to_settlement"]
|
||||
}
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
evidence_id: "ev-1",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06",
|
||||
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|doc-1|2020-06"
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "hybrid_store_plus_live",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
evidence_kind: "mechanism_link",
|
||||
mechanism_note: "failed_edge:payment_to_settlement",
|
||||
confidence: "medium",
|
||||
limitation: {
|
||||
reason_code: "weak_source_mapping",
|
||||
note: null
|
||||
},
|
||||
payload: {
|
||||
risk_score: 4
|
||||
}
|
||||
}
|
||||
],
|
||||
problem_units: units,
|
||||
problem_unit_summary: {
|
||||
schema_version: "problem_unit_summary_v0_1",
|
||||
units_total: units.length,
|
||||
duplicate_collapses: 0,
|
||||
unit_types: units.map((unit) => unit.problem_unit_type),
|
||||
type_distribution: {
|
||||
[units[0]?.problem_unit_type ?? "broken_chain_segment"]: units.length
|
||||
},
|
||||
severity_distribution: {
|
||||
low: 0,
|
||||
medium: 0,
|
||||
high: units.length
|
||||
},
|
||||
confidence_distribution: {
|
||||
low: 0,
|
||||
medium: units.length,
|
||||
high: 0
|
||||
},
|
||||
primary_unit_type: units[0]?.problem_unit_type ?? null
|
||||
},
|
||||
why_included: ["semantic retrieval profile", "route=hybrid_store_plus_live"],
|
||||
selection_reason: ["domain_scope + relation_patterns + route profile"],
|
||||
risk_factors: ["broken_chain", "closure_risk"],
|
||||
business_interpretation: ["problem-first signal"],
|
||||
confidence: "medium",
|
||||
limitations: ["Evidence is snapshot-only and may lag source-of-record."],
|
||||
errors: [],
|
||||
...extras
|
||||
};
|
||||
}
|
||||
|
||||
function composeCase(userMessage: string, retrieval: UnifiedRetrievalResult) {
|
||||
return composeAssistantAnswer({
|
||||
userMessage,
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить проблемный механизм",
|
||||
subject_tokens: ["chain"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(true),
|
||||
groundingCheck: buildGrounding("partial"),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true,
|
||||
enableLifecycleAnswerV1: true
|
||||
});
|
||||
}
|
||||
|
||||
function extractSection(text: string, title: string): string {
|
||||
const escaped = title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const stopTitles = [
|
||||
"Коротко",
|
||||
"Что сломано",
|
||||
"Почему это похоже на проблему",
|
||||
"На чем это основано",
|
||||
"Что проверить первым",
|
||||
"Ограничения"
|
||||
];
|
||||
const stopPattern = stopTitles.map((item) => item.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
|
||||
const re = new RegExp(`${escaped}:([\\s\\S]*?)(?=(?:${stopPattern}):|$)`, "i");
|
||||
const match = String(text ?? "").match(re);
|
||||
return match?.[1]?.trim() ?? "";
|
||||
}
|
||||
|
||||
describe("assistant wave6 problem-first answer contract", () => {
|
||||
it("enforces leakage guard in direct user-facing answer", () => {
|
||||
const units = [buildProblemUnit({ id: "pu-1", type: "broken_chain_segment", defect: "failed_edge:payment_to_settlement", account: "60" })];
|
||||
const output = composeCase("Покажи проблему по расчетам.", buildRetrieval(units));
|
||||
|
||||
expect(output.assistant_reply).not.toMatch(
|
||||
/graph_|domain_scope|relation_patterns|route|profile|hybrid_store_plus_live|store_canonical|semantic_profile|lifecycle_defect_type/i
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps narrative mechanism-first and avoids entity-list direct answer", () => {
|
||||
const units = [buildProblemUnit({ id: "pu-1", type: "broken_chain_segment", defect: "failed_edge:payment_to_settlement", account: "60" })];
|
||||
const output = composeCase("Проверь по 60 счету, где разрыв.", buildRetrieval(units));
|
||||
const brokenSection = extractSection(output.assistant_reply, "Что сломано");
|
||||
|
||||
expect(brokenSection).toMatch(/не подтвержден|разрыв|зависл|закрыти/i);
|
||||
expect(brokenSection).not.toMatch(/^\s*-\s*(Document|Record|Entity)\b/i);
|
||||
});
|
||||
|
||||
it("does not expose route/profile explanation in user-facing text", () => {
|
||||
const units = [buildProblemUnit({ id: "pu-1", type: "document_conflict", defect: "posting_mismatch", account: "60" })];
|
||||
const output = composeCase("Где конфликт документа и проводки?", buildRetrieval(units));
|
||||
|
||||
expect(output.assistant_reply).not.toMatch(/route|profile|semantic|domain_scope|relation_patterns|typed_domain_path/i);
|
||||
});
|
||||
|
||||
it("collapses duplicate problem lines for the same mechanism", () => {
|
||||
const units = [
|
||||
buildProblemUnit({ id: "pu-1", type: "broken_chain_segment", defect: "failed_edge:payment_to_settlement", account: "60" }),
|
||||
buildProblemUnit({ id: "pu-2", type: "unresolved_settlement_cluster", defect: "payment_to_settlement", account: "60" })
|
||||
];
|
||||
const output = composeCase("Проверь хвост по расчетам.", buildRetrieval(units));
|
||||
const brokenSection = extractSection(output.assistant_reply, "Что сломано");
|
||||
const bulletLines = brokenSection
|
||||
.split(/\r?\n/g)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.startsWith("- "));
|
||||
|
||||
expect(bulletLines.length).toBe(1);
|
||||
});
|
||||
|
||||
it("shows explicit limitation when period is missing", () => {
|
||||
const units = [buildProblemUnit({ id: "pu-1", type: "lifecycle_anomaly_node", defect: "missing_expected_transition", account: "97", lifecycleDomain: "deferred_expense" })];
|
||||
const output = composeCase("Проверь по 97 счету зависание списания.", buildRetrieval(units));
|
||||
const limitationsSection = extractSection(output.assistant_reply, "Ограничения");
|
||||
|
||||
expect(limitationsSection).toMatch(/период/i);
|
||||
});
|
||||
|
||||
it("returns short accountant-readable answers for P0 domains without technical dump", () => {
|
||||
const cases: Array<{
|
||||
message: string;
|
||||
retrieval: UnifiedRetrievalResult;
|
||||
domainHint: RegExp;
|
||||
}> = [
|
||||
{
|
||||
message: "Проверь хвосты по расчетам 60/62.",
|
||||
retrieval: buildRetrieval([
|
||||
buildProblemUnit({ id: "pu-60", type: "broken_chain_segment", defect: "failed_edge:payment_to_settlement", account: "60" })
|
||||
]),
|
||||
domainHint: /расчет|оплат|закрыти/i
|
||||
},
|
||||
{
|
||||
message: "Проверь НДС-цепочку по документу.",
|
||||
retrieval: buildRetrieval([
|
||||
buildProblemUnit({ id: "pu-vat", type: "cross_branch_inconsistency_cluster", defect: "invoice_linked", account: "68", lifecycleDomain: "vat_flow" })
|
||||
]),
|
||||
domainHint: /ндс|регистр|книг/i
|
||||
},
|
||||
{
|
||||
message: "Проверь закрытие месяца и затраты 20-44.",
|
||||
retrieval: buildRetrieval([
|
||||
buildProblemUnit({ id: "pu-close", type: "period_risk_cluster", defect: "close_operation_runs", account: "20", lifecycleDomain: "period_close" })
|
||||
]),
|
||||
domainHint: /закрыти|месяц|затрат/i
|
||||
}
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const output = composeCase(testCase.message, testCase.retrieval);
|
||||
expect(output.assistant_reply).toMatch(testCase.domainHint);
|
||||
expect(output.assistant_reply).toContain("Коротко:");
|
||||
expect(output.assistant_reply).toContain("Что сломано:");
|
||||
expect(output.assistant_reply).toContain("Почему это похоже на проблему:");
|
||||
expect(output.assistant_reply).toContain("На чем это основано:");
|
||||
expect(output.assistant_reply).toContain("Что проверить первым:");
|
||||
expect(output.assistant_reply).toContain("Ограничения:");
|
||||
expect(output.assistant_reply.length).toBeLessThan(1800);
|
||||
expect(output.assistant_reply).not.toMatch(/graph_|domain_scope|relation_patterns|semantic_profile|route|profile/i);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const PROBLEM_UNITS_FLAG = "FEATURE_ASSISTANT_PROBLEM_UNITS_V1";
|
||||
const GRAPH_RUNTIME_FLAG = "FEATURE_ASSISTANT_GRAPH_RUNTIME_V1";
|
||||
const ORIGINAL_PROBLEM_UNITS_FLAG = process.env[PROBLEM_UNITS_FLAG];
|
||||
const ORIGINAL_GRAPH_RUNTIME_FLAG = process.env[GRAPH_RUNTIME_FLAG];
|
||||
|
||||
function restoreFlags(): void {
|
||||
if (ORIGINAL_PROBLEM_UNITS_FLAG === undefined) {
|
||||
delete process.env[PROBLEM_UNITS_FLAG];
|
||||
} else {
|
||||
process.env[PROBLEM_UNITS_FLAG] = ORIGINAL_PROBLEM_UNITS_FLAG;
|
||||
}
|
||||
if (ORIGINAL_GRAPH_RUNTIME_FLAG === undefined) {
|
||||
delete process.env[GRAPH_RUNTIME_FLAG];
|
||||
} else {
|
||||
process.env[GRAPH_RUNTIME_FLAG] = ORIGINAL_GRAPH_RUNTIME_FLAG;
|
||||
}
|
||||
}
|
||||
|
||||
async function normalizeWithFlags(input: {
|
||||
problemUnits: "0" | "1";
|
||||
graphRuntime: "0" | "1";
|
||||
}) {
|
||||
process.env[PROBLEM_UNITS_FLAG] = input.problemUnits;
|
||||
process.env[GRAPH_RUNTIME_FLAG] = input.graphRuntime;
|
||||
vi.resetModules();
|
||||
const { normalizeRetrievalResult } = await import("../src/services/retrievalResultNormalizer");
|
||||
return normalizeRetrievalResult("F1", ["R1"], "hybrid_store_plus_live", {
|
||||
status: "ok",
|
||||
result_type: "list",
|
||||
items: [
|
||||
{
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-1",
|
||||
risk_score: 5
|
||||
}
|
||||
],
|
||||
summary: {
|
||||
broad_query_detected: false
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
evidence_id: "ev-1",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "hybrid_store_plus_live",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
failed_expected_edge: "payment_to_settlement",
|
||||
anomaly_patterns: ["broken_lifecycle", "missing_link"],
|
||||
confidence: "medium"
|
||||
}
|
||||
],
|
||||
why_included: ["synthetic-test"],
|
||||
selection_reason: ["synthetic-test"],
|
||||
risk_factors: ["broken_lifecycle"],
|
||||
business_interpretation: ["synthetic-test"],
|
||||
confidence: "medium",
|
||||
limitations: [],
|
||||
errors: []
|
||||
});
|
||||
}
|
||||
|
||||
describe.sequential("retrieval dual payload compatibility for graph runtime", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("keeps stage2 payload when graph runtime flag is OFF", async () => {
|
||||
const result = await normalizeWithFlags({
|
||||
problemUnits: "1",
|
||||
graphRuntime: "0"
|
||||
});
|
||||
|
||||
expect(Array.isArray(result.problem_units)).toBe(true);
|
||||
expect(result.problem_units?.length).toBeGreaterThan(0);
|
||||
expect(result.accounting_graph).toBeUndefined();
|
||||
expect(result.summary.graph_runtime_enabled).toBeUndefined();
|
||||
expect(result.problem_units?.some((item) => item.graph_binding)).toBe(false);
|
||||
});
|
||||
|
||||
it("adds graph runtime payload when graph runtime flag is ON", async () => {
|
||||
const result = await normalizeWithFlags({
|
||||
problemUnits: "1",
|
||||
graphRuntime: "1"
|
||||
});
|
||||
|
||||
expect(Array.isArray(result.problem_units)).toBe(true);
|
||||
expect(result.problem_units?.length).toBeGreaterThan(0);
|
||||
expect(result.accounting_graph?.schema_version).toBe("accounting_graph_v0_1");
|
||||
expect((result.accounting_graph?.nodes.length ?? 0) > 0).toBe(true);
|
||||
expect((result.accounting_graph?.edges.length ?? 0) > 0).toBe(true);
|
||||
expect(result.summary.graph_runtime_enabled).toBe(true);
|
||||
expect(typeof result.summary.graph_nodes_count).toBe("number");
|
||||
expect(typeof result.summary.graph_edges_count).toBe("number");
|
||||
expect(result.problem_unit_summary?.graph_summary).toBeDefined();
|
||||
expect(result.problem_units?.some((item) => Boolean(item.graph_binding?.graph_node_id))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -38,7 +38,7 @@ describe("routeHintAdapter", () => {
|
||||
}
|
||||
expect(summary.fallback.type).toBe("none");
|
||||
expect(summary.decisions[0]?.execution_readiness).toBe("executable_with_soft_assumptions");
|
||||
expect(summary.decisions[0]?.route).toBe("store_feature_risk");
|
||||
expect(summary.decisions[0]?.route).toBe("hybrid_store_plus_live");
|
||||
});
|
||||
|
||||
it("uses explicit v2.0.2 route_status/no_route_reason contract", () => {
|
||||
@@ -49,11 +49,180 @@ describe("routeHintAdapter", () => {
|
||||
}
|
||||
expect(summary.decisions[0]?.route_status).toBe("routed");
|
||||
expect(summary.decisions[0]?.no_route_reason).toBeNull();
|
||||
expect(summary.decisions[0]?.route).toBe("store_feature_risk");
|
||||
expect(summary.decisions[0]?.route).toBe("hybrid_store_plus_live");
|
||||
|
||||
const routerInput = toRouterInput(normalizedFixtureV2_0_2());
|
||||
const first = (routerInput.fragments as Array<Record<string, unknown>>)[0];
|
||||
expect(first.route_status).toBe("routed");
|
||||
expect(first.no_route_reason).toBeNull();
|
||||
});
|
||||
|
||||
it("promotes lifecycle chain intent to hybrid route even without multi-entity flag", () => {
|
||||
const summary = toRouteHintSummary({
|
||||
schema_version: "normalized_query_v2_0_2",
|
||||
user_message_raw: "Проверь по 97-му где расходы зависли и не дошли до списания",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
contains_multiple_tasks: false,
|
||||
fragments: [
|
||||
{
|
||||
fragment_id: "F1",
|
||||
raw_fragment_text: "где расходы будущих периодов зависли и не дошли до списания",
|
||||
normalized_fragment_text: "расходы будущих периодов зависли",
|
||||
domain_relevance: "in_scope",
|
||||
business_scope: "company_specific_accounting",
|
||||
entity_hints: [],
|
||||
account_hints: ["97"],
|
||||
document_hints: [],
|
||||
register_hints: [],
|
||||
time_scope: {
|
||||
type: "missing",
|
||||
value: null,
|
||||
confidence: "low"
|
||||
},
|
||||
flags: {
|
||||
has_multi_entity_scope: false,
|
||||
asks_for_chain_explanation: true,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: false,
|
||||
asks_for_anomaly_scan: false,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: false,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
candidate_labels: ["cross_entity", "anomaly_probe"],
|
||||
confidence: "high",
|
||||
execution_readiness: "executable",
|
||||
clarification_reason: null,
|
||||
soft_assumption_used: [],
|
||||
route_status: "routed",
|
||||
no_route_reason: null
|
||||
}
|
||||
],
|
||||
discarded_fragments: [],
|
||||
global_notes: {
|
||||
needs_clarification: false,
|
||||
clarification_reason: null
|
||||
}
|
||||
});
|
||||
expect(summary.mode).toBe("deterministic_v2");
|
||||
if (summary.mode !== "deterministic_v2") {
|
||||
throw new Error("Expected deterministic_v2 summary");
|
||||
}
|
||||
expect(summary.decisions[0]?.route).toBe("hybrid_store_plus_live");
|
||||
});
|
||||
|
||||
it("promotes mixed-ambiguity symptom fragment to hybrid when domain anchors are present", () => {
|
||||
const summary = toRouteHintSummary({
|
||||
schema_version: "normalized_query_v2_0_2",
|
||||
user_message_raw: "2020-06 account 60: payment posted but settlement remains open",
|
||||
message_in_scope: false,
|
||||
scope_confidence: "low",
|
||||
contains_multiple_tasks: false,
|
||||
fragments: [
|
||||
{
|
||||
fragment_id: "F1",
|
||||
raw_fragment_text: "2020-06 account 60: payment posted but settlement remains open",
|
||||
normalized_fragment_text: "2020-06 account 60 payment posted but settlement remains open",
|
||||
domain_relevance: "unclear",
|
||||
business_scope: "unclear",
|
||||
entity_hints: [],
|
||||
account_hints: ["60"],
|
||||
document_hints: [],
|
||||
register_hints: [],
|
||||
time_scope: {
|
||||
type: "explicit",
|
||||
value: "2020-06",
|
||||
confidence: "high"
|
||||
},
|
||||
flags: {
|
||||
has_multi_entity_scope: false,
|
||||
asks_for_chain_explanation: false,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: false,
|
||||
asks_for_anomaly_scan: false,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: false,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
candidate_labels: [],
|
||||
confidence: "low",
|
||||
execution_readiness: "needs_clarification",
|
||||
clarification_reason: "domain_or_scope_unclear",
|
||||
soft_assumption_used: [],
|
||||
route_status: "no_route",
|
||||
no_route_reason: "insufficient_specificity"
|
||||
}
|
||||
],
|
||||
discarded_fragments: [],
|
||||
global_notes: {
|
||||
needs_clarification: false,
|
||||
clarification_reason: null
|
||||
}
|
||||
});
|
||||
expect(summary.mode).toBe("deterministic_v2");
|
||||
if (summary.mode !== "deterministic_v2") {
|
||||
throw new Error("Expected deterministic_v2 summary");
|
||||
}
|
||||
expect(summary.decisions[0]?.route).toBe("hybrid_store_plus_live");
|
||||
expect(summary.decisions[0]?.route_status).toBe("routed");
|
||||
});
|
||||
|
||||
it("keeps canonical path only for factual fragments without symptom/lifecycle markers", () => {
|
||||
const summary = toRouteHintSummary({
|
||||
schema_version: "normalized_query_v2_0_2",
|
||||
user_message_raw: "Покажи последний документ по 10 счету за июнь 2020.",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
contains_multiple_tasks: false,
|
||||
fragments: [
|
||||
{
|
||||
fragment_id: "F1",
|
||||
raw_fragment_text: "последний документ по 10 счету за июнь 2020",
|
||||
normalized_fragment_text: "последний документ по 10 счету июнь 2020",
|
||||
domain_relevance: "in_scope",
|
||||
business_scope: "company_specific_accounting",
|
||||
entity_hints: ["документ"],
|
||||
account_hints: ["10"],
|
||||
document_hints: [],
|
||||
register_hints: [],
|
||||
time_scope: {
|
||||
type: "explicit",
|
||||
value: "2020-06",
|
||||
confidence: "high"
|
||||
},
|
||||
flags: {
|
||||
has_multi_entity_scope: false,
|
||||
asks_for_chain_explanation: false,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: false,
|
||||
asks_for_anomaly_scan: false,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: false,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
candidate_labels: ["simple_factual"],
|
||||
confidence: "high",
|
||||
execution_readiness: "executable",
|
||||
clarification_reason: null,
|
||||
soft_assumption_used: [],
|
||||
route_status: "routed",
|
||||
no_route_reason: null
|
||||
}
|
||||
],
|
||||
discarded_fragments: [],
|
||||
global_notes: {
|
||||
needs_clarification: false,
|
||||
clarification_reason: null
|
||||
}
|
||||
});
|
||||
expect(summary.mode).toBe("deterministic_v2");
|
||||
if (summary.mode !== "deterministic_v2") {
|
||||
throw new Error("Expected deterministic_v2 summary");
|
||||
}
|
||||
expect(summary.decisions[0]?.route).toBe("store_canonical");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildAccountingGraph } from "../src/services/stage4GraphRuntime";
|
||||
import type { CandidateEvidenceItem, ProblemUnit } from "../src/types/stage2ProblemUnits";
|
||||
|
||||
function buildProblemUnit(input: {
|
||||
id: string;
|
||||
type: ProblemUnit["problem_unit_type"];
|
||||
domain?: ProblemUnit["lifecycle_domain"];
|
||||
accounts?: string[];
|
||||
current?: string;
|
||||
expected?: string;
|
||||
missing?: string;
|
||||
invalid?: string;
|
||||
defect?: ProblemUnit["lifecycle_defect_type"];
|
||||
evidencePack?: string[];
|
||||
}): ProblemUnit {
|
||||
return {
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: input.id,
|
||||
problem_unit_type: input.type,
|
||||
title: "Synthetic unit",
|
||||
mechanism_summary: "Synthetic mechanism",
|
||||
business_defect_class: "broken_lifecycle",
|
||||
severity: {
|
||||
score: 0.76,
|
||||
grade: "high"
|
||||
},
|
||||
confidence: {
|
||||
score: 0.64,
|
||||
grade: "medium"
|
||||
},
|
||||
affected_entities: ["Document:DOC-1"],
|
||||
affected_documents: ["Document:DOC-1"],
|
||||
affected_postings: [],
|
||||
affected_accounts: input.accounts ?? [],
|
||||
affected_counterparties: ["Counterparty:CP-1"],
|
||||
affected_contracts: [],
|
||||
evidence_pack: input.evidencePack ?? ["cand-1"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
snapshot_limitations: [],
|
||||
...(input.domain
|
||||
? {
|
||||
lifecycle_domain: input.domain
|
||||
}
|
||||
: {}),
|
||||
...(input.current
|
||||
? {
|
||||
current_lifecycle_state: input.current
|
||||
}
|
||||
: {}),
|
||||
...(input.expected
|
||||
? {
|
||||
expected_lifecycle_state: input.expected
|
||||
}
|
||||
: {}),
|
||||
...(input.missing
|
||||
? {
|
||||
missing_transition: input.missing
|
||||
}
|
||||
: {}),
|
||||
...(input.invalid
|
||||
? {
|
||||
invalid_transition: input.invalid
|
||||
}
|
||||
: {}),
|
||||
...(input.defect
|
||||
? {
|
||||
lifecycle_defect_type: input.defect
|
||||
}
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
function buildCandidate(id: string): CandidateEvidenceItem {
|
||||
return {
|
||||
schema_version: "candidate_evidence_v0_1",
|
||||
candidate_id: id,
|
||||
route: "hybrid_store_plus_live",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06",
|
||||
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|doc-1|2020-06"
|
||||
},
|
||||
relation_pattern_hits: ["payment_to_settlement", "deferred_expense_to_writeoff"],
|
||||
anomaly_patterns: ["broken_lifecycle"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
confidence_hint: "medium"
|
||||
};
|
||||
}
|
||||
|
||||
describe("stage4GraphRuntime", () => {
|
||||
it("builds graph bindings for deferred expense lifecycle with missing transition", () => {
|
||||
const result = buildAccountingGraph({
|
||||
route: "hybrid_store_plus_live",
|
||||
candidateEvidence: [buildCandidate("cand-1")],
|
||||
problemUnits: [
|
||||
buildProblemUnit({
|
||||
id: "pu-97-1",
|
||||
type: "lifecycle_anomaly_node",
|
||||
domain: "deferred_expense",
|
||||
accounts: ["97"],
|
||||
current: "recognized",
|
||||
expected: "fully_written_off",
|
||||
missing: "recognized->partially_written_off"
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
expect(result.summary.bound_units).toBe(1);
|
||||
expect(result.summary.domain_distribution.deferred_expense).toBe(1);
|
||||
expect(result.summary.missing_links_count).toBeGreaterThan(0);
|
||||
expect(result.edges.some((item) => item.relation_type === "missing_transition")).toBe(true);
|
||||
expect(result.unit_bindings[0].relation_path.join("|")).toContain("deferred_expense_to_writeoff");
|
||||
});
|
||||
|
||||
it("marks cross-branch conflicts for vat graph branches", () => {
|
||||
const result = buildAccountingGraph({
|
||||
route: "store_feature_risk",
|
||||
candidateEvidence: [buildCandidate("cand-vat")],
|
||||
problemUnits: [
|
||||
buildProblemUnit({
|
||||
id: "pu-vat-1",
|
||||
type: "cross_branch_inconsistency_cluster",
|
||||
domain: "vat_flow",
|
||||
accounts: ["19", "68"],
|
||||
current: "vat_conflict",
|
||||
expected: "vat_reflected",
|
||||
invalid: "cross_branch_conflict_transition",
|
||||
defect: "cross_branch_state_conflict",
|
||||
evidencePack: ["cand-vat"]
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
expect(result.summary.domain_distribution.vat_flow).toBe(1);
|
||||
expect(result.summary.conflicting_links_count).toBeGreaterThan(0);
|
||||
expect(result.edges.some((item) => item.flags.includes("conflict_link"))).toBe(true);
|
||||
expect(result.unit_bindings[0].conflicting_links).toContain("cross_branch_conflict_transition");
|
||||
});
|
||||
|
||||
it("infers 97 domain mapping when lifecycle domain is absent", () => {
|
||||
const result = buildAccountingGraph({
|
||||
route: "store_feature_risk",
|
||||
candidateEvidence: [buildCandidate("cand-no-domain")],
|
||||
problemUnits: [
|
||||
buildProblemUnit({
|
||||
id: "pu-97-2",
|
||||
type: "lifecycle_anomaly_node",
|
||||
accounts: ["97"],
|
||||
current: "recognized",
|
||||
expected: "fully_written_off"
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
expect(result.summary.domain_distribution.deferred_expense).toBe(1);
|
||||
expect(result.summary.graph_coverage_grade).toBe("high");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,187 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { AssistantDataLayer } from "../src/services/assistantDataLayer.ts";
|
||||
|
||||
type DomainPrefix = "SET" | "VAT" | "CLS";
|
||||
interface RegressionCase {
|
||||
case_id: string;
|
||||
expected_prefix: DomainPrefix;
|
||||
query: string;
|
||||
}
|
||||
|
||||
function buildRecord(input: {
|
||||
id: string;
|
||||
account: string;
|
||||
period: string;
|
||||
description: string;
|
||||
unknownLinks?: number;
|
||||
withCounterparty?: boolean;
|
||||
zeroGuid?: boolean;
|
||||
}): Record<string, unknown> {
|
||||
const attributes: Record<string, unknown> = {
|
||||
Recorder: `${input.id}-REC`,
|
||||
Period: input.period,
|
||||
Description: input.description,
|
||||
Account: input.account,
|
||||
"trace@navigationLinkUrl": `/trace/${input.id}`
|
||||
};
|
||||
if (input.zeroGuid) {
|
||||
attributes.LinkGuid = "00000000-0000-0000-0000-000000000000";
|
||||
}
|
||||
|
||||
const links: Array<Record<string, unknown>> = [
|
||||
{
|
||||
relation: "document_refers_to_document",
|
||||
target_entity: "Document",
|
||||
target_id: `${input.id}-DOC-LINK`,
|
||||
source_field: "Recorder"
|
||||
}
|
||||
];
|
||||
if (input.withCounterparty !== false) {
|
||||
links.push({
|
||||
relation: "document_has_counterparty",
|
||||
target_entity: "Counterparty",
|
||||
target_id: `${input.id}-CP`,
|
||||
source_field: "Counterparty"
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
source_entity: "Document",
|
||||
source_id: input.id,
|
||||
display_name: input.id,
|
||||
unknown_link_count: input.unknownLinks ?? 1,
|
||||
problem_flags: ["risk_marker"],
|
||||
attributes,
|
||||
links
|
||||
};
|
||||
}
|
||||
|
||||
function createDataset() {
|
||||
const settlements = [
|
||||
buildRecord({ id: "SET-PC-1", account: "60", period: "2020-06-10T00:00:00", description: "supplier payment recorded but settlement chain is still open account 60" }),
|
||||
buildRecord({ id: "SET-PC-2", account: "62", period: "2020-06-11T00:00:00", description: "customer settlement tail payment to settlement relation broken account 62" }),
|
||||
buildRecord({ id: "SET-DOC-1", account: "60", period: "2020-06-20T00:00:00", description: "bank statement linked to settlement document payment chain account 60" }),
|
||||
buildRecord({ id: "SET-DOC-2", account: "62", period: "2020-06-21T00:00:00", description: "customer payment linked to settlement closure account 62" }),
|
||||
buildRecord({ id: "SET-KF-1", account: "60", period: "2020-06-22T00:00:00", description: "settlement key field record account 60 payment" })
|
||||
];
|
||||
|
||||
const vat = [
|
||||
buildRecord({ id: "VAT-PC-1", account: "68", period: "2020-06-12T00:00:00", description: "vat invoice linked to register and purchase book account 68" }),
|
||||
buildRecord({ id: "VAT-PC-2", account: "19", period: "2020-06-13T00:00:00", description: "vat source document present but invoice to vat link is broken account 19" }),
|
||||
buildRecord({ id: "VAT-NDS-1", account: "68", period: "2020-06-23T00:00:00", description: "vat register entry book generation deduction posted" }),
|
||||
buildRecord({ id: "VAT-NDS-2", account: "19", period: "2020-06-24T00:00:00", description: "invoice to vat register chain for deduction account 19" }),
|
||||
buildRecord({ id: "VAT-KF-1", account: "68", period: "2020-06-25T00:00:00", description: "vat key field invoice register linkage account 68" })
|
||||
];
|
||||
|
||||
const close = [
|
||||
buildRecord({ id: "CLS-PC-1", account: "20", period: "2020-06-14T00:00:00", description: "period close costs accumulated but allocation rules unresolved account 20" }),
|
||||
buildRecord({ id: "CLS-PC-2", account: "44", period: "2020-06-15T00:00:00", description: "month close operation runs with residuals not zero account 44" }),
|
||||
buildRecord({ id: "CLS-DOC-1", account: "20", period: "2020-06-26T00:00:00", description: "period close costs allocation writeoff account 20" }),
|
||||
buildRecord({ id: "CLS-DOC-2", account: "44", period: "2020-06-27T00:00:00", description: "month close residuals explained allocation account 44" }),
|
||||
buildRecord({ id: "CLS-KF-1", account: "20", period: "2020-06-28T00:00:00", description: "period close key field account 20 allocation" })
|
||||
];
|
||||
|
||||
const mixed = [
|
||||
buildRecord({ id: "MIX-PC-1", account: "68", period: "2020-12-31T00:00:00", description: "bank settlement vat mixed conflict record", zeroGuid: true }),
|
||||
buildRecord({ id: "MIX-NDS-1", account: "60", period: "2020-12-30T00:00:00", description: "mixed nds and settlement overlap record", zeroGuid: true }),
|
||||
buildRecord({ id: "MIX-DOC-1", account: "68", period: "2020-12-29T00:00:00", description: "mixed document with vat settlement and bank signals", zeroGuid: true }),
|
||||
buildRecord({ id: "MIX-KF-1", account: "44", period: "2020-12-28T00:00:00", description: "mixed key field with period close and vat overlap", zeroGuid: true })
|
||||
];
|
||||
|
||||
return {
|
||||
keyFields: [settlements[4], vat[4], close[4], mixed[3]],
|
||||
problemCases: [mixed[0], vat[0], settlements[0], close[0], settlements[1], vat[1], close[1]],
|
||||
journals: [close[2], close[3], settlements[3]],
|
||||
ndsRegisters: [mixed[1], vat[2], vat[3]],
|
||||
docs: [mixed[2], settlements[2], settlements[3], vat[2], vat[3], close[2], close[3]]
|
||||
};
|
||||
}
|
||||
|
||||
function createSnapshotRoot(dataset: any): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "assistant-wave5-debug-"));
|
||||
const write = (fileName: string, records: Array<Record<string, unknown>>) => {
|
||||
fs.writeFileSync(path.resolve(root, fileName), JSON.stringify({ records }, null, 2), "utf-8");
|
||||
};
|
||||
write("09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json", dataset.keyFields);
|
||||
write("03_snapshot_fragment_problem_cases.json", dataset.problemCases);
|
||||
write("07_samples_DocumentJournals.json", dataset.journals);
|
||||
write("08_samples_NDS_registers.json", dataset.ndsRegisters);
|
||||
write("04_samples_SpisanieSRaschetnogoScheta.json", dataset.docs);
|
||||
write("05_samples_RealizaciyaTovarovUslug.json", []);
|
||||
write("06_samples_PostuplenieTovarovUslug.json", []);
|
||||
return root;
|
||||
}
|
||||
|
||||
function resolvePrefixFromId(sourceId: string): DomainPrefix | "OTHER" {
|
||||
if (sourceId.startsWith("SET")) return "SET";
|
||||
if (sourceId.startsWith("VAT")) return "VAT";
|
||||
if (sourceId.startsWith("CLS")) return "CLS";
|
||||
return "OTHER";
|
||||
}
|
||||
|
||||
const SETTLEMENT_QUERIES = [
|
||||
"Show why payment recorded but settlement for account 60 is still open.",
|
||||
"Account 62: payment posted, settlement closure is missing.",
|
||||
"Find settlement tails for account 60 where payment did not close chain.",
|
||||
"Bank and settlements 60/62: where link to settlement is broken.",
|
||||
"Why does account 60 keep open settlement after payment record.",
|
||||
"Account 62 settlement problem: payment done, closure not reached.",
|
||||
"Detect symptom where payment exists but settlement remains open on 60.",
|
||||
"Find lifecycle gap in payment to settlement for account 62.",
|
||||
"60-62 settlement chain has residual tail after payment.",
|
||||
"Locate unresolved settlement after bank payment on account 60."
|
||||
];
|
||||
|
||||
const VAT_QUERIES = [
|
||||
"VAT check: source document exists but invoice link is missing on account 68.",
|
||||
"Account 19 VAT chain: document to register to book is broken.",
|
||||
"Find VAT symptom where invoice linked but book entry was not generated.",
|
||||
"Show VAT lifecycle gaps for account 68 in document-register-book flow.",
|
||||
"VAT deduction issue on 19: source document present but deduction not posted.",
|
||||
"Find broken invoice to VAT register relation for account 68.",
|
||||
"VAT problem-first: document exists, register is present, book entry missing.",
|
||||
"Locate VAT residual issue where deduction chain is incomplete on 19.",
|
||||
"VAT 68: invoice and register mismatch in purchase/sales book.",
|
||||
"Detect VAT symptom with broken doc-register-book chain for account 68."
|
||||
];
|
||||
|
||||
const CLOSE_QUERIES = [
|
||||
"Month close: costs on accounts 20 and 44 are not allocated, residuals remain.",
|
||||
"Period close problem for 20/44: allocation rules unresolved.",
|
||||
"Find close lifecycle gap where costs accumulated but close operation fails 20 44.",
|
||||
"Account 20 and 44 month close symptom: residuals are not zero.",
|
||||
"Show period close issue when costs are accumulated but not distributed 20/44.",
|
||||
"Close operation run for 20 and 44 leaves unexplained residuals.",
|
||||
"Detect month close break in costs allocation chain on 20/44.",
|
||||
"Period close 20-44: allocation exists but residual tail remains.",
|
||||
"Find cost close mismatch: costs accumulated, close not completed 20 and 44.",
|
||||
"Month close domain check for accounts 20 and 44 with unresolved residuals."
|
||||
];
|
||||
|
||||
const cases: RegressionCase[] = [
|
||||
...SETTLEMENT_QUERIES.map((query, idx) => ({ case_id: `SET-${String(idx + 1).padStart(2, "0")}`, expected_prefix: "SET" as const, query })),
|
||||
...VAT_QUERIES.map((query, idx) => ({ case_id: `VAT-${String(idx + 1).padStart(2, "0")}`, expected_prefix: "VAT" as const, query })),
|
||||
...CLOSE_QUERIES.map((query, idx) => ({ case_id: `CLS-${String(idx + 1).padStart(2, "0")}`, expected_prefix: "CLS" as const, query }))
|
||||
];
|
||||
|
||||
const root = createSnapshotRoot(createDataset());
|
||||
const layer = new AssistantDataLayer(root);
|
||||
|
||||
let bad = 0;
|
||||
for (const c of cases) {
|
||||
const r = layer.executeRoute("store_feature_risk", c.query);
|
||||
const ids = (r.items as Array<any>).map((x) => String(x.source_id ?? ""));
|
||||
const top1 = ids[0] ?? "<empty>";
|
||||
const ok = top1 !== "<empty>" && resolvePrefixFromId(top1) === c.expected_prefix;
|
||||
if (!ok) {
|
||||
bad += 1;
|
||||
const guard = (r.summary as any)?.domain_purity_guard;
|
||||
console.log(`${c.case_id} FAIL top1=${top1} expected=${c.expected_prefix} status=${r.status}`);
|
||||
console.log(` query=${c.query}`);
|
||||
console.log(` ids=${ids.join(",")}`);
|
||||
console.log(` card=${guard?.domain_card_id} source_allowed=${guard?.source_selection_allowed} ranking_allowed=${guard?.ranking_allowed} promotion_allowed=${guard?.promotion_allowed}`);
|
||||
}
|
||||
}
|
||||
console.log(`bad=${bad}`);
|
||||
@@ -0,0 +1,67 @@
|
||||
import path from "path";
|
||||
import request from "supertest";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const originalLog = console.log;
|
||||
console.log = () => {};
|
||||
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = "1";
|
||||
|
||||
const { createApp } = await import("../src/server.ts");
|
||||
const app = createApp();
|
||||
|
||||
const baselinePath = path.resolve(
|
||||
process.cwd(),
|
||||
"../docs/runs/2026-03-27_Stage_04_Wave_07_P0_Eval_Harness_Formal_Product_Acceptance/artifacts/current_report.json"
|
||||
);
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
compare_with_report_file: baselinePath,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
console.log = originalLog;
|
||||
|
||||
if (response.status !== 200) {
|
||||
console.error(`status=${response.status}`);
|
||||
console.error(JSON.stringify(response.body, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const report = response.body.report;
|
||||
originalLog(
|
||||
JSON.stringify(
|
||||
{
|
||||
run_id: report?.run_id,
|
||||
verdict: report?.acceptance_gate?.verdict,
|
||||
metrics: report?.metrics?.raw,
|
||||
assertions: report?.assertions,
|
||||
run_report_json_path: report?.artifacts?.run_report_json_path,
|
||||
run_report_md_path: report?.artifacts?.run_report_md_path,
|
||||
comparison_json_path: report?.comparison?.artifacts?.comparison_report_json_path,
|
||||
comparison_md_path: report?.comparison?.artifacts?.comparison_report_md_path,
|
||||
comparison_verdict_delta: report?.comparison?.verdict_delta
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import path from "path";
|
||||
import request from "supertest";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const originalLog = console.log;
|
||||
console.log = () => {};
|
||||
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = "1";
|
||||
|
||||
const { createApp } = await import("../src/server.ts");
|
||||
const app = createApp();
|
||||
|
||||
const baselinePath = path.resolve(
|
||||
process.cwd(),
|
||||
"../docs/runs/2026-03-27_Stage_04_Wave_08_Route_Correctness_Recovery_Domain_Purity_Closure/artifacts/current_report.json"
|
||||
);
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_2.json",
|
||||
compare_with_report_file: baselinePath,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
console.log = originalLog;
|
||||
|
||||
if (response.status !== 200) {
|
||||
console.error(`status=${response.status}`);
|
||||
console.error(JSON.stringify(response.body, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const report = response.body.report;
|
||||
originalLog(
|
||||
JSON.stringify(
|
||||
{
|
||||
run_id: report?.run_id,
|
||||
acceptance_verdict: report?.acceptance_gate?.verdict,
|
||||
baseline_stability_verdict: report?.baseline_stability_gate?.verdict,
|
||||
metrics: report?.metrics?.raw,
|
||||
quality_gap_metrics: report?.quality_gap_metrics?.raw,
|
||||
assertions: report?.assertions,
|
||||
run_report_json_path: report?.artifacts?.run_report_json_path,
|
||||
run_report_md_path: report?.artifacts?.run_report_md_path,
|
||||
comparison_json_path: report?.comparison?.artifacts?.comparison_report_json_path,
|
||||
comparison_md_path: report?.comparison?.artifacts?.comparison_report_md_path,
|
||||
comparison_verdict_delta: report?.comparison?.verdict_delta
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-001",
|
||||
"raw_question": "По каким поставщикам на конец июня не бьются взаиморасчеты, покажи документы, оплаты и хвосты.",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-002",
|
||||
"raw_question": "Сделай рейтинг самых рисковых хвостов перед закрытием периода за июнь.",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-003",
|
||||
"raw_question": "Покажи документ по номеру 000123 и строку проводки, нужен точный source-of-record.",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-004",
|
||||
"raw_question": "По 97 счету проверь, где возможна ошибка дат начала и окончания списания.",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-005",
|
||||
"raw_question": "Есть ли аномальные материалы на счете 10, которые зависли и выглядят нелогично?",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-006",
|
||||
"raw_question": "По каким реализациям 90/62 хвосты не закрылись оплатой, разложи по цепочке документов.",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-007",
|
||||
"raw_question": "Что у нас выглядит самым проблемным перед закрытием июня, если смотреть на компанию в целом?",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-009",
|
||||
"raw_question": "Где у нас пахнет ручной ошибкой по июню?",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-010",
|
||||
"raw_question": "Какое сальдо по 68.02 за июнь?",
|
||||
"expected": {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"case_id": "NQ-1774273974528",
|
||||
"raw_question": "По каким поставщикам не бьются взаиморасчеты по 60 счету, и разложи это по документам, оплатам и закрывающим?",
|
||||
"expected": {
|
||||
@@ -13,4 +13,4 @@
|
||||
],
|
||||
"expected_output_shape": "reconciliation_report"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval--DLjm5dCSP",
|
||||
"timestamp": "2026-03-26T14:59:09.726Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -133,4 +133,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval--JtoGUT4Aw",
|
||||
"timestamp": "2026-03-24T10:06:13.697Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -132,4 +132,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval--lulgUEKkp",
|
||||
"timestamp": "2026-03-26T15:04:48.691Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -108,4 +108,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-0M5PNZp9FY",
|
||||
"timestamp": "2026-03-26T12:41:47.018Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -132,4 +132,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-0sMxWQCA77",
|
||||
"timestamp": "2026-03-26T11:01:27.248Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -108,4 +108,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-1I2yuUiev6",
|
||||
"timestamp": "2026-03-27T22:25:33.046Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "oKL_tEdXJzxMJ5",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "rwbRlpRZD5Ct7i",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-1ZO6LtR6Re",
|
||||
"timestamp": "2026-03-25T19:37:38.656Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -132,4 +132,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-2ui-ojyBuu",
|
||||
"timestamp": "2026-03-27T23:00:27.280Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "OqFf4QS6Y8IeVm",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "qPu-4JESf8r8GX",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "GEf3pwXKJWXPD9",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-3sC2svvJlI",
|
||||
"timestamp": "2026-03-23T14:33:51.265Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -1066,4 +1066,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-3t1L3QY0wE",
|
||||
"timestamp": "2026-03-26T14:29:33.451Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -108,4 +108,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-43n6WEu5ka",
|
||||
"timestamp": "2026-03-27T20:07:04.463Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "aKVQOW7oZXDoh-",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "d31RtLJyc_1TF8",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "l_EscYF5jFhLo4",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-4Jd4YjGjIL",
|
||||
"timestamp": "2026-03-26T12:55:04.731Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -108,4 +108,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-4_ULiatTGc",
|
||||
"timestamp": "2026-03-27T19:54:03.827Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "fg64eigCctlx0H",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "AzrqlzdcTcOlKI",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-4xS4Pyr7Uv",
|
||||
"timestamp": "2026-03-25T20:06:37.364Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -132,4 +132,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-68KSDPHUoo",
|
||||
"timestamp": "2026-03-25T19:04:30.554Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -132,4 +132,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-6O6YT6OLbm",
|
||||
"timestamp": "2026-03-27T22:26:40.837Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "YG1XhXLLNtyRXi",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "UsnLAYPdib2TtP",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "OSe2GzHkTiUk_s",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-73H4_ECGyy",
|
||||
"timestamp": "2026-03-24T10:05:43.126Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -132,4 +132,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-7I74UBMAdj",
|
||||
"timestamp": "2026-03-27T19:06:07.206Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "FJx9pQRvK9stcY",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "Har9wm_uOvFuMw",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-7MQmPgo3_O",
|
||||
"timestamp": "2026-03-23T18:38:44.797Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -100,4 +100,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-7awjIpz1KA",
|
||||
"timestamp": "2026-03-26T15:06:11.446Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -133,4 +133,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-86xJU1J7RH",
|
||||
"timestamp": "2026-03-26T12:55:30.782Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -132,4 +132,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-8HiKD7tzkR",
|
||||
"timestamp": "2026-03-26T14:55:02.675Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -108,4 +108,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
{
|
||||
"run_id": "eval-94ypeytPFD",
|
||||
"timestamp": "2026-03-26T14:48:14.207Z",
|
||||
"mode": "single-pass-strict",
|
||||
@@ -108,4 +108,4 @@
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user