Этап 4 / Волна 10: корректировка settlement-кейса — доменная фиксация синтеза, честное покрытие, удержание фокуса / Этап 4 / Волна 11: бизнес-якоря, доменное заземление и устранение утечки дебага
This commit is contained in:
@@ -11,6 +11,14 @@ function toBooleanFlag(value: string | undefined, defaultValue: boolean): boolea
|
||||
return !(lowered === "0" || lowered === "false" || lowered === "off" || lowered === "no");
|
||||
}
|
||||
|
||||
function toNumberFlag(value: string | undefined, defaultValue: number): number {
|
||||
if (!value || value.trim() === "") {
|
||||
return defaultValue;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : defaultValue;
|
||||
}
|
||||
|
||||
export const PORT = Number(process.env.PORT ?? 8787);
|
||||
export const TIMEZONE = process.env.TZ_FALLBACK ?? "Europe/Moscow";
|
||||
export const DEFAULT_OPENAI_BASE_URL = process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1";
|
||||
@@ -45,7 +53,7 @@ export const FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = toBooleanFlag(
|
||||
);
|
||||
export const FEATURE_ASSISTANT_ANSWER_POLICY_V11 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1,
|
||||
@@ -53,15 +61,15 @@ export const FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = toBooleanFlag(
|
||||
);
|
||||
export const FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_STAGE2_EVAL_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1,
|
||||
@@ -69,12 +77,27 @@ export const FEATURE_ASSISTANT_STAGE2_EVAL_V1 = toBooleanFlag(
|
||||
);
|
||||
export const FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1,
|
||||
false
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1,
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1,
|
||||
true
|
||||
);
|
||||
export const FEATURE_ASSISTANT_MCP_RUNTIME_V1 = toBooleanFlag(
|
||||
process.env.FEATURE_ASSISTANT_MCP_RUNTIME_V1,
|
||||
false
|
||||
);
|
||||
export const ASSISTANT_MCP_PROXY_URL = (process.env.ASSISTANT_MCP_PROXY_URL ?? "http://127.0.0.1:6003").replace(
|
||||
/\/+$/,
|
||||
""
|
||||
);
|
||||
export const ASSISTANT_MCP_CHANNEL = process.env.ASSISTANT_MCP_CHANNEL ?? "default";
|
||||
export const ASSISTANT_MCP_TIMEOUT_MS = toNumberFlag(process.env.ASSISTANT_MCP_TIMEOUT_MS, 1200);
|
||||
export const ASSISTANT_MCP_LIVE_LIMIT = Math.max(1, Math.trunc(toNumberFlag(process.env.ASSISTANT_MCP_LIVE_LIMIT, 24)));
|
||||
|
||||
export const DATA_DIR = process.env.DATA_DIR ?? path.resolve(MODULE_ROOT, "data");
|
||||
export const TRACES_DIR = path.resolve(DATA_DIR, "traces");
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
import {
|
||||
P0_DEFAULT_ACCEPTANCE_THRESHOLDS,
|
||||
P0_DEFAULT_QUALITY_GAP_THRESHOLDS,
|
||||
type P0AcceptanceThresholds,
|
||||
type P0EvalMetricVector,
|
||||
type P0QualityGapMetricVector,
|
||||
type P0QualityGapThresholds
|
||||
} from "./p0_metric_definitions";
|
||||
|
||||
export type P0AcceptanceVerdict = "P0_ACCEPTED" | "P0_ACCEPTED_WITH_LIMITATIONS" | "P0_NOT_ACCEPTED";
|
||||
export type P0BaselineStabilityVerdict = "P0_BASELINE_STABLE" | "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS";
|
||||
|
||||
interface MetricCheck {
|
||||
metric: keyof P0EvalMetricVector;
|
||||
value: number;
|
||||
threshold: number;
|
||||
comparator: ">=" | "<=";
|
||||
passed: boolean;
|
||||
severity: "blocking" | "quality";
|
||||
}
|
||||
|
||||
interface QualityGapMetricCheck {
|
||||
metric: keyof P0QualityGapMetricVector;
|
||||
value: number;
|
||||
threshold: number;
|
||||
comparator: ">=" | "<=";
|
||||
passed: boolean;
|
||||
}
|
||||
|
||||
export interface P0AcceptanceResult {
|
||||
verdict: P0AcceptanceVerdict;
|
||||
passed_all: boolean;
|
||||
blocking_failures: MetricCheck[];
|
||||
quality_failures: MetricCheck[];
|
||||
metric_checks: MetricCheck[];
|
||||
rationale: string[];
|
||||
}
|
||||
|
||||
export interface P0BaselineStabilityResult {
|
||||
verdict: P0BaselineStabilityVerdict;
|
||||
baseline_integrity_passed: boolean;
|
||||
quality_gaps_open: boolean;
|
||||
blocking_regressions: MetricCheck[];
|
||||
legacy_quality_failures: MetricCheck[];
|
||||
quality_gap_failures: QualityGapMetricCheck[];
|
||||
quality_gap_checks: QualityGapMetricCheck[];
|
||||
rationale: string[];
|
||||
}
|
||||
|
||||
function buildMetricChecks(input: {
|
||||
metrics: P0EvalMetricVector;
|
||||
thresholds: P0AcceptanceThresholds;
|
||||
}): MetricCheck[] {
|
||||
const { metrics, thresholds } = input;
|
||||
return [
|
||||
{
|
||||
metric: "problem_first_answer_rate",
|
||||
value: metrics.problem_first_answer_rate,
|
||||
threshold: thresholds.problem_first_answer_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.problem_first_answer_rate >= thresholds.problem_first_answer_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "entity_leakage_rate",
|
||||
value: metrics.entity_leakage_rate,
|
||||
threshold: thresholds.entity_leakage_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.entity_leakage_rate <= thresholds.entity_leakage_rate_max,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "route_correctness_rate",
|
||||
value: metrics.route_correctness_rate,
|
||||
threshold: thresholds.route_correctness_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.route_correctness_rate >= thresholds.route_correctness_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "domain_purity_rate",
|
||||
value: metrics.domain_purity_rate,
|
||||
threshold: thresholds.domain_purity_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.domain_purity_rate >= thresholds.domain_purity_rate_min,
|
||||
severity: "blocking"
|
||||
},
|
||||
{
|
||||
metric: "mechanism_coherence_score",
|
||||
value: metrics.mechanism_coherence_score,
|
||||
threshold: thresholds.mechanism_coherence_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.mechanism_coherence_score >= thresholds.mechanism_coherence_score_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "accountant_actionability_score",
|
||||
value: metrics.accountant_actionability_score,
|
||||
threshold: thresholds.accountant_actionability_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.accountant_actionability_score >= thresholds.accountant_actionability_score_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "limitation_honesty_rate",
|
||||
value: metrics.limitation_honesty_rate,
|
||||
threshold: thresholds.limitation_honesty_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.limitation_honesty_rate >= thresholds.limitation_honesty_rate_min,
|
||||
severity: "quality"
|
||||
},
|
||||
{
|
||||
metric: "top_problem_unit_match_rate",
|
||||
value: metrics.top_problem_unit_match_rate,
|
||||
threshold: thresholds.top_problem_unit_match_rate_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.top_problem_unit_match_rate >= thresholds.top_problem_unit_match_rate_min,
|
||||
severity: "quality"
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
function toFailureLine(check: MetricCheck): string {
|
||||
return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
|
||||
}
|
||||
|
||||
function toQualityGapFailureLine(check: QualityGapMetricCheck): string {
|
||||
return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
|
||||
}
|
||||
|
||||
function buildQualityGapChecks(input: {
|
||||
metrics: P0QualityGapMetricVector;
|
||||
thresholds: P0QualityGapThresholds;
|
||||
}): QualityGapMetricCheck[] {
|
||||
const { metrics, thresholds } = input;
|
||||
return [
|
||||
{
|
||||
metric: "generic_explanation_rate",
|
||||
value: metrics.generic_explanation_rate,
|
||||
threshold: thresholds.generic_explanation_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.generic_explanation_rate <= thresholds.generic_explanation_rate_max
|
||||
},
|
||||
{
|
||||
metric: "false_confidence_rate",
|
||||
value: metrics.false_confidence_rate,
|
||||
threshold: thresholds.false_confidence_rate_max,
|
||||
comparator: "<=",
|
||||
passed: metrics.false_confidence_rate <= thresholds.false_confidence_rate_max
|
||||
},
|
||||
{
|
||||
metric: "mechanism_specificity_score",
|
||||
value: metrics.mechanism_specificity_score,
|
||||
threshold: thresholds.mechanism_specificity_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.mechanism_specificity_score >= thresholds.mechanism_specificity_score_min
|
||||
},
|
||||
{
|
||||
metric: "followup_context_retention_score",
|
||||
value: metrics.followup_context_retention_score,
|
||||
threshold: thresholds.followup_context_retention_score_min,
|
||||
comparator: ">=",
|
||||
passed: metrics.followup_context_retention_score >= thresholds.followup_context_retention_score_min
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
export function evaluateP0AcceptanceGate(input: {
|
||||
metrics: P0EvalMetricVector;
|
||||
thresholds?: Partial<P0AcceptanceThresholds>;
|
||||
}): P0AcceptanceResult {
|
||||
const thresholds: P0AcceptanceThresholds = {
|
||||
...P0_DEFAULT_ACCEPTANCE_THRESHOLDS,
|
||||
...(input.thresholds ?? {})
|
||||
};
|
||||
const checks = buildMetricChecks({
|
||||
metrics: input.metrics,
|
||||
thresholds
|
||||
});
|
||||
const blockingFailures = checks.filter((item) => item.severity === "blocking" && !item.passed);
|
||||
const qualityFailures = checks.filter((item) => item.severity === "quality" && !item.passed);
|
||||
|
||||
let verdict: P0AcceptanceVerdict = "P0_NOT_ACCEPTED";
|
||||
if (blockingFailures.length === 0 && qualityFailures.length === 0) {
|
||||
verdict = "P0_ACCEPTED";
|
||||
} else if (blockingFailures.length === 0) {
|
||||
verdict = "P0_ACCEPTED_WITH_LIMITATIONS";
|
||||
}
|
||||
|
||||
const rationale: string[] = [];
|
||||
if (blockingFailures.length > 0) {
|
||||
rationale.push("Blocking metrics failed:");
|
||||
rationale.push(...blockingFailures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (qualityFailures.length > 0) {
|
||||
rationale.push("Quality metrics failed:");
|
||||
rationale.push(...qualityFailures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (rationale.length === 0) {
|
||||
rationale.push("All acceptance thresholds are satisfied.");
|
||||
}
|
||||
|
||||
return {
|
||||
verdict,
|
||||
passed_all: checks.every((item) => item.passed),
|
||||
blocking_failures: blockingFailures,
|
||||
quality_failures: qualityFailures,
|
||||
metric_checks: checks,
|
||||
rationale
|
||||
};
|
||||
}
|
||||
|
||||
export function evaluateP0BaselineStabilityGate(input: {
|
||||
metrics: P0EvalMetricVector;
|
||||
qualityGapMetrics: P0QualityGapMetricVector;
|
||||
acceptanceThresholds?: Partial<P0AcceptanceThresholds>;
|
||||
qualityGapThresholds?: Partial<P0QualityGapThresholds>;
|
||||
}): P0BaselineStabilityResult {
|
||||
const acceptance = evaluateP0AcceptanceGate({
|
||||
metrics: input.metrics,
|
||||
thresholds: input.acceptanceThresholds
|
||||
});
|
||||
const qualityGapThresholds: P0QualityGapThresholds = {
|
||||
...P0_DEFAULT_QUALITY_GAP_THRESHOLDS,
|
||||
...(input.qualityGapThresholds ?? {})
|
||||
};
|
||||
const qualityGapChecks = buildQualityGapChecks({
|
||||
metrics: input.qualityGapMetrics,
|
||||
thresholds: qualityGapThresholds
|
||||
});
|
||||
const qualityGapFailures = qualityGapChecks.filter((item) => !item.passed);
|
||||
|
||||
const baselineIntegrityPassed = acceptance.blocking_failures.length === 0;
|
||||
const qualityGapsOpen = acceptance.quality_failures.length > 0 || qualityGapFailures.length > 0;
|
||||
const verdict: P0BaselineStabilityVerdict =
|
||||
baselineIntegrityPassed && !qualityGapsOpen ? "P0_BASELINE_STABLE" : "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS";
|
||||
|
||||
const rationale: string[] = [];
|
||||
if (!baselineIntegrityPassed) {
|
||||
rationale.push("Blocking baseline regressions detected:");
|
||||
rationale.push(...acceptance.blocking_failures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (acceptance.quality_failures.length > 0) {
|
||||
rationale.push("Legacy quality thresholds still below full acceptance:");
|
||||
rationale.push(...acceptance.quality_failures.map((item) => `- ${toFailureLine(item)}`));
|
||||
}
|
||||
if (qualityGapFailures.length > 0) {
|
||||
rationale.push("Wave 9 quality-gap thresholds still open:");
|
||||
rationale.push(...qualityGapFailures.map((item) => `- ${toQualityGapFailureLine(item)}`));
|
||||
}
|
||||
if (rationale.length === 0) {
|
||||
rationale.push("Baseline is stable and Wave 9 quality-gap thresholds are satisfied.");
|
||||
}
|
||||
|
||||
return {
|
||||
verdict,
|
||||
baseline_integrity_passed: baselineIntegrityPassed,
|
||||
quality_gaps_open: qualityGapsOpen,
|
||||
blocking_regressions: acceptance.blocking_failures,
|
||||
legacy_quality_failures: acceptance.quality_failures,
|
||||
quality_gap_failures: qualityGapFailures,
|
||||
quality_gap_checks: qualityGapChecks,
|
||||
rationale
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,425 @@
|
||||
import type { ProblemUnitType } from "../types/stage2ProblemUnits";
|
||||
|
||||
export type P0DomainId = "settlements_60_62" | "vat_document_register_book" | "month_close_costs_20_44";
|
||||
|
||||
export type P0QueryClass =
|
||||
| "symptom_first"
|
||||
| "lifecycle_first"
|
||||
| "causal_query"
|
||||
| "mixed_ambiguity"
|
||||
| "followup_investigation"
|
||||
| "noisy_input"
|
||||
| "translit_noisy"
|
||||
| "multi_intent";
|
||||
|
||||
export type P0RouteExpected = "hybrid_store_plus_live" | "store_feature_risk" | "store_canonical" | "batch_refresh_then_store" | "live_mcp_drilldown" | "no_route";
|
||||
|
||||
export interface P0ExpectedAnnotations {
|
||||
domain_expected: P0DomainId;
|
||||
route_expected: P0RouteExpected | P0RouteExpected[];
|
||||
problem_unit_type_expected: ProblemUnitType;
|
||||
mechanism_class_expected: string;
|
||||
evidence_expectation: string;
|
||||
must_show_limitation: boolean;
|
||||
must_not_leak_internal: boolean;
|
||||
first_check_action_expected: string;
|
||||
}
|
||||
|
||||
export interface P0EvalCase {
|
||||
case_id: string;
|
||||
domain: P0DomainId;
|
||||
user_query_raw: string;
|
||||
query_class: P0QueryClass;
|
||||
expected_problem_unit_type: ProblemUnitType;
|
||||
expected_mechanism_summary: string;
|
||||
expected_first_check: string;
|
||||
must_include_limitations: boolean;
|
||||
forbidden_leakage_tokens: string[];
|
||||
followup_seed_query?: string;
|
||||
followup_expected_context_tokens?: string[];
|
||||
notes: string;
|
||||
expected_annotations: P0ExpectedAnnotations;
|
||||
}
|
||||
|
||||
export interface P0EvalCorpusFile {
|
||||
suite_id: string;
|
||||
suite_version: string;
|
||||
schema_version: string;
|
||||
scenario_count: number;
|
||||
case_ids: string[];
|
||||
cases: P0EvalCase[];
|
||||
}
|
||||
|
||||
export interface P0EvalMetricVector {
|
||||
problem_first_answer_rate: number;
|
||||
mechanism_coherence_score: number;
|
||||
entity_leakage_rate: number;
|
||||
accountant_actionability_score: number;
|
||||
route_correctness_rate: number;
|
||||
domain_purity_rate: number;
|
||||
limitation_honesty_rate: number;
|
||||
top_problem_unit_match_rate: number;
|
||||
}
|
||||
|
||||
export type P0MetricName = keyof P0EvalMetricVector;
|
||||
|
||||
export interface P0QualityGapMetricVector {
|
||||
generic_explanation_rate: number;
|
||||
false_confidence_rate: number;
|
||||
mechanism_specificity_score: number;
|
||||
followup_context_retention_score: number;
|
||||
}
|
||||
|
||||
export type P0QualityGapMetricName = keyof P0QualityGapMetricVector;
|
||||
|
||||
export interface P0MetricDefinition {
|
||||
description: string;
|
||||
unit: "rate" | "score";
|
||||
direction: "higher_is_better" | "lower_is_better";
|
||||
}
|
||||
|
||||
export interface P0QualityGapMetricDefinition {
|
||||
description: string;
|
||||
unit: "rate" | "score";
|
||||
direction: "higher_is_better" | "lower_is_better";
|
||||
}
|
||||
|
||||
export const P0_METRIC_DEFINITIONS: Record<P0MetricName, P0MetricDefinition> = {
|
||||
problem_first_answer_rate: {
|
||||
description: "Share of cases where answer is problem-first and avoids entity-dump framing.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
mechanism_coherence_score: {
|
||||
description: "Average 0..5 score for mechanism clarity and alignment with expected defect class.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
entity_leakage_rate: {
|
||||
description: "Share of cases with internal/technical leakage in user-facing answer.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
accountant_actionability_score: {
|
||||
description: "Average 0..5 score for first-check concreteness and practical accounting actionability.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
route_correctness_rate: {
|
||||
description: "Share of cases routed to expected path class for query type.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
domain_purity_rate: {
|
||||
description: "Share of cases where top-3 retrieval stays inside expected P0 domain boundary.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
limitation_honesty_rate: {
|
||||
description: "Share of cases where mandatory limitations are explicit when evidence is incomplete.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
top_problem_unit_match_rate: {
|
||||
description: "Share of cases where top problem unit type matches expected annotation.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
}
|
||||
};
|
||||
|
||||
export const P0_QUALITY_GAP_METRIC_DEFINITIONS: Record<P0QualityGapMetricName, P0QualityGapMetricDefinition> = {
|
||||
generic_explanation_rate: {
|
||||
description: "Share of cases where direct answer stays generic and mechanism/actionability specificity is insufficient.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
false_confidence_rate: {
|
||||
description: "Share of cases with overconfident framing not supported by evidence/limitation state.",
|
||||
unit: "rate",
|
||||
direction: "lower_is_better"
|
||||
},
|
||||
mechanism_specificity_score: {
|
||||
description: "Average 0..5 score of mechanism-level specificity in accountant-facing answer.",
|
||||
unit: "score",
|
||||
direction: "higher_is_better"
|
||||
},
|
||||
followup_context_retention_score: {
|
||||
description: "Rate of correct context retention in follow-up investigation cases.",
|
||||
unit: "rate",
|
||||
direction: "higher_is_better"
|
||||
}
|
||||
};
|
||||
|
||||
export const P0_DEFAULT_FORBIDDEN_LEAKAGE_TOKENS = [
|
||||
"graph_",
|
||||
"domain_scope",
|
||||
"relation_patterns",
|
||||
"account_scope",
|
||||
"semantic_profile",
|
||||
"route",
|
||||
"profile",
|
||||
"store_canonical",
|
||||
"hybrid_store_plus_live",
|
||||
"deterministic_v2",
|
||||
"candidate_evidence",
|
||||
"problem_unit",
|
||||
"raw_entities",
|
||||
"graph_domain_scope"
|
||||
];
|
||||
|
||||
export interface P0AcceptanceThresholds {
|
||||
problem_first_answer_rate_min: number;
|
||||
mechanism_coherence_score_min: number;
|
||||
entity_leakage_rate_max: number;
|
||||
accountant_actionability_score_min: number;
|
||||
route_correctness_rate_min: number;
|
||||
domain_purity_rate_min: number;
|
||||
limitation_honesty_rate_min: number;
|
||||
top_problem_unit_match_rate_min: number;
|
||||
}
|
||||
|
||||
export interface P0QualityGapThresholds {
|
||||
generic_explanation_rate_max: number;
|
||||
false_confidence_rate_max: number;
|
||||
mechanism_specificity_score_min: number;
|
||||
followup_context_retention_score_min: number;
|
||||
}
|
||||
|
||||
export const P0_DEFAULT_ACCEPTANCE_THRESHOLDS: P0AcceptanceThresholds = {
|
||||
problem_first_answer_rate_min: 0.8,
|
||||
mechanism_coherence_score_min: 3.5,
|
||||
entity_leakage_rate_max: 0.08,
|
||||
accountant_actionability_score_min: 3.5,
|
||||
route_correctness_rate_min: 0.85,
|
||||
domain_purity_rate_min: 0.9,
|
||||
limitation_honesty_rate_min: 0.9,
|
||||
top_problem_unit_match_rate_min: 0.65
|
||||
};
|
||||
|
||||
export const P0_DEFAULT_QUALITY_GAP_THRESHOLDS: P0QualityGapThresholds = {
|
||||
generic_explanation_rate_max: 0.2,
|
||||
false_confidence_rate_max: 0.12,
|
||||
mechanism_specificity_score_min: 3.0,
|
||||
followup_context_retention_score_min: 0.75
|
||||
};
|
||||
|
||||
function isProblemUnitType(value: unknown): value is ProblemUnitType {
|
||||
return (
|
||||
value === "document_conflict" ||
|
||||
value === "broken_chain_segment" ||
|
||||
value === "lifecycle_anomaly_node" ||
|
||||
value === "unresolved_settlement_cluster" ||
|
||||
value === "period_risk_cluster" ||
|
||||
value === "cross_branch_inconsistency_cluster"
|
||||
);
|
||||
}
|
||||
|
||||
function isP0DomainId(value: unknown): value is P0DomainId {
|
||||
return value === "settlements_60_62" || value === "vat_document_register_book" || value === "month_close_costs_20_44";
|
||||
}
|
||||
|
||||
function isP0QueryClass(value: unknown): value is P0QueryClass {
|
||||
return (
|
||||
value === "symptom_first" ||
|
||||
value === "lifecycle_first" ||
|
||||
value === "causal_query" ||
|
||||
value === "mixed_ambiguity" ||
|
||||
value === "followup_investigation" ||
|
||||
value === "noisy_input" ||
|
||||
value === "translit_noisy" ||
|
||||
value === "multi_intent"
|
||||
);
|
||||
}
|
||||
|
||||
function isRouteExpected(value: unknown): value is P0RouteExpected {
|
||||
return (
|
||||
value === "hybrid_store_plus_live" ||
|
||||
value === "store_feature_risk" ||
|
||||
value === "store_canonical" ||
|
||||
value === "batch_refresh_then_store" ||
|
||||
value === "live_mcp_drilldown" ||
|
||||
value === "no_route"
|
||||
);
|
||||
}
|
||||
|
||||
function assertString(value: unknown, fieldPath: string): string {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': non-empty string is required.`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function assertBoolean(value: unknown, fieldPath: string): boolean {
|
||||
if (typeof value !== "boolean") {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': boolean is required.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertStringArray(value: unknown, fieldPath: string): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': string[] is required.`);
|
||||
}
|
||||
const output = value.map((item, index) => assertString(item, `${fieldPath}[${index}]`));
|
||||
return output;
|
||||
}
|
||||
|
||||
function assertRouteExpected(value: unknown, fieldPath: string): P0RouteExpected | P0RouteExpected[] {
|
||||
if (Array.isArray(value)) {
|
||||
const routes = value.map((item, index) => {
|
||||
if (!isRouteExpected(item)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}[${index}]': unsupported route '${String(item)}'.`);
|
||||
}
|
||||
return item;
|
||||
});
|
||||
if (routes.length === 0) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': route list must not be empty.`);
|
||||
}
|
||||
return routes;
|
||||
}
|
||||
if (!isRouteExpected(value)) {
|
||||
throw new Error(`Invalid corpus field '${fieldPath}': unsupported route '${String(value)}'.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseCase(raw: unknown, index: number): P0EvalCase {
|
||||
if (!raw || typeof raw !== "object") {
|
||||
throw new Error(`Invalid corpus case at index ${index}: object is required.`);
|
||||
}
|
||||
const row = raw as Record<string, unknown>;
|
||||
const expected = row.expected_annotations;
|
||||
if (!expected || typeof expected !== "object") {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations is required.`);
|
||||
}
|
||||
const annotations = expected as Record<string, unknown>;
|
||||
|
||||
const domain = row.domain;
|
||||
if (!isP0DomainId(domain)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported domain '${String(domain)}'.`);
|
||||
}
|
||||
const queryClass = row.query_class;
|
||||
if (!isP0QueryClass(queryClass)) {
|
||||
throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported query_class '${String(queryClass)}'.`);
|
||||
}
|
||||
const expectedProblemType = row.expected_problem_unit_type;
|
||||
if (!isProblemUnitType(expectedProblemType)) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': expected_problem_unit_type '${String(expectedProblemType)}' is not supported.`
|
||||
);
|
||||
}
|
||||
const annotationProblemType = annotations.problem_unit_type_expected;
|
||||
if (!isProblemUnitType(annotationProblemType)) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.problem_unit_type_expected '${String(annotationProblemType)}' is not supported.`
|
||||
);
|
||||
}
|
||||
const annotationDomain = annotations.domain_expected;
|
||||
if (!isP0DomainId(annotationDomain)) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.domain_expected '${String(annotationDomain)}' is not supported.`
|
||||
);
|
||||
}
|
||||
|
||||
const followupSeedQuery =
|
||||
typeof row.followup_seed_query === "string" && row.followup_seed_query.trim() ? row.followup_seed_query.trim() : undefined;
|
||||
const followupExpectedContextTokens = Array.isArray(row.followup_expected_context_tokens)
|
||||
? assertStringArray(row.followup_expected_context_tokens, `cases[${index}].followup_expected_context_tokens`)
|
||||
: undefined;
|
||||
if (queryClass === "followup_investigation" && !followupSeedQuery) {
|
||||
throw new Error(
|
||||
`Invalid corpus case '${String(row.case_id ?? index)}': followup_investigation requires followup_seed_query.`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
case_id: assertString(row.case_id, `cases[${index}].case_id`),
|
||||
domain,
|
||||
user_query_raw: assertString(row.user_query_raw, `cases[${index}].user_query_raw`),
|
||||
query_class: queryClass,
|
||||
expected_problem_unit_type: expectedProblemType,
|
||||
expected_mechanism_summary: assertString(row.expected_mechanism_summary, `cases[${index}].expected_mechanism_summary`),
|
||||
expected_first_check: assertString(row.expected_first_check, `cases[${index}].expected_first_check`),
|
||||
must_include_limitations: assertBoolean(row.must_include_limitations, `cases[${index}].must_include_limitations`),
|
||||
forbidden_leakage_tokens: assertStringArray(row.forbidden_leakage_tokens, `cases[${index}].forbidden_leakage_tokens`),
|
||||
followup_seed_query: followupSeedQuery,
|
||||
followup_expected_context_tokens: followupExpectedContextTokens,
|
||||
notes: assertString(row.notes, `cases[${index}].notes`),
|
||||
expected_annotations: {
|
||||
domain_expected: annotationDomain,
|
||||
route_expected: assertRouteExpected(annotations.route_expected, `cases[${index}].expected_annotations.route_expected`),
|
||||
problem_unit_type_expected: annotationProblemType,
|
||||
mechanism_class_expected: assertString(
|
||||
annotations.mechanism_class_expected,
|
||||
`cases[${index}].expected_annotations.mechanism_class_expected`
|
||||
),
|
||||
evidence_expectation: assertString(annotations.evidence_expectation, `cases[${index}].expected_annotations.evidence_expectation`),
|
||||
must_show_limitation: assertBoolean(annotations.must_show_limitation, `cases[${index}].expected_annotations.must_show_limitation`),
|
||||
must_not_leak_internal: assertBoolean(
|
||||
annotations.must_not_leak_internal,
|
||||
`cases[${index}].expected_annotations.must_not_leak_internal`
|
||||
),
|
||||
first_check_action_expected: assertString(
|
||||
annotations.first_check_action_expected,
|
||||
`cases[${index}].expected_annotations.first_check_action_expected`
|
||||
)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function validateP0EvalCorpus(input: unknown): P0EvalCorpusFile {
|
||||
if (!input || typeof input !== "object") {
|
||||
throw new Error("Invalid P0 corpus: root object is required.");
|
||||
}
|
||||
const parsed = input as Record<string, unknown>;
|
||||
const casesRaw = parsed.cases;
|
||||
if (!Array.isArray(casesRaw)) {
|
||||
throw new Error("Invalid P0 corpus: cases[] is required.");
|
||||
}
|
||||
const cases = casesRaw.map((item, index) => parseCase(item, index));
|
||||
if (cases.length < 30 || cases.length > 150) {
|
||||
throw new Error(`Invalid P0 corpus: expected 30..150 cases, got ${cases.length}.`);
|
||||
}
|
||||
|
||||
const caseIds = assertStringArray(parsed.case_ids, "case_ids");
|
||||
const declaredIds = [...caseIds].sort();
|
||||
const actualIds = cases.map((item) => item.case_id).sort();
|
||||
const idsMatch = declaredIds.length === actualIds.length && declaredIds.every((value, index) => value === actualIds[index]);
|
||||
if (!idsMatch) {
|
||||
throw new Error("Invalid P0 corpus: case_ids[] does not match cases[].");
|
||||
}
|
||||
const scenarioCount = Number(parsed.scenario_count ?? 0);
|
||||
if (!Number.isFinite(scenarioCount) || scenarioCount !== cases.length) {
|
||||
throw new Error("Invalid P0 corpus: scenario_count must match cases.length.");
|
||||
}
|
||||
|
||||
const perDomain = cases.reduce<Record<P0DomainId, number>>(
|
||||
(acc, item) => {
|
||||
acc[item.domain] += 1;
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
settlements_60_62: 0,
|
||||
vat_document_register_book: 0,
|
||||
month_close_costs_20_44: 0
|
||||
}
|
||||
);
|
||||
for (const [domain, count] of Object.entries(perDomain)) {
|
||||
if (count < 10) {
|
||||
throw new Error(`Invalid P0 corpus: domain '${domain}' must contain at least 10 cases, got ${count}.`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
suite_id: assertString(parsed.suite_id, "suite_id"),
|
||||
suite_version: assertString(parsed.suite_version, "suite_version"),
|
||||
schema_version: assertString(parsed.schema_version, "schema_version"),
|
||||
scenario_count: scenarioCount,
|
||||
case_ids: caseIds,
|
||||
cases
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeExpectedRoutes(value: P0RouteExpected | P0RouteExpected[]): P0RouteExpected[] {
|
||||
return Array.isArray(value) ? value : [value];
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -214,48 +214,56 @@ export class AssistantSessionLogger {
|
||||
constructor(private readonly rootDir: string = ASSISTANT_SESSIONS_DIR) {}
|
||||
|
||||
public persistSession(session: AssistantSessionState): void {
|
||||
ensureDir(this.rootDir);
|
||||
const filePath = path.resolve(this.rootDir, `${session.session_id}.json`);
|
||||
try {
|
||||
ensureDir(this.rootDir);
|
||||
const filePath = path.resolve(this.rootDir, `${session.session_id}.json`);
|
||||
|
||||
const startedAt = session.items[0]?.created_at ?? session.updated_at;
|
||||
const userMessages = session.items.filter((item) => item.role === "user").length;
|
||||
const assistantMessages = session.items.filter((item) => item.role === "assistant").length;
|
||||
const assistantItems = session.items.filter((item) => item.role === "assistant");
|
||||
const lastAssistant = assistantItems.length > 0 ? assistantItems[assistantItems.length - 1] : null;
|
||||
const startedAt = session.items[0]?.created_at ?? session.updated_at;
|
||||
const userMessages = session.items.filter((item) => item.role === "user").length;
|
||||
const assistantMessages = session.items.filter((item) => item.role === "assistant").length;
|
||||
const assistantItems = session.items.filter((item) => item.role === "assistant");
|
||||
const lastAssistant = assistantItems.length > 0 ? assistantItems[assistantItems.length - 1] : null;
|
||||
|
||||
const traceIds = unique(session.items.map((item) => item.trace_id));
|
||||
const replyTypes = Array.from(
|
||||
new Set(
|
||||
session.items
|
||||
.map((item) => item.reply_type)
|
||||
.filter((item): item is AssistantReplyType => typeof item === "string" && item.length > 0)
|
||||
)
|
||||
);
|
||||
const turns = buildTurns(session.items);
|
||||
const traceIds = unique(session.items.map((item) => item.trace_id));
|
||||
const replyTypes = Array.from(
|
||||
new Set(
|
||||
session.items
|
||||
.map((item) => item.reply_type)
|
||||
.filter((item): item is AssistantReplyType => typeof item === "string" && item.length > 0)
|
||||
)
|
||||
);
|
||||
const turns = buildTurns(session.items);
|
||||
|
||||
const record: AssistantSessionLogRecord = {
|
||||
schema_version: "assistant_session_log_v1",
|
||||
session_id: session.session_id,
|
||||
started_at: startedAt,
|
||||
updated_at: session.updated_at,
|
||||
counters: {
|
||||
total_messages: session.items.length,
|
||||
user_messages: userMessages,
|
||||
assistant_messages: assistantMessages
|
||||
},
|
||||
trace_ids: traceIds,
|
||||
reply_types: replyTypes,
|
||||
investigation_state: session.investigation_state,
|
||||
turns,
|
||||
conversation: session.items,
|
||||
last_assistant: {
|
||||
message_id: lastAssistant?.message_id ?? null,
|
||||
reply_type: lastAssistant?.reply_type ?? null,
|
||||
trace_id: lastAssistant?.trace_id ?? null,
|
||||
created_at: lastAssistant?.created_at ?? null
|
||||
const record: AssistantSessionLogRecord = {
|
||||
schema_version: "assistant_session_log_v1",
|
||||
session_id: session.session_id,
|
||||
started_at: startedAt,
|
||||
updated_at: session.updated_at,
|
||||
counters: {
|
||||
total_messages: session.items.length,
|
||||
user_messages: userMessages,
|
||||
assistant_messages: assistantMessages
|
||||
},
|
||||
trace_ids: traceIds,
|
||||
reply_types: replyTypes,
|
||||
investigation_state: session.investigation_state,
|
||||
turns,
|
||||
conversation: session.items,
|
||||
last_assistant: {
|
||||
message_id: lastAssistant?.message_id ?? null,
|
||||
reply_type: lastAssistant?.reply_type ?? null,
|
||||
trace_id: lastAssistant?.trace_id ?? null,
|
||||
created_at: lastAssistant?.created_at ?? null
|
||||
}
|
||||
};
|
||||
|
||||
writeJsonFile(filePath, record);
|
||||
} catch (error) {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
if (code === "ENOSPC") {
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
writeJsonFile(filePath, record);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
FEATURE_ASSISTANT_STAGE2_EVAL_V1,
|
||||
REPORTS_DIR
|
||||
} from "../config";
|
||||
import { P0EvalRunner } from "../eval/p0_eval_runner";
|
||||
import type { AssistantMessageResponsePayload } from "../types/assistant";
|
||||
import type {
|
||||
EvalTarget,
|
||||
@@ -328,6 +329,98 @@ const ASSISTANT_STAGE1_COMPARISON_SCHEMA_VERSION = "assistant_stage1_eval_compar
|
||||
const DEFAULT_ASSISTANT_STAGE2_SUITE_FILE = "assistant_stage2_canonical_v0_1.json";
|
||||
const ASSISTANT_STAGE2_RUN_SCHEMA_VERSION = "assistant_stage2_eval_run_v0_1";
|
||||
const ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION = "assistant_stage2_eval_comparison_v0_1";
|
||||
const INMEM_EVAL_REPORT_PREFIX = "inmem_eval_report:";
|
||||
const INMEM_EVAL_REPORTS = new Map<string, Record<string, unknown>>();
|
||||
|
||||
function isNoSpaceError(error: unknown): boolean {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
return code === "ENOSPC";
|
||||
}
|
||||
|
||||
function tryWriteJsonFile(pathname: string, value: unknown): boolean {
|
||||
try {
|
||||
writeJsonFile(pathname, value);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function tryWriteTextFile(pathname: string, value: string): boolean {
|
||||
try {
|
||||
fs.writeFileSync(pathname, value, "utf-8");
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function putInMemoryEvalReport(report: Record<string, unknown>): string {
|
||||
const key = `${INMEM_EVAL_REPORT_PREFIX}${nanoid(12)}`;
|
||||
INMEM_EVAL_REPORTS.set(key, report);
|
||||
return key;
|
||||
}
|
||||
|
||||
function readEvalReportByRef(ref: string): { report: Record<string, unknown>; resolved_path: string } {
|
||||
if (ref.startsWith(INMEM_EVAL_REPORT_PREFIX)) {
|
||||
const report = INMEM_EVAL_REPORTS.get(ref);
|
||||
if (!report) {
|
||||
throw new Error(`In-memory eval report not found: ${ref}`);
|
||||
}
|
||||
return {
|
||||
report,
|
||||
resolved_path: ref
|
||||
};
|
||||
}
|
||||
|
||||
const resolvedPath = resolveReadablePath(ref);
|
||||
const report = JSON.parse(fs.readFileSync(resolvedPath, "utf-8")) as Record<string, unknown>;
|
||||
return {
|
||||
report,
|
||||
resolved_path: resolvedPath
|
||||
};
|
||||
}
|
||||
|
||||
function compactAssistantStage1Report(report: Record<string, unknown>): Record<string, unknown> {
|
||||
const results = Array.isArray(report.results) ? (report.results as Array<Record<string, unknown>>) : [];
|
||||
const compactResults = results.map((item) => ({
|
||||
case_id: item.case_id ?? null,
|
||||
scenario_tag: item.scenario_tag ?? null,
|
||||
accountant_usefulness_score: item.accountant_usefulness_score ?? null,
|
||||
accountant_metrics:
|
||||
typeof item.accountant_metrics === "object" && item.accountant_metrics !== null ? item.accountant_metrics : null
|
||||
}));
|
||||
return {
|
||||
...report,
|
||||
results: compactResults
|
||||
};
|
||||
}
|
||||
|
||||
function compactAssistantStage2Report(report: Record<string, unknown>): Record<string, unknown> {
|
||||
const results = Array.isArray(report.results) ? (report.results as Array<Record<string, unknown>>) : [];
|
||||
const compactResults = results.map((item) => {
|
||||
const metricSubscores = (item.metric_subscores ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
case_id: item.case_id ?? null,
|
||||
metric_subscores: {
|
||||
problem_clarity_score: metricSubscores.problem_clarity_score ?? null,
|
||||
mechanism_coherence_score: metricSubscores.mechanism_coherence_score ?? null,
|
||||
problem_first_answer_rate: metricSubscores.problem_first_answer_rate ?? null,
|
||||
entity_leakage_rate: metricSubscores.entity_leakage_rate ?? null
|
||||
}
|
||||
};
|
||||
});
|
||||
return {
|
||||
...report,
|
||||
results: compactResults
|
||||
};
|
||||
}
|
||||
|
||||
type AssistantMetricKey = keyof AssistantEvalMetricVector;
|
||||
type AssistantStage2MetricKey = keyof AssistantStage2MetricVector;
|
||||
@@ -1140,7 +1233,7 @@ export class EvalService {
|
||||
};
|
||||
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
tryWriteJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
|
||||
return report;
|
||||
}
|
||||
@@ -1534,8 +1627,9 @@ export class EvalService {
|
||||
currentReport: Record<string, unknown>;
|
||||
baselineReportFile: string;
|
||||
}): Record<string, unknown> {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs.readFileSync(baselinePath, "utf-8")) as Record<string, unknown>;
|
||||
const baselineRef = readEvalReportByRef(input.baselineReportFile);
|
||||
const baselinePath = baselineRef.resolved_path;
|
||||
const baselineReport = baselineRef.report;
|
||||
const currentReport = input.currentReport;
|
||||
const metricKeys: AssistantMetricKey[] = [
|
||||
"retrieval_differentiation_rate",
|
||||
@@ -1633,14 +1727,15 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const jsonPath = path.resolve(REPORTS_DIR, `${comparisonId}.json`);
|
||||
const mdPath = path.resolve(REPORTS_DIR, `${comparisonId}.md`);
|
||||
writeJsonFile(jsonPath, comparisonReport);
|
||||
fs.writeFileSync(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport), "utf-8");
|
||||
const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
|
||||
const mdWritten = tryWriteTextFile(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport));
|
||||
const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
|
||||
|
||||
return {
|
||||
...comparisonReport,
|
||||
artifacts: {
|
||||
comparison_report_json_path: jsonPath,
|
||||
comparison_report_md_path: mdPath
|
||||
comparison_report_json_path: comparisonRef,
|
||||
comparison_report_md_path: mdWritten ? mdPath : null
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1649,8 +1744,9 @@ export class EvalService {
|
||||
currentReport: Record<string, unknown>;
|
||||
baselineReportFile: string;
|
||||
}): Record<string, unknown> {
|
||||
const baselinePath = resolveReadablePath(input.baselineReportFile);
|
||||
const baselineReport = JSON.parse(fs.readFileSync(baselinePath, "utf-8")) as Record<string, unknown>;
|
||||
const baselineRef = readEvalReportByRef(input.baselineReportFile);
|
||||
const baselinePath = baselineRef.resolved_path;
|
||||
const baselineReport = baselineRef.report;
|
||||
const currentReport = input.currentReport;
|
||||
const metricKeys: AssistantStage2MetricKey[] = [
|
||||
"problem_unit_precision",
|
||||
@@ -1760,14 +1856,15 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const jsonPath = path.resolve(REPORTS_DIR, `${comparisonId}.json`);
|
||||
const mdPath = path.resolve(REPORTS_DIR, `${comparisonId}.md`);
|
||||
writeJsonFile(jsonPath, comparisonReport);
|
||||
fs.writeFileSync(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport), "utf-8");
|
||||
const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
|
||||
const mdWritten = tryWriteTextFile(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport));
|
||||
const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
|
||||
|
||||
return {
|
||||
...comparisonReport,
|
||||
artifacts: {
|
||||
comparison_report_json_path: jsonPath,
|
||||
comparison_report_md_path: mdPath
|
||||
comparison_report_json_path: comparisonRef,
|
||||
comparison_report_md_path: mdWritten ? mdPath : null
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1803,7 +1900,7 @@ export class EvalService {
|
||||
|
||||
try {
|
||||
for (const turn of suiteCase.turns) {
|
||||
const response = await assistantService.handleMessage({
|
||||
const response = (await assistantService.handleMessage({
|
||||
session_id: sessionId,
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
@@ -1819,7 +1916,7 @@ export class EvalService {
|
||||
domainPrompt: payload.normalizeConfig.domainPrompt,
|
||||
fewShotExamples: payload.normalizeConfig.fewShotExamples,
|
||||
useMock: payload.useMock
|
||||
});
|
||||
})) as AssistantMessageResponsePayload;
|
||||
turnResponses.push(response);
|
||||
requestsTotal += 1;
|
||||
}
|
||||
@@ -2099,12 +2196,14 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const runJsonPath = path.resolve(REPORTS_DIR, `${runId}.json`);
|
||||
const runMdPath = path.resolve(REPORTS_DIR, `${runId}.md`);
|
||||
writeJsonFile(runJsonPath, report);
|
||||
fs.writeFileSync(runMdPath, buildAssistantEvalMarkdownReport(report), "utf-8");
|
||||
const compactReport = compactAssistantStage1Report(report);
|
||||
const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
|
||||
const mdWritten = tryWriteTextFile(runMdPath, buildAssistantEvalMarkdownReport(compactReport));
|
||||
const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
|
||||
|
||||
report.artifacts = {
|
||||
run_report_json_path: runJsonPath,
|
||||
run_report_md_path: runMdPath
|
||||
run_report_json_path: runReportRef,
|
||||
run_report_md_path: mdWritten ? runMdPath : null
|
||||
};
|
||||
|
||||
if (payload.compareWithReportFile) {
|
||||
@@ -2151,7 +2250,7 @@ export class EvalService {
|
||||
|
||||
try {
|
||||
for (const turn of suiteCase.turns) {
|
||||
const response = await assistantService.handleMessage({
|
||||
const response = (await assistantService.handleMessage({
|
||||
session_id: sessionId,
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
@@ -2167,7 +2266,7 @@ export class EvalService {
|
||||
domainPrompt: payload.normalizeConfig.domainPrompt,
|
||||
fewShotExamples: payload.normalizeConfig.fewShotExamples,
|
||||
useMock: payload.useMock
|
||||
});
|
||||
})) as AssistantMessageResponsePayload;
|
||||
turnResponses.push(response);
|
||||
requestsTotal += 1;
|
||||
}
|
||||
@@ -2393,12 +2492,14 @@ export class EvalService {
|
||||
ensureDir(REPORTS_DIR);
|
||||
const runJsonPath = path.resolve(REPORTS_DIR, `${runId}.json`);
|
||||
const runMdPath = path.resolve(REPORTS_DIR, `${runId}.md`);
|
||||
writeJsonFile(runJsonPath, report);
|
||||
fs.writeFileSync(runMdPath, buildAssistantStage2EvalMarkdownReport(report), "utf-8");
|
||||
const compactReport = compactAssistantStage2Report(report);
|
||||
const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
|
||||
const mdWritten = tryWriteTextFile(runMdPath, buildAssistantStage2EvalMarkdownReport(compactReport));
|
||||
const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
|
||||
|
||||
report.artifacts = {
|
||||
run_report_json_path: runJsonPath,
|
||||
run_report_md_path: runMdPath
|
||||
run_report_json_path: runReportRef,
|
||||
run_report_md_path: mdWritten ? runMdPath : null
|
||||
};
|
||||
|
||||
if (payload.compareWithReportFile) {
|
||||
@@ -2411,6 +2512,33 @@ export class EvalService {
|
||||
return report;
|
||||
}
|
||||
|
||||
private async runAssistantP0(payload: {
|
||||
normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
|
||||
caseIds?: string[];
|
||||
useMock?: boolean;
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
compareWithReportFile?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
if (!FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
|
||||
throw new ApiError(
|
||||
"ASSISTANT_P0_EVAL_DISABLED",
|
||||
"Assistant P0 eval target is disabled by FEATURE_ASSISTANT_STAGE2_EVAL_V1.",
|
||||
409
|
||||
);
|
||||
}
|
||||
|
||||
const runner = new P0EvalRunner(this.normalizerService);
|
||||
return runner.run({
|
||||
normalizeConfig: payload.normalizeConfig,
|
||||
caseIds: payload.caseIds,
|
||||
useMock: payload.useMock,
|
||||
mode: payload.mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
|
||||
public async run(payload: {
|
||||
normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
|
||||
caseIds?: string[];
|
||||
@@ -2446,6 +2574,17 @@ export class EvalService {
|
||||
});
|
||||
}
|
||||
|
||||
if (evalTarget === "assistant_p0") {
|
||||
return this.runAssistantP0({
|
||||
normalizeConfig: payload.normalizeConfig,
|
||||
caseIds: payload.caseIds,
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
});
|
||||
}
|
||||
|
||||
const promptVersion = String(payload.normalizeConfig.promptVersion ?? "").toLowerCase();
|
||||
const schemaVersion = String(payload.normalizeConfig.schemaVersion ?? "").toLowerCase();
|
||||
const isV2 =
|
||||
@@ -2659,7 +2798,7 @@ export class EvalService {
|
||||
};
|
||||
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
tryWriteJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
|
||||
|
||||
const shouldWriteV11Artifacts =
|
||||
mode === "single-pass-strict" &&
|
||||
@@ -2668,14 +2807,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV11Artifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1 Eval Run"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2687,14 +2825,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV1121EvalArtifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.2.1 Eval Run"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2705,14 +2842,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV111MicroArtifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.1 Micro Eval"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2723,14 +2859,13 @@ export class EvalService {
|
||||
|
||||
if (shouldWriteV112MicroArtifacts) {
|
||||
ensureDir(REPORTS_DIR);
|
||||
writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
|
||||
fs.writeFileSync(
|
||||
tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
|
||||
tryWriteTextFile(
|
||||
path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.md"),
|
||||
buildMarkdownReport({
|
||||
...report,
|
||||
report_title: "LLM Normalizer v1.1.2 Micro Eval"
|
||||
}),
|
||||
"utf-8"
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,103 @@ function collectOpenUncertainties(
|
||||
return capStrings([...requirementNotes, ...limitationNotes], INVESTIGATION_MAX_UNCERTAINTIES);
|
||||
}
|
||||
|
||||
function normalizeAccountPrefix(value: string): string | null {
|
||||
const account = String(value ?? "").trim();
|
||||
if (!account) {
|
||||
return null;
|
||||
}
|
||||
const match = account.match(/^(\d{2})/);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function isSettlementAccount(value: string): boolean {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
return prefix === "60" || prefix === "62" || prefix === "51" || prefix === "76";
|
||||
}
|
||||
|
||||
function isVatAccount(value: string): boolean {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
return prefix === "19" || prefix === "68";
|
||||
}
|
||||
|
||||
function isCloseCostsAccount(value: string): boolean {
|
||||
const prefix = normalizeAccountPrefix(value);
|
||||
if (!prefix) {
|
||||
return false;
|
||||
}
|
||||
const account = Number(prefix);
|
||||
return (account >= 20 && account <= 44) || prefix === "97";
|
||||
}
|
||||
|
||||
function inferFollowupActiveDomain(input: {
|
||||
userMessage: string;
|
||||
focusAccounts: string[];
|
||||
routeSummary: RouteHintSummary | null;
|
||||
previous: InvestigationStateWithProblemUnits;
|
||||
}): string | null {
|
||||
const corpus = `${input.userMessage} ${input.previous.focus.active_query_subject ?? ""}`.toLowerCase();
|
||||
const hasSettlementSignal =
|
||||
input.focusAccounts.some((item) => isSettlementAccount(item)) ||
|
||||
/(60(?:\.\d{2})?|62(?:\.\d{2})?|оплат|расчет|расч[её]т|зачет|зач[её]т|аванс|долг|поставщ|покупат|settlement|payment|supplier|customer)/i.test(
|
||||
corpus
|
||||
);
|
||||
if (hasSettlementSignal) {
|
||||
return "settlements_60_62";
|
||||
}
|
||||
|
||||
const hasVatSignal =
|
||||
input.focusAccounts.some((item) => isVatAccount(item)) ||
|
||||
/(ндс|счет[\s-]?фактур|сч[её]т[\s-]?фактур|книг[аи]|vat|invoice|book|register)/i.test(corpus);
|
||||
if (hasVatSignal) {
|
||||
return "vat_document_register_book";
|
||||
}
|
||||
|
||||
const hasCloseSignal =
|
||||
input.focusAccounts.some((item) => isCloseCostsAccount(item)) ||
|
||||
/(закрыти|закрытие|месяц|затрат|распредел|списан|period\s*close|month\s*close|allocation|residual|cost)/i.test(corpus);
|
||||
if (hasCloseSignal) {
|
||||
return "month_close_costs_20_44";
|
||||
}
|
||||
|
||||
const routeDomain = deriveDomain(input.routeSummary);
|
||||
if (routeDomain && routeDomain !== "no_route") {
|
||||
return routeDomain;
|
||||
}
|
||||
|
||||
return input.previous.followup_context?.active_domain ?? input.previous.focus.domain ?? null;
|
||||
}
|
||||
|
||||
function collectUncoveredRequirementIds(coverageReport: RequirementCoverageReport): string[] {
|
||||
return capStrings(
|
||||
[
|
||||
...coverageReport.requirements_uncovered,
|
||||
...coverageReport.requirements_partially_covered,
|
||||
...coverageReport.clarification_needed_for,
|
||||
...coverageReport.out_of_scope_requirements
|
||||
],
|
||||
INVESTIGATION_MAX_REQUIREMENT_LINKS
|
||||
);
|
||||
}
|
||||
|
||||
function collectEvidenceSummary(retrievalResults: UnifiedRetrievalResult[]): string[] {
|
||||
const lines = retrievalResults.map((result) => {
|
||||
const requirementRef = result.requirement_ids[0] ?? result.fragment_id;
|
||||
return `${requirementRef}:${result.status}:${result.route}`;
|
||||
});
|
||||
return capStrings(lines, 6);
|
||||
}
|
||||
|
||||
function settlementFocusActions(activeDomain: string | null): string[] {
|
||||
if (activeDomain !== "settlements_60_62") {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
"Проверьте договор и объект расчетов по платежу.",
|
||||
"Сверьте регистр расчетов и привязку платежа к закрывающему документу.",
|
||||
"Проверьте зачет аванса или взаимозачет по связке 60/62."
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeEntityBacklinks(values: ProblemUnitEntityBacklink[]): ProblemUnitEntityBacklink[] {
|
||||
const result: ProblemUnitEntityBacklink[] = [];
|
||||
const seen = new Set<string>();
|
||||
@@ -273,7 +370,27 @@ export function cloneInvestigationState(state: InvestigationStateWithProblemUnit
|
||||
followup_context: state.followup_context
|
||||
? {
|
||||
...state.followup_context,
|
||||
referenced_requirement_ids: [...state.followup_context.referenced_requirement_ids]
|
||||
referenced_requirement_ids: [...state.followup_context.referenced_requirement_ids],
|
||||
...(state.followup_context.active_requirement_ids
|
||||
? {
|
||||
active_requirement_ids: [...state.followup_context.active_requirement_ids]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.uncovered_requirement_ids
|
||||
? {
|
||||
uncovered_requirement_ids: [...state.followup_context.uncovered_requirement_ids]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.settlement_next_actions
|
||||
? {
|
||||
settlement_next_actions: [...state.followup_context.settlement_next_actions]
|
||||
}
|
||||
: {}),
|
||||
...(state.followup_context.evidence_summary
|
||||
? {
|
||||
evidence_summary: [...state.followup_context.evidence_summary]
|
||||
}
|
||||
: {})
|
||||
}
|
||||
: null
|
||||
};
|
||||
@@ -320,12 +437,27 @@ export function createEmptyInvestigationState(
|
||||
export function updateInvestigationState(input: UpdateInvestigationStateInput): InvestigationStateWithProblemUnits {
|
||||
const previous = input.previous;
|
||||
const focusFromMessage = capStrings(detectAccounts(input.userMessage), INVESTIGATION_MAX_PRIMARY_ACCOUNTS);
|
||||
const mergedFocusAccounts = capStrings(
|
||||
[...focusFromMessage, ...previous.focus.primary_accounts],
|
||||
INVESTIGATION_MAX_PRIMARY_ACCOUNTS
|
||||
);
|
||||
const requirementIds = capStrings(
|
||||
input.requirements.map((item) => item.requirement_id),
|
||||
INVESTIGATION_MAX_REQUIREMENT_LINKS
|
||||
);
|
||||
const mainRequirement = input.requirements[0]?.requirement_text ?? input.userMessage;
|
||||
const problemUnitState = updateProblemUnitState(previous, input.retrievalResults);
|
||||
const uncoveredRequirementIds = collectUncoveredRequirementIds(input.coverageReport);
|
||||
const activeDomain = inferFollowupActiveDomain({
|
||||
userMessage: input.userMessage,
|
||||
focusAccounts: mergedFocusAccounts,
|
||||
routeSummary: input.routeSummary,
|
||||
previous
|
||||
});
|
||||
const focusDomain = activeDomain ?? deriveDomain(input.routeSummary) ?? previous.focus.domain;
|
||||
const settlementNextActions = settlementFocusActions(activeDomain);
|
||||
const lastProblemUnitId = problemUnitState?.active_problem_units[0] ?? null;
|
||||
const evidenceSummary = collectEvidenceSummary(input.retrievalResults);
|
||||
|
||||
return {
|
||||
schema_version: INVESTIGATION_STATE_SCHEMA_VERSION,
|
||||
@@ -335,12 +467,9 @@ export function updateInvestigationState(input: UpdateInvestigationStateInput):
|
||||
updated_at: input.timestamp,
|
||||
question_id: input.questionId,
|
||||
focus: {
|
||||
domain: deriveDomain(input.routeSummary) ?? previous.focus.domain,
|
||||
domain: focusDomain,
|
||||
period: detectPeriod(input.userMessage) ?? previous.focus.period,
|
||||
primary_accounts: capStrings(
|
||||
[...focusFromMessage, ...previous.focus.primary_accounts],
|
||||
INVESTIGATION_MAX_PRIMARY_ACCOUNTS
|
||||
),
|
||||
primary_accounts: mergedFocusAccounts,
|
||||
active_query_subject: mainRequirement.slice(0, 180)
|
||||
},
|
||||
narrowing_status: deriveNarrowingStatus(input.routeSummary, input.coverageReport),
|
||||
@@ -353,7 +482,13 @@ export function updateInvestigationState(input: UpdateInvestigationStateInput):
|
||||
followup_context: {
|
||||
previous_question_id: previous.question_id,
|
||||
last_user_message: input.userMessage.slice(0, 240),
|
||||
referenced_requirement_ids: requirementIds
|
||||
referenced_requirement_ids: requirementIds,
|
||||
active_domain: activeDomain,
|
||||
active_requirement_ids: requirementIds,
|
||||
uncovered_requirement_ids: uncoveredRequirementIds,
|
||||
last_problem_unit_id: lastProblemUnitId,
|
||||
settlement_next_actions: settlementNextActions,
|
||||
evidence_summary: evidenceSummary
|
||||
},
|
||||
query_mode_hint: deriveQueryModeHint(input.routeSummary),
|
||||
...(problemUnitState
|
||||
|
||||
@@ -4,7 +4,11 @@ import type {
|
||||
RetrievalResultType,
|
||||
UnifiedRetrievalResult
|
||||
} from "../types/assistant";
|
||||
import { FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1, FEATURE_ASSISTANT_PROBLEM_UNITS_V1 } from "../config";
|
||||
import {
|
||||
FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1,
|
||||
FEATURE_ASSISTANT_GRAPH_RUNTIME_V1,
|
||||
FEATURE_ASSISTANT_PROBLEM_UNITS_V1
|
||||
} from "../config";
|
||||
import { EVIDENCE_SOURCE_REF_SCHEMA_VERSION } from "../types/stage1Contracts";
|
||||
import type {
|
||||
EvidenceConfidence,
|
||||
@@ -15,6 +19,8 @@ import type {
|
||||
EvidenceSourceRef
|
||||
} from "../types/stage1Contracts";
|
||||
import { assembleProblemUnits } from "./problemUnitAssembler";
|
||||
import { buildAccountingGraph } from "./stage4GraphRuntime";
|
||||
import type { GraphSignalSummary } from "../types/stage4Graph";
|
||||
|
||||
interface RawRetrievalResult {
|
||||
status?: string;
|
||||
@@ -124,6 +130,22 @@ function mergeSummaryWithProblemUnitMeta(
|
||||
};
|
||||
}
|
||||
|
||||
function mergeSummaryWithGraphMeta(summary: Record<string, unknown>, graphSummary: GraphSignalSummary): Record<string, unknown> {
|
||||
return {
|
||||
...summary,
|
||||
graph_runtime_enabled: true,
|
||||
graph_total_units: graphSummary.total_units,
|
||||
graph_bound_units: graphSummary.bound_units,
|
||||
graph_nodes_count: graphSummary.node_count,
|
||||
graph_edges_count: graphSummary.edge_count,
|
||||
graph_missing_links_count: graphSummary.missing_links_count,
|
||||
graph_conflicting_links_count: graphSummary.conflicting_links_count,
|
||||
graph_coverage_grade: graphSummary.graph_coverage_grade,
|
||||
graph_domain_distribution: graphSummary.domain_distribution,
|
||||
graph_relation_distribution: graphSummary.relation_distribution
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeConfidence(value: unknown): RetrievalConfidence {
|
||||
if (value === "high" || value === "medium" || value === "low") {
|
||||
return value;
|
||||
@@ -526,24 +548,60 @@ export function normalizeRetrievalResult(
|
||||
business_interpretation: baseResult.business_interpretation
|
||||
});
|
||||
|
||||
const enrichedSummary = mergeSummaryWithProblemUnitMeta(summary, {
|
||||
candidateEvidenceCount: assembled.candidate_evidence.length,
|
||||
problemUnitsCount: assembled.problem_units.length,
|
||||
unitTypes: assembled.problem_unit_summary.unit_types,
|
||||
duplicateCollapses: assembled.problem_unit_summary.duplicate_collapses,
|
||||
severityDistribution: assembled.problem_unit_summary.severity_distribution,
|
||||
confidenceDistribution: assembled.problem_unit_summary.confidence_distribution,
|
||||
lifecycleEnrichedUnits: assembled.problem_unit_summary.lifecycle_enriched_units ?? 0,
|
||||
lifecycleDomainDistribution: (assembled.problem_unit_summary.lifecycle_domain_distribution ?? {}) as Record<string, number>,
|
||||
lifecycleDefectDistribution: (assembled.problem_unit_summary.lifecycle_defect_distribution ?? {}) as Record<string, number>
|
||||
const graphBuild = FEATURE_ASSISTANT_GRAPH_RUNTIME_V1
|
||||
? buildAccountingGraph({
|
||||
route,
|
||||
candidateEvidence: assembled.candidate_evidence,
|
||||
problemUnits: assembled.problem_units
|
||||
})
|
||||
: null;
|
||||
|
||||
const graphBindingByUnitId = new Map(graphBuild?.unit_bindings.map((item) => [item.problem_unit_id, item] as const) ?? []);
|
||||
const graphBoundProblemUnits = assembled.problem_units.map((unit) => {
|
||||
const binding = graphBindingByUnitId.get(unit.problem_unit_id);
|
||||
if (!binding) {
|
||||
return unit;
|
||||
}
|
||||
return {
|
||||
...unit,
|
||||
graph_binding: binding
|
||||
};
|
||||
});
|
||||
|
||||
const graphBoundSummary = graphBuild
|
||||
? {
|
||||
...assembled.problem_unit_summary,
|
||||
graph_summary: graphBuild.summary
|
||||
}
|
||||
: assembled.problem_unit_summary;
|
||||
|
||||
let enrichedSummary = mergeSummaryWithProblemUnitMeta(summary, {
|
||||
candidateEvidenceCount: assembled.candidate_evidence.length,
|
||||
problemUnitsCount: graphBoundProblemUnits.length,
|
||||
unitTypes: graphBoundSummary.unit_types,
|
||||
duplicateCollapses: graphBoundSummary.duplicate_collapses,
|
||||
severityDistribution: graphBoundSummary.severity_distribution,
|
||||
confidenceDistribution: graphBoundSummary.confidence_distribution,
|
||||
lifecycleEnrichedUnits: graphBoundSummary.lifecycle_enriched_units ?? 0,
|
||||
lifecycleDomainDistribution: (graphBoundSummary.lifecycle_domain_distribution ?? {}) as Record<string, number>,
|
||||
lifecycleDefectDistribution: (graphBoundSummary.lifecycle_defect_distribution ?? {}) as Record<string, number>
|
||||
});
|
||||
|
||||
if (graphBuild) {
|
||||
enrichedSummary = mergeSummaryWithGraphMeta(enrichedSummary, graphBuild.summary);
|
||||
}
|
||||
|
||||
return {
|
||||
...baseResult,
|
||||
summary: enrichedSummary,
|
||||
raw_entities: items,
|
||||
candidate_evidence: assembled.candidate_evidence,
|
||||
problem_units: assembled.problem_units,
|
||||
problem_unit_summary: assembled.problem_unit_summary
|
||||
problem_units: graphBoundProblemUnits,
|
||||
problem_unit_summary: graphBoundSummary,
|
||||
...(graphBuild
|
||||
? {
|
||||
accounting_graph: graphBuild
|
||||
}
|
||||
: {})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type {
|
||||
DeterministicRouteHint,
|
||||
NoRouteReason,
|
||||
NormalizedPayload,
|
||||
NormalizedQueryV1,
|
||||
@@ -42,6 +43,244 @@ function toRouteHintSummaryV1(normalized: NormalizedQueryV1): RouteHintSummaryV1
|
||||
type V2Family = NormalizedQueryV2 | NormalizedQueryV2_0_1 | NormalizedQueryV2_0_2;
|
||||
type V2FamilyFragment = V2Family["fragments"][number];
|
||||
|
||||
type RouteQueryClass =
|
||||
| "exact_object_trace"
|
||||
| "ranking_or_period_summary"
|
||||
| "symptom_first"
|
||||
| "lifecycle_first"
|
||||
| "chain_break"
|
||||
| "period_impact"
|
||||
| "causal_query"
|
||||
| "mixed_ambiguity"
|
||||
| "rule_check_without_symptom"
|
||||
| "canonical_fact_lookup";
|
||||
|
||||
interface RouteDisciplineRule {
|
||||
query_class: RouteQueryClass;
|
||||
required_route: Exclude<DeterministicRouteHint, "no_route">;
|
||||
allowed_fallback: DeterministicRouteHint[];
|
||||
forbidden_fallback: DeterministicRouteHint[];
|
||||
description: string;
|
||||
}
|
||||
|
||||
const ACCOUNT_HINT_PATTERN =
|
||||
/(?:\b(?:account|acct|schet|счет|сч)\s*[:#]?\s*(?:[1-9][0-9](?:[./-][0-9]{1,2})?)|\b(?:19|20|21|23|25|26|28|29|44|51|60|62|68)\b)/i;
|
||||
const PERIOD_PATTERN = /\b20\d{2}(?:[-./](?:0[1-9]|1[0-2]))?\b/i;
|
||||
const SYMPTOM_MARKER_PATTERN =
|
||||
/(?:\bsymptom\b|\banomaly\b|\bproblem\b|\bissue\b|\btail\b|\bhanging\b|\bblocked\b|\bincomplete\b|remains?\s+open|not\s+(?:confirmed|observed|resolved|closed)|не\s+(?:подтвержден|закрыт|наблюдается)|хвост|сбой|проблем)/i;
|
||||
const LIFECYCLE_MARKER_PATTERN =
|
||||
/(?:\blifecycle\b|\bchain\b|\btransition\b|\bstep\b|\btrace\b|цепоч|этап|переход|связк|где\s+разрыв)/i;
|
||||
const CHAIN_BREAK_PATTERN =
|
||||
/(?:\bbreak\b|\bbroken\b|\bgap\b|missing\s+(?:transition|step|link)|chain\s+break|разрыв|обрыв|нет\s+переход|не\s+дошл|не\s+наблюд)/i;
|
||||
const PERIOD_IMPACT_PATTERN =
|
||||
/(?:period\s*close|month\s*close|month-end|residual|allocation|20[/-]44|закрыти|остатк|распредел|конец\s+месяц)/i;
|
||||
const CAUSAL_PATTERN = /(?:\bwhy\b|\bbecause\b|\breason\b|explain\s+mechanism|почему|объясни|механизм|причин)/i;
|
||||
const AMBIGUITY_PATTERN =
|
||||
/(?:\bmaybe\b|\bperhaps\b|not\s+sure|i\s+only\s+know|part\s+may\s+be\s+missing|возможно|может\s+быть|не\s+уверен|не\s+знаю|часть\s+цепочки\s+не\s+подтвержд)/i;
|
||||
const TRANSLIT_PROBLEM_PATTERN = /(?:raschet|oplata|zakryt|nds|vychet|zatrat|ostatok|cepoch|perehod|pochemu|prichin|period)/i;
|
||||
const DOMAIN_LEXICAL_ANCHOR_PATTERN =
|
||||
/(?:\b(?:settlement|payment|bank|supplier|customer|vat|nds|invoice|register|book|period\s*close|month\s*close|close\s*operation|allocation|residual|cost|expenses?)\b|оплат|расчет|РЅРґСЃ|СЃС‡[её]С‚.?фактур|РєРЅРёРі[аи]|затрат|закрыт|остатк)/i;
|
||||
|
||||
export const ROUTE_DISCIPLINE_RULE_TABLE: RouteDisciplineRule[] = [
|
||||
{
|
||||
query_class: "exact_object_trace",
|
||||
required_route: "live_mcp_drilldown",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical", "hybrid_store_plus_live", "store_feature_risk", "batch_refresh_then_store"],
|
||||
description: "Exact object trace queries always run via live drilldown."
|
||||
},
|
||||
{
|
||||
query_class: "ranking_or_period_summary",
|
||||
required_route: "batch_refresh_then_store",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical", "hybrid_store_plus_live"],
|
||||
description: "Ranking and period summary queries require analytical batch path."
|
||||
},
|
||||
{
|
||||
query_class: "symptom_first",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Symptom-first intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "lifecycle_first",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Lifecycle-first intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "chain_break",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Chain-break intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "period_impact",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Period-impact problem intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "causal_query",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Causal/mechanism intents are deterministically promoted to hybrid path."
|
||||
},
|
||||
{
|
||||
query_class: "mixed_ambiguity",
|
||||
required_route: "hybrid_store_plus_live",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Mixed ambiguity keeps hybrid as primary route, with explicit no-route fallback."
|
||||
},
|
||||
{
|
||||
query_class: "rule_check_without_symptom",
|
||||
required_route: "store_feature_risk",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["store_canonical"],
|
||||
description: "Rule checks without symptom/lifecycle signals run via risk profile path."
|
||||
},
|
||||
{
|
||||
query_class: "canonical_fact_lookup",
|
||||
required_route: "store_canonical",
|
||||
allowed_fallback: ["no_route"],
|
||||
forbidden_fallback: ["hybrid_store_plus_live"],
|
||||
description: "Only plain factual lookups are allowed to stay on canonical path."
|
||||
}
|
||||
];
|
||||
|
||||
const ROUTE_DISCIPLINE_RULE_MAP = new Map<RouteQueryClass, RouteDisciplineRule>(
|
||||
ROUTE_DISCIPLINE_RULE_TABLE.map((item) => [item.query_class, item])
|
||||
);
|
||||
|
||||
function mergedFragmentText(fragment: V2FamilyFragment): string {
|
||||
return `${fragment.raw_fragment_text ?? ""} ${fragment.normalized_fragment_text ?? ""}`.toLowerCase();
|
||||
}
|
||||
|
||||
function hasLifecycleDomainHint(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
const accountHints = Array.isArray(fragment.account_hints) ? fragment.account_hints.map((item) => String(item)) : [];
|
||||
if (accountHints.some((item) => /^(97|01|02|08|19|20|21|23|25|26|28|29|44|68(?:\.\d+)?|51|60|62)$/.test(item))) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
fragment.candidate_labels.includes("anomaly_probe") ||
|
||||
fragment.candidate_labels.includes("period_close_risk") ||
|
||||
PERIOD_IMPACT_PATTERN.test(lowerText)
|
||||
);
|
||||
}
|
||||
|
||||
function hasSymptomSignal(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return (
|
||||
fragment.flags.asks_for_anomaly_scan ||
|
||||
fragment.candidate_labels.includes("anomaly_probe") ||
|
||||
fragment.candidate_labels.includes("period_close_risk") ||
|
||||
SYMPTOM_MARKER_PATTERN.test(lowerText) ||
|
||||
TRANSLIT_PROBLEM_PATTERN.test(lowerText)
|
||||
);
|
||||
}
|
||||
|
||||
function hasLifecycleSignal(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return (
|
||||
fragment.flags.asks_for_chain_explanation ||
|
||||
fragment.flags.mentions_period_close_context ||
|
||||
LIFECYCLE_MARKER_PATTERN.test(lowerText) ||
|
||||
hasLifecycleDomainHint(fragment, lowerText)
|
||||
);
|
||||
}
|
||||
|
||||
function hasChainBreakSignal(lowerText: string): boolean {
|
||||
return CHAIN_BREAK_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function hasPeriodImpactSignal(lowerText: string): boolean {
|
||||
return PERIOD_IMPACT_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function hasCausalSignal(lowerText: string): boolean {
|
||||
return CAUSAL_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function hasAmbiguitySignal(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return (
|
||||
AMBIGUITY_PATTERN.test(lowerText) ||
|
||||
fragment.confidence === "low" ||
|
||||
fragment.domain_relevance === "unclear" ||
|
||||
fragment.business_scope === "unclear"
|
||||
);
|
||||
}
|
||||
|
||||
function hasAccountOrPeriodAnchor(fragment: V2FamilyFragment, lowerText: string): boolean {
|
||||
return fragment.account_hints.length > 0 || ACCOUNT_HINT_PATTERN.test(lowerText) || PERIOD_PATTERN.test(lowerText);
|
||||
}
|
||||
|
||||
function resolveRouteClass(fragment: V2FamilyFragment): RouteDisciplineRule {
|
||||
const lowerText = mergedFragmentText(fragment);
|
||||
const symptomSignal = hasSymptomSignal(fragment, lowerText);
|
||||
const lifecycleSignal = hasLifecycleSignal(fragment, lowerText);
|
||||
const chainBreakSignal = hasChainBreakSignal(lowerText);
|
||||
const periodImpactSignal = hasPeriodImpactSignal(lowerText);
|
||||
const causalSignal = hasCausalSignal(lowerText);
|
||||
const ambiguitySignal = hasAmbiguitySignal(fragment, lowerText);
|
||||
const accountOrPeriodAnchor = hasAccountOrPeriodAnchor(fragment, lowerText);
|
||||
|
||||
if (fragment.flags.asks_for_exact_object_trace) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("exact_object_trace")!;
|
||||
}
|
||||
if (fragment.flags.asks_for_ranking_or_top || fragment.flags.asks_for_period_summary) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("ranking_or_period_summary")!;
|
||||
}
|
||||
if (ambiguitySignal && (symptomSignal || lifecycleSignal || chainBreakSignal || periodImpactSignal || causalSignal)) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("mixed_ambiguity")!;
|
||||
}
|
||||
if (chainBreakSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("chain_break")!;
|
||||
}
|
||||
if (periodImpactSignal && accountOrPeriodAnchor) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("period_impact")!;
|
||||
}
|
||||
if (lifecycleSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("lifecycle_first")!;
|
||||
}
|
||||
if (symptomSignal) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("symptom_first")!;
|
||||
}
|
||||
if (causalSignal && accountOrPeriodAnchor) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("causal_query")!;
|
||||
}
|
||||
if (fragment.flags.asks_for_rule_check) {
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("rule_check_without_symptom")!;
|
||||
}
|
||||
return ROUTE_DISCIPLINE_RULE_MAP.get("canonical_fact_lookup")!;
|
||||
}
|
||||
|
||||
function shouldPromoteFromNoRoute(fragment: V2FamilyFragment, rule: RouteDisciplineRule): boolean {
|
||||
if (rule.required_route === "store_canonical") {
|
||||
return false;
|
||||
}
|
||||
if (explicitNoRouteReason(fragment) === "out_of_scope") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const lowerText = mergedFragmentText(fragment);
|
||||
const hasProblemSignal =
|
||||
hasSymptomSignal(fragment, lowerText) ||
|
||||
hasLifecycleSignal(fragment, lowerText) ||
|
||||
hasChainBreakSignal(lowerText) ||
|
||||
hasPeriodImpactSignal(lowerText) ||
|
||||
hasCausalSignal(lowerText);
|
||||
|
||||
const hasAnchor =
|
||||
hasAccountOrPeriodAnchor(fragment, lowerText) ||
|
||||
fragment.candidate_labels.includes("cross_entity") ||
|
||||
DOMAIN_LEXICAL_ANCHOR_PATTERN.test(lowerText);
|
||||
return hasProblemSignal && hasAnchor;
|
||||
}
|
||||
|
||||
function reasonForNoRoute(noRouteReason: NoRouteReason | null | undefined): string {
|
||||
if (noRouteReason === "out_of_scope") {
|
||||
return "Fragment is out-of-scope for company-specific accounting contour.";
|
||||
@@ -98,37 +337,33 @@ function decideRouteForFragment(fragment: V2FamilyFragment): RouteDecisionV2 {
|
||||
const readiness = executionReadiness(fragment);
|
||||
const clarification = clarificationReason(fragment);
|
||||
const soft = softAssumptions(fragment);
|
||||
const routeRule = resolveRouteClass(fragment);
|
||||
|
||||
if (status === "no_route") {
|
||||
return buildNoRouteDecision(fragment, noRouteReason);
|
||||
}
|
||||
|
||||
if (readiness === "needs_clarification" || readiness === "no_route") {
|
||||
return buildNoRouteDecision(fragment, noRouteReason ?? "insufficient_specificity");
|
||||
}
|
||||
|
||||
if (fragment.domain_relevance !== "in_scope") {
|
||||
if (fragment.domain_relevance === "out_of_scope") {
|
||||
return buildNoRouteDecision(fragment, "out_of_scope");
|
||||
}
|
||||
|
||||
if (fragment.flags.asks_for_exact_object_trace) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "live_mcp_drilldown",
|
||||
reason: "Exact object trace requested."
|
||||
};
|
||||
if (status === "no_route" || readiness === "needs_clarification" || readiness === "no_route") {
|
||||
if (shouldPromoteFromNoRoute(fragment, routeRule)) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: routeRule.required_route,
|
||||
reason: `${routeRule.description} Query class: ${routeRule.query_class}. Promoted from no-route by anchor/symptom guardrail.`
|
||||
};
|
||||
}
|
||||
return buildNoRouteDecision(fragment, noRouteReason);
|
||||
}
|
||||
|
||||
if (fragment.flags.asks_for_ranking_or_top || fragment.flags.asks_for_period_summary) {
|
||||
if (status === "routed" || status === null) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
@@ -140,80 +375,8 @@ function decideRouteForFragment(fragment: V2FamilyFragment): RouteDecisionV2 {
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "batch_refresh_then_store",
|
||||
reason: "Ranking/summary semantics require batch analytical route."
|
||||
};
|
||||
}
|
||||
|
||||
if (fragment.flags.has_multi_entity_scope && fragment.flags.asks_for_chain_explanation) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "hybrid_store_plus_live",
|
||||
reason: "Multi-entity causal chain requested."
|
||||
};
|
||||
}
|
||||
|
||||
if (fragment.flags.asks_for_rule_check && !fragment.flags.asks_for_chain_explanation) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_feature_risk",
|
||||
reason: "Rule-control check without causal decomposition."
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
fragment.flags.asks_for_anomaly_scan &&
|
||||
!fragment.flags.asks_for_ranking_or_top &&
|
||||
!(fragment.flags.has_multi_entity_scope && fragment.flags.asks_for_chain_explanation)
|
||||
) {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_feature_risk",
|
||||
reason: "Anomaly scan without heavy ranking or causal chain."
|
||||
};
|
||||
}
|
||||
|
||||
if (status === "routed") {
|
||||
return {
|
||||
fragment_id: fragment.fragment_id,
|
||||
domain_relevance: fragment.domain_relevance,
|
||||
business_scope: fragment.business_scope,
|
||||
candidate_labels: fragment.candidate_labels,
|
||||
decision_flags: fragment.flags,
|
||||
execution_readiness: readiness,
|
||||
clarification_reason: clarification,
|
||||
soft_assumption_used: soft,
|
||||
route_status: "routed",
|
||||
no_route_reason: null,
|
||||
route: "store_canonical",
|
||||
reason: "Routed fragment without deep analytical or causal signals."
|
||||
route: routeRule.required_route,
|
||||
reason: `${routeRule.description} Query class: ${routeRule.query_class}. Allowed fallback: ${routeRule.allowed_fallback.join(", ")}. Forbidden fallback: ${routeRule.forbidden_fallback.join(", ")}.`
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,662 @@
|
||||
import type { CandidateEvidenceItem, ProblemUnit } from "../types/stage2ProblemUnits";
|
||||
import type { LifecycleDomain } from "../types/stage3Lifecycle";
|
||||
import type {
|
||||
AccountingGraphBuildResult,
|
||||
AccountingGraphEdge,
|
||||
AccountingGraphEdgeFlag,
|
||||
AccountingGraphNode,
|
||||
AccountingGraphRelationType,
|
||||
AccountingGraphProvenance,
|
||||
GraphConfidenceGrade,
|
||||
GraphDomainKey,
|
||||
GraphSignalSummary,
|
||||
ProblemUnitGraphBinding
|
||||
} from "../types/stage4Graph";
|
||||
import { ACCOUNTING_GRAPH_SCHEMA_VERSION } from "../types/stage4Graph";
|
||||
|
||||
interface BuildAccountingGraphInput {
|
||||
route: string;
|
||||
candidateEvidence: CandidateEvidenceItem[];
|
||||
problemUnits: ProblemUnit[];
|
||||
}
|
||||
|
||||
interface GraphNodeCreateInput {
|
||||
node_type: AccountingGraphNode["node_type"];
|
||||
domain: GraphDomainKey;
|
||||
stable_key: string;
|
||||
label: string;
|
||||
confidence: GraphConfidenceGrade;
|
||||
attributes?: Record<string, unknown>;
|
||||
provenance?: Partial<AccountingGraphProvenance>;
|
||||
}
|
||||
|
||||
interface GraphEdgeCreateInput {
|
||||
relation_type: AccountingGraphRelationType;
|
||||
from_node_id: string;
|
||||
to_node_id: string;
|
||||
domain: GraphDomainKey;
|
||||
confidence: GraphConfidenceGrade;
|
||||
flags?: AccountingGraphEdgeFlag[];
|
||||
provenance?: Partial<AccountingGraphProvenance>;
|
||||
}
|
||||
|
||||
const GRAPH_CONFIDENCE_ORDER: Record<GraphConfidenceGrade, number> = {
|
||||
low: 1,
|
||||
medium: 2,
|
||||
high: 3
|
||||
};
|
||||
|
||||
const DOMAIN_PATH_HINTS: Record<LifecycleDomain, string[]> = {
|
||||
bank_settlement: ["payment_to_settlement", "wrong_closing_document_type"],
|
||||
customer_settlement: ["invoice_to_payment", "payment_to_closure"],
|
||||
deferred_expense: ["deferred_expense_to_writeoff", "writeoff_sequence"],
|
||||
fixed_asset: ["asset_card_to_depreciation", "card_document_register_alignment"],
|
||||
vat_flow: ["invoice_to_vat_register", "cross_branch_alignment"],
|
||||
period_close: ["period_close_dependency_chain", "closure_blocker_transition"]
|
||||
};
|
||||
|
||||
function uniqueStrings(values: Array<string | null | undefined>, limit = 16): string[] {
|
||||
return Array.from(new Set(values.map((item) => String(item ?? "").trim()).filter(Boolean))).slice(0, limit);
|
||||
}
|
||||
|
||||
function compactToken(value: string): string {
|
||||
const normalized = value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
return normalized.length > 0 ? normalized.slice(0, 48) : "x";
|
||||
}
|
||||
|
||||
function stableNodeId(type: AccountingGraphNode["node_type"], domain: GraphDomainKey, stableKey: string): string {
|
||||
return `gnd-${compactToken(type)}-${compactToken(domain)}-${compactToken(stableKey)}`;
|
||||
}
|
||||
|
||||
function stableEdgeId(relation: AccountingGraphRelationType, fromNode: string, toNode: string): string {
|
||||
return `ged-${compactToken(relation)}-${compactToken(fromNode)}-${compactToken(toNode)}`;
|
||||
}
|
||||
|
||||
function mergeConfidence(left: GraphConfidenceGrade, right: GraphConfidenceGrade): GraphConfidenceGrade {
|
||||
return GRAPH_CONFIDENCE_ORDER[right] > GRAPH_CONFIDENCE_ORDER[left] ? right : left;
|
||||
}
|
||||
|
||||
function mergeProvenance(
|
||||
left: AccountingGraphProvenance,
|
||||
right: Partial<AccountingGraphProvenance> | undefined,
|
||||
routeFallback: string
|
||||
): AccountingGraphProvenance {
|
||||
return {
|
||||
route: String(right?.route ?? left.route ?? routeFallback),
|
||||
candidate_ids: uniqueStrings([...(left.candidate_ids ?? []), ...(right?.candidate_ids ?? [])], 24),
|
||||
evidence_ids: uniqueStrings([...(left.evidence_ids ?? []), ...(right?.evidence_ids ?? [])], 24)
|
||||
};
|
||||
}
|
||||
|
||||
class GraphAccumulator {
|
||||
private readonly nodesById = new Map<string, AccountingGraphNode>();
|
||||
private readonly edgesById = new Map<string, AccountingGraphEdge>();
|
||||
|
||||
constructor(private readonly route: string) {}
|
||||
|
||||
public upsertNode(input: GraphNodeCreateInput): AccountingGraphNode {
|
||||
const node_id = stableNodeId(input.node_type, input.domain, input.stable_key);
|
||||
const existing = this.nodesById.get(node_id);
|
||||
if (!existing) {
|
||||
const created: AccountingGraphNode = {
|
||||
node_id,
|
||||
node_type: input.node_type,
|
||||
domain: input.domain,
|
||||
label: input.label,
|
||||
confidence: input.confidence,
|
||||
attributes: input.attributes ?? {},
|
||||
provenance: {
|
||||
route: String(input.provenance?.route ?? this.route),
|
||||
candidate_ids: uniqueStrings(input.provenance?.candidate_ids ?? [], 24),
|
||||
evidence_ids: uniqueStrings(input.provenance?.evidence_ids ?? [], 24)
|
||||
}
|
||||
};
|
||||
this.nodesById.set(node_id, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
existing.confidence = mergeConfidence(existing.confidence, input.confidence);
|
||||
existing.provenance = mergeProvenance(existing.provenance, input.provenance, this.route);
|
||||
if (Object.keys(input.attributes ?? {}).length > 0) {
|
||||
existing.attributes = {
|
||||
...existing.attributes,
|
||||
...input.attributes
|
||||
};
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
public upsertEdge(input: GraphEdgeCreateInput): AccountingGraphEdge {
|
||||
const edge_id = stableEdgeId(input.relation_type, input.from_node_id, input.to_node_id);
|
||||
const existing = this.edgesById.get(edge_id);
|
||||
if (!existing) {
|
||||
const created: AccountingGraphEdge = {
|
||||
edge_id,
|
||||
relation_type: input.relation_type,
|
||||
from_node_id: input.from_node_id,
|
||||
to_node_id: input.to_node_id,
|
||||
domain: input.domain,
|
||||
confidence: input.confidence,
|
||||
flags: uniqueStrings(input.flags ?? [], 8) as AccountingGraphEdgeFlag[],
|
||||
provenance: {
|
||||
route: String(input.provenance?.route ?? this.route),
|
||||
candidate_ids: uniqueStrings(input.provenance?.candidate_ids ?? [], 24),
|
||||
evidence_ids: uniqueStrings(input.provenance?.evidence_ids ?? [], 24)
|
||||
}
|
||||
};
|
||||
this.edgesById.set(edge_id, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
existing.confidence = mergeConfidence(existing.confidence, input.confidence);
|
||||
existing.flags = uniqueStrings([...(existing.flags ?? []), ...(input.flags ?? [])], 8) as AccountingGraphEdgeFlag[];
|
||||
existing.provenance = mergeProvenance(existing.provenance, input.provenance, this.route);
|
||||
return existing;
|
||||
}
|
||||
|
||||
public export(): { nodes: AccountingGraphNode[]; edges: AccountingGraphEdge[] } {
|
||||
return {
|
||||
nodes: Array.from(this.nodesById.values()).sort((left, right) => left.node_id.localeCompare(right.node_id)),
|
||||
edges: Array.from(this.edgesById.values()).sort((left, right) => left.edge_id.localeCompare(right.edge_id))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function inferDomainFromUnit(unit: ProblemUnit): GraphDomainKey {
|
||||
if (unit.lifecycle_domain) {
|
||||
return unit.lifecycle_domain;
|
||||
}
|
||||
|
||||
const accountText = unit.affected_accounts.join(" ").toLowerCase();
|
||||
if (/\b97\b/.test(accountText) || unit.problem_unit_type === "lifecycle_anomaly_node") {
|
||||
return "deferred_expense";
|
||||
}
|
||||
if (/\b(01|02|08)\b/.test(accountText)) {
|
||||
return "fixed_asset";
|
||||
}
|
||||
if (/\b(19|68)\b/.test(accountText) || unit.problem_unit_type === "cross_branch_inconsistency_cluster") {
|
||||
return "vat_flow";
|
||||
}
|
||||
if (unit.problem_unit_type === "period_risk_cluster" || unit.period_impact?.impact_class === "close_risk") {
|
||||
return "period_close";
|
||||
}
|
||||
if (/\b62\b/.test(accountText)) {
|
||||
return "customer_settlement";
|
||||
}
|
||||
if (/\b(51|60|76)\b/.test(accountText) || unit.problem_unit_type === "unresolved_settlement_cluster") {
|
||||
return "bank_settlement";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function relationPathHints(domain: GraphDomainKey, unit: ProblemUnit): string[] {
|
||||
const path: string[] = [`domain:${domain}`];
|
||||
if (unit.current_lifecycle_state && unit.expected_lifecycle_state) {
|
||||
path.push(`state:${unit.current_lifecycle_state}->${unit.expected_lifecycle_state}`);
|
||||
} else if (unit.current_lifecycle_state) {
|
||||
path.push(`state:${unit.current_lifecycle_state}`);
|
||||
}
|
||||
|
||||
if (domain !== "unknown") {
|
||||
path.push(...(DOMAIN_PATH_HINTS[domain] ?? []));
|
||||
}
|
||||
if (unit.missing_transition) {
|
||||
path.push(`missing:${unit.missing_transition}`);
|
||||
}
|
||||
if (unit.invalid_transition) {
|
||||
path.push(`conflict:${unit.invalid_transition}`);
|
||||
}
|
||||
return uniqueStrings(path, 10);
|
||||
}
|
||||
|
||||
function graphConfidenceFromUnit(unit: ProblemUnit): GraphConfidenceGrade {
|
||||
return unit.lifecycle_confidence?.grade ?? unit.confidence.grade;
|
||||
}
|
||||
|
||||
function coverageGrade(boundUnits: number, totalUnits: number): GraphConfidenceGrade {
|
||||
if (totalUnits <= 0) {
|
||||
return "low";
|
||||
}
|
||||
const ratio = boundUnits / totalUnits;
|
||||
if (ratio >= 0.8) return "high";
|
||||
if (ratio >= 0.4) return "medium";
|
||||
return "low";
|
||||
}
|
||||
|
||||
function buildSummary(input: {
|
||||
nodes: AccountingGraphNode[];
|
||||
edges: AccountingGraphEdge[];
|
||||
bindings: ProblemUnitGraphBinding[];
|
||||
totalUnits: number;
|
||||
}): GraphSignalSummary {
|
||||
const domain_distribution: GraphSignalSummary["domain_distribution"] = {};
|
||||
const relation_distribution: GraphSignalSummary["relation_distribution"] = {};
|
||||
|
||||
for (const binding of input.bindings) {
|
||||
const domainMarker = binding.relation_path.find((item) => item.startsWith("domain:")) ?? "domain:unknown";
|
||||
const domainKey = domainMarker.replace(/^domain:/, "") as GraphDomainKey;
|
||||
domain_distribution[domainKey] = (domain_distribution[domainKey] ?? 0) + 1;
|
||||
}
|
||||
|
||||
for (const edge of input.edges) {
|
||||
relation_distribution[edge.relation_type] = (relation_distribution[edge.relation_type] ?? 0) + 1;
|
||||
}
|
||||
|
||||
const missing_links_count = input.bindings.reduce((acc, item) => acc + item.missing_links.length, 0);
|
||||
const conflicting_links_count = input.bindings.reduce((acc, item) => acc + item.conflicting_links.length, 0);
|
||||
const bound_units = input.bindings.length;
|
||||
|
||||
return {
|
||||
total_units: input.totalUnits,
|
||||
bound_units,
|
||||
node_count: input.nodes.length,
|
||||
edge_count: input.edges.length,
|
||||
missing_links_count,
|
||||
conflicting_links_count,
|
||||
graph_coverage_grade: coverageGrade(bound_units, input.totalUnits),
|
||||
domain_distribution,
|
||||
relation_distribution
|
||||
};
|
||||
}
|
||||
|
||||
export function buildAccountingGraph(input: BuildAccountingGraphInput): AccountingGraphBuildResult {
|
||||
const accumulator = new GraphAccumulator(input.route);
|
||||
const candidateById = new Map(input.candidateEvidence.map((item) => [item.candidate_id, item] as const));
|
||||
const bindings: ProblemUnitGraphBinding[] = [];
|
||||
const issues: string[] = [];
|
||||
|
||||
if (input.problemUnits.length === 0) {
|
||||
issues.push("no_problem_units_for_graph_build");
|
||||
}
|
||||
|
||||
for (const unit of input.problemUnits) {
|
||||
const domain = inferDomainFromUnit(unit);
|
||||
const confidence = graphConfidenceFromUnit(unit);
|
||||
const candidateIds = uniqueStrings(unit.evidence_pack, 12).filter((item) => candidateById.has(item));
|
||||
const evidenceIds = uniqueStrings(unit.evidence_pack, 12);
|
||||
|
||||
const domainNode = accumulator.upsertNode({
|
||||
node_type: "domain",
|
||||
domain,
|
||||
stable_key: `domain:${domain}`,
|
||||
label: domain,
|
||||
confidence,
|
||||
attributes: {
|
||||
domain
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
|
||||
const problemNode = accumulator.upsertNode({
|
||||
node_type: "problem_unit",
|
||||
domain,
|
||||
stable_key: `problem:${unit.problem_unit_id}`,
|
||||
label: unit.title || unit.problem_unit_id,
|
||||
confidence,
|
||||
attributes: {
|
||||
problem_unit_id: unit.problem_unit_id,
|
||||
problem_unit_type: unit.problem_unit_type,
|
||||
business_defect_class: unit.business_defect_class,
|
||||
lifecycle_defect_type: unit.lifecycle_defect_type ?? null
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "belongs_to_domain",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: domainNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: candidateIds,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
|
||||
for (const account of uniqueStrings(unit.affected_accounts, 4)) {
|
||||
const accountNode = accumulator.upsertNode({
|
||||
node_type: "account",
|
||||
domain,
|
||||
stable_key: `account:${account}`,
|
||||
label: account,
|
||||
confidence,
|
||||
attributes: {
|
||||
account
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_account",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: accountNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const document of uniqueStrings(unit.affected_documents, 3)) {
|
||||
const documentNode = accumulator.upsertNode({
|
||||
node_type: "document",
|
||||
domain,
|
||||
stable_key: `document:${document}`,
|
||||
label: document,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_document",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: documentNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const counterparty of uniqueStrings(unit.affected_counterparties, 3)) {
|
||||
const counterpartyNode = accumulator.upsertNode({
|
||||
node_type: "counterparty",
|
||||
domain,
|
||||
stable_key: `counterparty:${counterparty}`,
|
||||
label: counterparty,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "affects_counterparty",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: counterpartyNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.current_lifecycle_state) {
|
||||
const currentNode = accumulator.upsertNode({
|
||||
node_type: "lifecycle_state",
|
||||
domain,
|
||||
stable_key: `current_state:${unit.current_lifecycle_state}`,
|
||||
label: unit.current_lifecycle_state,
|
||||
confidence,
|
||||
attributes: {
|
||||
state_role: "current"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "current_state",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: currentNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.expected_lifecycle_state) {
|
||||
const expectedNode = accumulator.upsertNode({
|
||||
node_type: "lifecycle_state",
|
||||
domain,
|
||||
stable_key: `expected_state:${unit.expected_lifecycle_state}`,
|
||||
label: unit.expected_lifecycle_state,
|
||||
confidence,
|
||||
attributes: {
|
||||
state_role: "expected"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "expected_state",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: expectedNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["expected_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.missing_transition) {
|
||||
const missingNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `missing_transition:${unit.missing_transition}`,
|
||||
label: unit.missing_transition,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "missing"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "missing_transition",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: missingNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["missing_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.invalid_transition) {
|
||||
const invalidNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `invalid_transition:${unit.invalid_transition}`,
|
||||
label: unit.invalid_transition,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "invalid"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "invalid_transition",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: invalidNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["conflict_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (unit.lifecycle_defect_type) {
|
||||
const defectNode = accumulator.upsertNode({
|
||||
node_type: "defect",
|
||||
domain,
|
||||
stable_key: `defect:${unit.lifecycle_defect_type}`,
|
||||
label: unit.lifecycle_defect_type,
|
||||
confidence,
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "has_defect",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: defectNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: unit.lifecycle_defect_type === "cross_branch_state_conflict" ? ["conflict_link"] : ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
evidence_ids: evidenceIds
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const candidateId of candidateIds.slice(0, 6)) {
|
||||
const candidate = candidateById.get(candidateId);
|
||||
const evidenceLabel = candidate
|
||||
? `${candidate.source_ref.entity}:${candidate.source_ref.id}`
|
||||
: `candidate:${candidateId}`;
|
||||
const evidenceNode = accumulator.upsertNode({
|
||||
node_type: "evidence",
|
||||
domain,
|
||||
stable_key: `evidence:${candidateId}`,
|
||||
label: evidenceLabel,
|
||||
confidence,
|
||||
attributes: {
|
||||
candidate_id: candidateId
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "supported_by_evidence",
|
||||
from_node_id: problemNode.node_id,
|
||||
to_node_id: evidenceNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
|
||||
if (candidate && candidate.relation_pattern_hits.length > 0) {
|
||||
for (const relationHint of uniqueStrings(candidate.relation_pattern_hits, 2)) {
|
||||
const hintNode = accumulator.upsertNode({
|
||||
node_type: "transition",
|
||||
domain,
|
||||
stable_key: `hint:${relationHint}`,
|
||||
label: relationHint,
|
||||
confidence,
|
||||
attributes: {
|
||||
transition_role: "hint"
|
||||
},
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
accumulator.upsertEdge({
|
||||
relation_type: "supports_path",
|
||||
from_node_id: evidenceNode.node_id,
|
||||
to_node_id: hintNode.node_id,
|
||||
domain,
|
||||
confidence,
|
||||
flags: ["actual_link"],
|
||||
provenance: {
|
||||
route: input.route,
|
||||
candidate_ids: [candidateId],
|
||||
evidence_ids: [candidateId]
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const missing_links = uniqueStrings([
|
||||
unit.missing_transition,
|
||||
...(unit.lifecycle_resolution?.missing_transitions ?? [])
|
||||
]);
|
||||
const conflicting_links = uniqueStrings([
|
||||
unit.invalid_transition,
|
||||
...(unit.lifecycle_resolution?.invalid_transitions ?? [])
|
||||
]);
|
||||
|
||||
bindings.push({
|
||||
problem_unit_id: unit.problem_unit_id,
|
||||
graph_node_id: problemNode.node_id,
|
||||
relation_path: relationPathHints(domain, unit),
|
||||
missing_links,
|
||||
conflicting_links,
|
||||
provenance_evidence_ids: evidenceIds,
|
||||
graph_confidence: confidence
|
||||
});
|
||||
}
|
||||
|
||||
const exported = accumulator.export();
|
||||
const summary = buildSummary({
|
||||
nodes: exported.nodes,
|
||||
edges: exported.edges,
|
||||
bindings,
|
||||
totalUnits: input.problemUnits.length
|
||||
});
|
||||
|
||||
if (summary.bound_units < summary.total_units) {
|
||||
issues.push("some_problem_units_not_bound_to_graph");
|
||||
}
|
||||
if (summary.node_count === 0 || summary.edge_count === 0) {
|
||||
issues.push("graph_runtime_empty");
|
||||
}
|
||||
|
||||
return {
|
||||
schema_version: ACCOUNTING_GRAPH_SCHEMA_VERSION,
|
||||
nodes: exported.nodes,
|
||||
edges: exported.edges,
|
||||
unit_bindings: bindings,
|
||||
summary,
|
||||
issues: uniqueStrings(issues, 8)
|
||||
};
|
||||
}
|
||||
@@ -52,10 +52,22 @@ function redactSecrets(payload: Record<string, unknown>): Record<string, unknown
|
||||
return output;
|
||||
}
|
||||
|
||||
function isNoSpaceError(error: unknown): boolean {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
return code === "ENOSPC";
|
||||
}
|
||||
|
||||
export function saveTrace(record: TraceRecord): void {
|
||||
ensureDir(TRACES_DIR);
|
||||
const target = path.resolve(TRACES_DIR, `${record.trace_id}.json`);
|
||||
writeJsonFile(target, record);
|
||||
try {
|
||||
ensureDir(TRACES_DIR);
|
||||
const target = path.resolve(TRACES_DIR, `${record.trace_id}.json`);
|
||||
writeJsonFile(target, record);
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function listTraces(limit = 100): HistoryListItem[] {
|
||||
@@ -97,8 +109,15 @@ export function getTrace(traceId: string): TraceRecord | null {
|
||||
}
|
||||
|
||||
export function savePreset(preset: PromptPreset): void {
|
||||
ensureDir(PRESETS_DIR);
|
||||
writeJsonFile(path.resolve(PRESETS_DIR, `${preset.id}.json`), preset);
|
||||
try {
|
||||
ensureDir(PRESETS_DIR);
|
||||
writeJsonFile(path.resolve(PRESETS_DIR, `${preset.id}.json`), preset);
|
||||
} catch (error) {
|
||||
if (isNoSpaceError(error)) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function listPresets(): PromptPreset[] {
|
||||
@@ -114,9 +133,15 @@ export function listPresets(): PromptPreset[] {
|
||||
}
|
||||
|
||||
export function saveEvalCase(casePayload: Record<string, unknown>): string {
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
const id = String(casePayload.case_id ?? `NQ-${Date.now()}`);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${id}.json`), casePayload);
|
||||
try {
|
||||
ensureDir(EVAL_CASES_DIR);
|
||||
writeJsonFile(path.resolve(EVAL_CASES_DIR, `${id}.json`), casePayload);
|
||||
} catch (error) {
|
||||
if (!isNoSpaceError(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
ProblemUnit,
|
||||
ProblemUnitSummary
|
||||
} from "./stage2ProblemUnits";
|
||||
import type { AccountingGraphBuildResult } from "./stage4Graph";
|
||||
|
||||
export type AssistantFallbackType = "none" | "out_of_scope" | "clarification" | "partial" | "unknown";
|
||||
export type AssistantReplyType =
|
||||
@@ -99,6 +100,7 @@ export interface UnifiedRetrievalResult {
|
||||
candidate_evidence?: CandidateEvidenceItem[];
|
||||
problem_units?: ProblemUnit[];
|
||||
problem_unit_summary?: ProblemUnitSummary | null;
|
||||
accounting_graph?: AccountingGraphBuildResult;
|
||||
summary: Record<string, unknown>;
|
||||
evidence: EvidenceItem[];
|
||||
why_included: string[];
|
||||
@@ -138,6 +140,10 @@ export interface AssistantDebugPayload {
|
||||
answer_structure_v11: AnswerStructureV11 | null;
|
||||
investigation_state_snapshot: InvestigationStateWithProblemUnits | null;
|
||||
normalized: NormalizeResponsePayload["normalized"];
|
||||
normalizer_output?: {
|
||||
contains_multiple_tasks?: boolean;
|
||||
scope_confidence?: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface AssistantConversationItem {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { AssistantEvalBroadnessLevel, AssistantEvalQuestionType } from "./stage1Contracts";
|
||||
import type { ProblemUnitType } from "./stage2ProblemUnits";
|
||||
|
||||
export type EvalTarget = "normalizer" | "assistant_stage1" | "assistant_stage2";
|
||||
export type EvalTarget = "normalizer" | "assistant_stage1" | "assistant_stage2" | "assistant_p0";
|
||||
|
||||
export interface AssistantStage1SuiteCaseTurn {
|
||||
user_message: string;
|
||||
|
||||
@@ -33,6 +33,12 @@ export interface InvestigationFollowupContext {
|
||||
previous_question_id: string | null;
|
||||
last_user_message: string;
|
||||
referenced_requirement_ids: string[];
|
||||
active_domain?: string | null;
|
||||
active_requirement_ids?: string[];
|
||||
uncovered_requirement_ids?: string[];
|
||||
last_problem_unit_id?: string | null;
|
||||
settlement_next_actions?: string[];
|
||||
evidence_summary?: string[];
|
||||
}
|
||||
|
||||
export interface InvestigationState {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InvestigationState } from "./stage1Contracts";
|
||||
import type { EvidenceSourceRef } from "./stage1Contracts";
|
||||
import type { LifecycleConfidence, LifecycleDefectType, LifecycleDomain, LifecycleResolution } from "./stage3Lifecycle";
|
||||
import type { GraphSignalSummary, ProblemUnitGraphBinding } from "./stage4Graph";
|
||||
|
||||
export const CANDIDATE_EVIDENCE_SCHEMA_VERSION = "candidate_evidence_v0_1" as const;
|
||||
export const PROBLEM_UNIT_SCHEMA_VERSION = "problem_unit_v0_1" as const;
|
||||
@@ -89,6 +90,7 @@ export interface ProblemUnit {
|
||||
lifecycle_resolution?: LifecycleResolution;
|
||||
lifecycle_ranking_score?: number;
|
||||
lifecycle_ranking_basis?: string[];
|
||||
graph_binding?: ProblemUnitGraphBinding;
|
||||
}
|
||||
|
||||
export interface ProblemUnitSummary {
|
||||
@@ -103,6 +105,7 @@ export interface ProblemUnitSummary {
|
||||
lifecycle_enriched_units?: number;
|
||||
lifecycle_domain_distribution?: Partial<Record<LifecycleDomain, number>>;
|
||||
lifecycle_defect_distribution?: Partial<Record<LifecycleDefectType, number>>;
|
||||
graph_summary?: GraphSignalSummary;
|
||||
}
|
||||
|
||||
export interface InvestigationProblemUnitState {
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { LifecycleDomain } from "./stage3Lifecycle";
|
||||
|
||||
export const ACCOUNTING_GRAPH_SCHEMA_VERSION = "accounting_graph_v0_1" as const;
|
||||
|
||||
export type GraphConfidenceGrade = "low" | "medium" | "high";
|
||||
export type GraphDomainKey = LifecycleDomain | "unknown";
|
||||
|
||||
export const ACCOUNTING_GRAPH_NODE_TYPES = [
|
||||
"domain",
|
||||
"problem_unit",
|
||||
"account",
|
||||
"document",
|
||||
"counterparty",
|
||||
"lifecycle_state",
|
||||
"transition",
|
||||
"defect",
|
||||
"evidence"
|
||||
] as const;
|
||||
|
||||
export type AccountingGraphNodeType = (typeof ACCOUNTING_GRAPH_NODE_TYPES)[number];
|
||||
|
||||
export const ACCOUNTING_GRAPH_RELATION_TYPES = [
|
||||
"belongs_to_domain",
|
||||
"affects_account",
|
||||
"affects_document",
|
||||
"affects_counterparty",
|
||||
"current_state",
|
||||
"expected_state",
|
||||
"missing_transition",
|
||||
"invalid_transition",
|
||||
"has_defect",
|
||||
"supported_by_evidence",
|
||||
"supports_path"
|
||||
] as const;
|
||||
|
||||
export type AccountingGraphRelationType = (typeof ACCOUNTING_GRAPH_RELATION_TYPES)[number];
|
||||
export type AccountingGraphEdgeFlag = "expected_link" | "actual_link" | "missing_link" | "conflict_link";
|
||||
|
||||
export interface AccountingGraphProvenance {
|
||||
route: string;
|
||||
candidate_ids: string[];
|
||||
evidence_ids: string[];
|
||||
}
|
||||
|
||||
export interface AccountingGraphNode {
|
||||
node_id: string;
|
||||
node_type: AccountingGraphNodeType;
|
||||
label: string;
|
||||
domain: GraphDomainKey;
|
||||
confidence: GraphConfidenceGrade;
|
||||
attributes: Record<string, unknown>;
|
||||
provenance: AccountingGraphProvenance;
|
||||
}
|
||||
|
||||
export interface AccountingGraphEdge {
|
||||
edge_id: string;
|
||||
relation_type: AccountingGraphRelationType;
|
||||
from_node_id: string;
|
||||
to_node_id: string;
|
||||
domain: GraphDomainKey;
|
||||
confidence: GraphConfidenceGrade;
|
||||
flags: AccountingGraphEdgeFlag[];
|
||||
provenance: AccountingGraphProvenance;
|
||||
}
|
||||
|
||||
export interface ProblemUnitGraphBinding {
|
||||
problem_unit_id: string;
|
||||
graph_node_id: string;
|
||||
relation_path: string[];
|
||||
missing_links: string[];
|
||||
conflicting_links: string[];
|
||||
provenance_evidence_ids: string[];
|
||||
graph_confidence: GraphConfidenceGrade;
|
||||
}
|
||||
|
||||
export interface GraphSignalSummary {
|
||||
total_units: number;
|
||||
bound_units: number;
|
||||
node_count: number;
|
||||
edge_count: number;
|
||||
missing_links_count: number;
|
||||
conflicting_links_count: number;
|
||||
graph_coverage_grade: GraphConfidenceGrade;
|
||||
domain_distribution: Partial<Record<GraphDomainKey, number>>;
|
||||
relation_distribution: Partial<Record<AccountingGraphRelationType, number>>;
|
||||
}
|
||||
|
||||
export interface AccountingGraphBuildResult {
|
||||
schema_version: typeof ACCOUNTING_GRAPH_SCHEMA_VERSION;
|
||||
nodes: AccountingGraphNode[];
|
||||
edges: AccountingGraphEdge[];
|
||||
unit_bindings: ProblemUnitGraphBinding[];
|
||||
summary: GraphSignalSummary;
|
||||
issues: string[];
|
||||
}
|
||||
@@ -37,5 +37,13 @@ export function logJson(entry: JsonLogEntry): void {
|
||||
details: redactObject(entry.details)
|
||||
};
|
||||
// Structured JSON logs for diagnostics/trace aggregation.
|
||||
process.stdout.write(JSON.stringify(safe) + "\n");
|
||||
try {
|
||||
process.stdout.write(JSON.stringify(safe) + "\n");
|
||||
} catch (error) {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
if (code === "ENOSPC" || code === "EPIPE") {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user