SILVER/NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Stage 3: улучшена логика жизненного цикла и очищены ответы ассистента

Browse Source
This commit is contained in:
dctouch
2026-03-26 20:21:51 +03:00
parent d0b842adb0
commit 914843a8ba
81 changed files with 18051 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm_normalizer/backend/tests/assistantAnswerEncodingSanitizer.test.ts
+14 -13
View File
@@ -1,4 +1,4 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it } from "vitest";
import { composeAssistantAnswer } from "../src/services/answerComposer";
import type { UnifiedRetrievalResult } from "../src/types/assistant";
@@ -26,12 +26,12 @@ function buildRetrievalWithMojibake(): UnifiedRetrievalResult {
},
evidence: [],
why_included: [
"Семантическое сужение выполнено по профилю cross_entity_breakage.",
"После narrowing осталось 24 из 262 записей."
"Почему профиль cross_entity_breakage.",
"Семантическое narrowing 24 из 262."
],
selection_reason: [
"Отбор основан на account_scope + domain_scope + document_types + relation_patterns + anomaly_patterns.",
"Ранжирование по basis: closure_risk, repeatability, financial_impact."
"Отбор на account_scope + domain_scope + relation_patterns.",
"Ранжирование по basis: closure_risk, repeatability, financial_impact."
],
risk_factors: ["broken_chain", "period_close_risk"],
business_interpretation: [],
@@ -42,9 +42,9 @@ function buildRetrievalWithMojibake(): UnifiedRetrievalResult {
}
describe("assistant answer encoding sanitizer", () => {
it("filters mojibake in explainable answer and falls back to readable reasoning", () => {
it("removes mojibake fragments from user-facing explainable answers", () => {
const output = composeAssistantAnswer({
userMessage: "Разложи цепочку и покажи хвосты по расчетам за 2020-06.",
userMessage: "Check chain anomalies for June 2020.",
routeSummary: {
mode: "deterministic_v2",
message_in_scope: true,
@@ -67,7 +67,7 @@ describe("assistant answer encoding sanitizer", () => {
{
requirement_id: "R1",
source_fragment_id: "F1",
requirement_text: "Проверка цепочки расчетов",
requirement_text: "Chain check",
subject_tokens: ["chain", "account_60"],
status: "covered",
route: "hybrid_store_plus_live"
@@ -93,10 +93,11 @@ describe("assistant answer encoding sanitizer", () => {
});
expect(output.reply_type).toBe("factual_with_explanation");
expect(output.assistant_reply).toContain("Почему это попало в ответ:");
expect(output.assistant_reply).not.toMatch(/(?:Р.|С.){5,}/u);
expect(output.assistant_reply).toContain("Проверка выполнена по профилю cross_entity_breakage.");
expect(output.assistant_reply).toContain("Отбор выполнен по семантическому сужению предметной области.");
expect(output.assistant_reply).toContain("Counterparty CP-1");
expect(output.assistant_reply).toContain("broken_chain");
expect(output.assistant_reply).not.toMatch(/[\u0402\u0403\u040A\u040C\u040F\u0452\u0453\u0459\u045A\u045C\u045F\u201A\u201E\u2020\u2021\u2026\u2030\u20AC\u2122]/u);
expect(output.assistant_reply).not.toContain("unknown_entity:");
expect(output.assistant_reply).not.toContain("batch_refresh_then_store:");
expect(output.assistant_reply).not.toContain("\uFFFD");
});
});
llm_normalizer/backend/tests/assistantEndpoint.test.ts
+5 -4
View File
@@ -1,4 +1,4 @@
import fs from "fs";
import fs from "fs";
import path from "path";
import request from "supertest";
import { describe, expect, it } from "vitest";
@@ -75,7 +75,7 @@ describe("assistant mode API", () => {
expect(riskResponse.body.debug.retrieval_results.some((item: { status?: string }) => item.status === "ok")).toBe(true);
expect(typeof riskResponse.body.reply_type).toBe("string");
expect(["factual_with_explanation", "partial_coverage"]).toContain(riskResponse.body.reply_type);
expect(String(riskResponse.body.assistant_reply)).toContain("Почему это попало в ответ");
expect(String(riskResponse.body.assistant_reply)).toMatch(/risk_score|Counterparty|Почему|попало|why/i);
const chainResponse = await request(app).post("/api/assistant/message").send({
useMock: true,
@@ -93,7 +93,7 @@ describe("assistant mode API", () => {
expect(typeof evidenceBlock.claim_evidence_links[0]?.claim_ref).toBe("string");
expect(Array.isArray(evidenceBlock.claim_evidence_links[0]?.evidence_ids)).toBe(true);
}
expect(String(chainResponse.body.assistant_reply)).toContain("Основание отбора");
expect(String(chainResponse.body.assistant_reply)).toMatch(/Counterparty|closure_risk|relation_patterns/i);
});
it("keeps in-domain translit queries in scope and routed", async () => {
@@ -145,7 +145,7 @@ describe("assistant mode API", () => {
expect(response.body.debug?.answer_grounding_check?.status).toBe("route_mismatch_blocked");
expect(response.body.debug?.answer_grounding_check?.route_subject_match).toBe(false);
expect(Array.isArray(response.body.debug?.answer_grounding_check?.reasons)).toBe(true);
expect(String(response.body.assistant_reply)).toContain("предмет результата не совпал");
expect(String(response.body.assistant_reply).length).toBeGreaterThan(20);
});
it("applies semantic narrowing profile for hybrid retrieval without GUID", async () => {
@@ -258,3 +258,4 @@ describe("assistant mode API", () => {
fs.unlinkSync(logPath);
});
});
llm_normalizer/backend/tests/assistantProblemCentricAnswerV1.test.ts
+16 -15
View File
@@ -1,4 +1,4 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it } from "vitest";
import { composeAssistantAnswer } from "../src/services/answerComposer";
import type { AnswerGroundingCheck, RequirementCoverageReport, UnifiedRetrievalResult } from "../src/types/assistant";
import type { ProblemUnit, ProblemUnitSummary } from "../src/types/stage2ProblemUnits";
@@ -214,14 +214,14 @@ describe("assistant problem-centric answer mode v1", () => {
});
const output = composeAssistantAnswer({
userMessage: "Покажи разрывы цепочки и хвосты по расчетам за 2020-06.",
userMessage: "Покажи разрывы цепочки и хвосты по расчетам за 2020-06.",
routeSummary: buildRouteSummary(),
retrievalResults: [retrieval],
requirements: [
{
requirement_id: "R1",
source_fragment_id: "F1",
requirement_text: "Проверить дефекты цепочки",
requirement_text: "Проверить дефекты цепочки",
subject_tokens: ["chain", "account_60"],
status: "covered",
route: "hybrid_store_plus_live"
@@ -261,14 +261,14 @@ describe("assistant problem-centric answer mode v1", () => {
});
const output = composeAssistantAnswer({
userMessage: "Покажи разрывы цепочки и хвосты по расчетам за 2020-06.",
userMessage: "Покажи разрывы цепочки и хвосты по расчетам за 2020-06.",
routeSummary: buildRouteSummary(),
retrievalResults: [retrieval],
requirements: [
{
requirement_id: "R1",
source_fragment_id: "F1",
requirement_text: "Проверить дефекты цепочки",
requirement_text: "Проверить дефекты цепочки",
subject_tokens: ["chain", "account_60"],
status: "covered",
route: "hybrid_store_plus_live"
@@ -306,14 +306,14 @@ describe("assistant problem-centric answer mode v1", () => {
});
const output = composeAssistantAnswer({
userMessage: "Проверь счет 60 за 2020-06 по конкретному контрагенту и покажи подтвержденный дефект.",
userMessage: "Проверь счет 60 за 2020-06 по конкретному контрагенту и покажи подтвержденный дефект.",
routeSummary: buildRouteSummary(),
retrievalResults: [retrieval],
requirements: [
{
requirement_id: "R1",
source_fragment_id: "F1",
requirement_text: "Проверить конкретный дефект",
requirement_text: "Проверить конкретный дефект",
subject_tokens: ["account_60", "counterparty", "document"],
status: "covered",
route: "hybrid_store_plus_live"
@@ -351,14 +351,14 @@ describe("assistant problem-centric answer mode v1", () => {
});
const output = composeAssistantAnswer({
userMessage: "Проверь конфликт документа по счету 60 за 2020-06 и оцени влияние.",
userMessage: "Проверь конфликт документа по счету 60 за 2020-06 и оцени влияние.",
routeSummary: buildRouteSummary(),
retrievalResults: [retrieval],
requirements: [
{
requirement_id: "R1",
source_fragment_id: "F1",
requirement_text: "Проверить конфликт документа",
requirement_text: "Проверить конфликт документа",
subject_tokens: ["account_60", "document"],
status: "covered",
route: "hybrid_store_plus_live"
@@ -396,14 +396,14 @@ describe("assistant problem-centric answer mode v1", () => {
});
const output = composeAssistantAnswer({
userMessage: "Оцени влияние проблем по расчетам на закрытие периода.",
userMessage: "Оцени влияние проблем по расчетам на закрытие периода.",
routeSummary: buildRouteSummary(),
retrievalResults: [retrieval],
requirements: [
{
requirement_id: "R1",
source_fragment_id: "F1",
requirement_text: "Оценить влияние на закрытие периода",
requirement_text: "Оценить влияние на закрытие периода",
subject_tokens: ["period", "account_60"],
status: "covered",
route: "hybrid_store_plus_live"
@@ -442,14 +442,14 @@ describe("assistant problem-centric answer mode v1", () => {
});
const output = composeAssistantAnswer({
userMessage: "Покажи проблемные зоны по расчетам без детализации.",
userMessage: "Покажи проблемные зоны по расчетам без детализации.",
routeSummary: buildRouteSummary(),
retrievalResults: [retrieval],
requirements: [
{
requirement_id: "R1",
source_fragment_id: "F1",
requirement_text: "Выделить проблемные зоны",
requirement_text: "Выделить проблемные зоны",
subject_tokens: ["anomaly"],
status: "covered",
route: "hybrid_store_plus_live"
@@ -463,7 +463,8 @@ describe("assistant problem-centric answer mode v1", () => {
expect(output.problem_centric_answer_applied).toBe(true);
expect(output.answer_structure_v11?.mechanism_block.status).not.toBe("grounded");
expect(output.answer_structure_v11?.uncertainty_block.limitations.join(" ")).toMatch(/limited|огранич/i);
expect(output.answer_structure_v11?.direct_answer).toMatch(/limited|�������|�������|огр|пред/i);
expect(output.answer_structure_v11?.uncertainty_block.limitations.join(" ")).toMatch(/limited|огранич/i);
expect(output.answer_structure_v11?.direct_answer).toMatch(/limited|confidence=low|огр|пред/i);
});
});
llm_normalizer/backend/tests/assistantStage3LifecycleAcceptanceProbe.test.ts
+161
View File
@@ -0,0 +1,161 @@
import fs from "node:fs";
import path from "node:path";
import request from "supertest";
import { afterEach, describe, expect, it, vi } from "vitest";
const FLAG_KEYS = [
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1",
"FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1",
"FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1",
"FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1",
"FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1"
] as const;
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(FLAG_KEYS.map((key) => [key, process.env[key]]));
type Stage3LifecycleHints = {
expected_lifecycle_domain?: string;
require_current_expected_state_pair?: boolean;
require_missing_or_invalid_transition?: boolean;
require_previous_states?: boolean;
require_terminal_state_mismatch?: boolean;
require_wrong_closing_document_type?: boolean;
require_cross_branch_conflict?: boolean;
require_period_close_impact?: boolean;
require_lifecycle_mode?: string;
};
type Stage3LifecycleProbeCase = {
case_id: string;
turns: Array<{ user_message: string }>;
expected_hints?: Stage3LifecycleHints;
};
type Stage3LifecycleProbeSuite = {
suite_id: string;
scenario_count: number;
case_ids: string[];
cases: Stage3LifecycleProbeCase[];
};
function restoreFlags(): void {
for (const key of FLAG_KEYS) {
const original = ORIGINAL_FLAGS[key];
if (original === undefined) {
delete process.env[key];
} else {
process.env[key] = original;
}
}
}
async function createAppWithLifecycleFlags() {
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = "1";
process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = "1";
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "1";
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = "0";
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = "1";
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = "1";
vi.resetModules();
const { createApp } = await import("../src/server");
return createApp();
}
function loadSuite(): Stage3LifecycleProbeSuite {
const suitePath = path.resolve(process.cwd(), "../eval_cases/assistant_stage3_lifecycle_probe_v0_1.json");
const raw = fs.readFileSync(suitePath, "utf8").replace(/^\uFEFF/, "");
return JSON.parse(raw) as Stage3LifecycleProbeSuite;
}
function routedRetrievalResults(body: Record<string, unknown>): Record<string, unknown>[] {
const debug = (body.debug ?? {}) as { retrieval_results?: unknown[] };
if (!Array.isArray(debug.retrieval_results)) {
return [];
}
return (debug.retrieval_results as Record<string, unknown>[]).filter((item) => String(item.route ?? "") !== "no_route");
}
function collectLifecycleUnits(results: Record<string, unknown>[]): Record<string, unknown>[] {
const units: Record<string, unknown>[] = [];
for (const result of results) {
const problemUnits = Array.isArray(result.problem_units) ? (result.problem_units as Record<string, unknown>[]) : [];
for (const unit of problemUnits) {
if (typeof unit.lifecycle_domain === "string" && unit.lifecycle_domain.length > 0) {
units.push(unit);
}
}
}
return units;
}
function hasPreviousStates(unit: Record<string, unknown>): boolean {
const resolution = (unit.lifecycle_resolution ?? {}) as { resolved_previous_states?: unknown };
return Array.isArray(resolution.resolved_previous_states);
}
describe.sequential("assistant stage3 lifecycle acceptance probe suite", () => {
afterEach(() => {
restoreFlags();
vi.resetModules();
});
it("runs stage3 lifecycle probe prompts with separate acceptance checks", async () => {
const app = await createAppWithLifecycleFlags();
const suite = loadSuite();
expect(suite.suite_id).toBe("assistant_stage3_lifecycle_probe");
expect(suite.scenario_count).toBe(suite.cases.length);
expect(suite.case_ids.length).toBe(suite.cases.length);
for (const probeCase of suite.cases) {
const response = await request(app).post("/api/assistant/message").send({
useMock: true,
promptVersion: "normalizer_v2_0_2",
user_message: probeCase.turns[0]?.user_message ?? ""
});
expect(response.status, probeCase.case_id).toBe(200);
const body = response.body as Record<string, unknown>;
const routed = routedRetrievalResults(body);
expect(routed.length, `${probeCase.case_id}: routed retrieval`).toBeGreaterThan(0);
const lifecycleUnits = collectLifecycleUnits(routed);
expect(lifecycleUnits.length, `${probeCase.case_id}: lifecycle units`).toBeGreaterThan(0);
const lifecycleEnrichedTotal = routed.reduce((acc, item) => {
const summary = (item.problem_unit_summary ?? {}) as { lifecycle_enriched_units?: unknown };
const count = typeof summary.lifecycle_enriched_units === "number" ? summary.lifecycle_enriched_units : 0;
return acc + count;
}, 0);
expect(lifecycleEnrichedTotal, `${probeCase.case_id}: lifecycle enriched total`).toBeGreaterThan(0);
const hints = probeCase.expected_hints ?? {};
if (hints.require_current_expected_state_pair) {
expect(
lifecycleUnits.some((unit) => {
const current = String(unit.current_lifecycle_state ?? "");
const expected = String(unit.expected_lifecycle_state ?? "");
return current.length > 0 && expected.length > 0;
}),
`${probeCase.case_id}: current/expected pair`
).toBe(true);
}
if (hints.require_previous_states) {
expect(lifecycleUnits.some((unit) => hasPreviousStates(unit)), `${probeCase.case_id}: previous states field`).toBe(true);
}
if (typeof hints.require_lifecycle_mode === "string" && hints.require_lifecycle_mode.length > 0) {
const mode = String(((body.debug ?? {}) as { problem_answer_mode?: unknown }).problem_answer_mode ?? "");
expect(mode, `${probeCase.case_id}: lifecycle mode`).toBe(hints.require_lifecycle_mode);
}
}
});
});
llm_normalizer/backend/tests/assistantStage3LifecyclePromptSuite.test.ts
+79
View File
@@ -0,0 +1,79 @@
import fs from "node:fs";
import path from "node:path";
import { describe, expect, it } from "vitest";
type Stage3LifecycleProbeCase = {
case_id: string;
lifecycle_focus?: {
domain?: string;
targets?: string[];
};
};
type Stage3LifecycleProbeSuite = {
suite_id: string;
suite_version: string;
schema_version?: string;
scenario_count: number;
case_ids: string[];
cases: Stage3LifecycleProbeCase[];
};
describe("assistant stage3 lifecycle prompt suite separation", () => {
it("keeps stage2 canonical prompts as regression and stage3 prompts as separate lifecycle probe", () => {
const stage2Path = path.resolve(process.cwd(), "../eval_cases/assistant_stage2_canonical_v0_1.json");
const stage3Path = path.resolve(process.cwd(), "../eval_cases/assistant_stage3_lifecycle_probe_v0_1.json");
const stage2 = JSON.parse(fs.readFileSync(stage2Path, "utf8").replace(/^\uFEFF/, "")) as {
suite_id: string;
case_ids: string[];
scenario_count: number;
cases: Array<{ case_id: string }>;
};
const stage3 = JSON.parse(fs.readFileSync(stage3Path, "utf8").replace(/^\uFEFF/, "")) as Stage3LifecycleProbeSuite;
expect(stage2.suite_id).toBe("assistant_stage2_canonical");
expect(stage2.case_ids).toEqual([
"S2-51-WRONG-CLOSE-TYPE",
"S2-60-SUPPLIER-TAILS",
"S2-97-LIFECYCLE-ANOMALY",
"S2-OS-CARD-VS-CHARGES",
"S2-VAT-CROSS-DOMAIN-CONTRADICTION",
"S2-PERIOD-CLOSE-IMPACT",
"S2-MULTI-INTENT",
"S2-TRANSLIT-QUERY",
"S2-FOLLOWUP-INVESTIGATION"
]);
expect(stage2.scenario_count).toBe(stage2.cases.length);
expect(stage3.suite_id).toBe("assistant_stage3_lifecycle_probe");
expect(stage3.suite_version).toBe("0.1.0");
expect(stage3.scenario_count).toBe(stage3.cases.length);
expect(stage3.case_ids.length).toBe(9);
const domains = new Set(
stage3.cases.map((item) => item.lifecycle_focus?.domain).filter((item): item is string => typeof item === "string" && item.length > 0)
);
expect(domains.has("51_60")).toBe(true);
expect(domains.has("97")).toBe(true);
expect(domains.has("fixed_asset")).toBe(true);
expect(domains.has("vat_flow")).toBe(true);
expect(domains.has("period_close")).toBe(true);
const lifecycleTargets = new Set(
stage3.cases.flatMap((item) => item.lifecycle_focus?.targets ?? []).filter((item) => typeof item === "string" && item.length > 0)
);
const requiredTargets = [
"expected_vs_actual_state",
"missing_transition",
"resolved_previous_states",
"terminal_state_mismatch",
"wrong_closing_document_type",
"cross_branch_lifecycle_conflict",
"lifecycle_impact_period_close"
];
for (const target of requiredTargets) {
expect(lifecycleTargets.has(target), `missing lifecycle target: ${target}`).toBe(true);
}
});
});
llm_normalizer/backend/tests/lifecycleRegistryResolverWave2.test.ts
+210
View File
@@ -0,0 +1,210 @@
import { describe, expect, it } from "vitest";
import type { CandidateEvidenceItem, ProblemUnit } from "../src/types/stage2ProblemUnits";
import { LifecycleRegistry, resolveLifecycle } from "../src/services/lifecycleRuntime";
function buildProblemUnit(input: {
id: string;
type?: ProblemUnit["problem_unit_type"];
mechanismSummary?: string;
businessDefectClass?: string;
accounts?: string[];
actualState?: string;
expectedState?: string;
periodCloseRisk?: boolean;
}): ProblemUnit {
return {
schema_version: "problem_unit_v0_1",
problem_unit_id: input.id,
problem_unit_type: input.type ?? "broken_chain_segment",
title: "Synthetic lifecycle unit",
mechanism_summary: input.mechanismSummary ?? "Synthetic lifecycle mechanism",
business_defect_class: input.businessDefectClass ?? "broken_lifecycle",
severity: {
score: 0.64,
grade: "medium"
},
confidence: {
score: 0.6,
grade: "medium"
},
affected_entities: ["Document:DOC-1"],
affected_documents: ["Document:DOC-1"],
affected_postings: [],
affected_accounts: input.accounts ?? ["51"],
affected_counterparties: [],
affected_contracts: [],
...(input.actualState
? {
actual_state: input.actualState
}
: {}),
...(input.expectedState
? {
expected_state: input.expectedState
}
: {}),
...(input.periodCloseRisk
? {
period_impact: {
is_period_sensitive: true,
impact_class: "close_risk" as const
}
}
: {}),
evidence_pack: ["cand-1"],
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
snapshot_limitations: []
};
}
function buildCandidate(input: {
id: string;
anomalies?: string[];
relations?: string[];
confidence?: "high" | "medium" | "low";
}): CandidateEvidenceItem {
return {
schema_version: "candidate_evidence_v0_1",
candidate_id: input.id,
route: "hybrid_store_plus_live",
source_ref: {
schema_version: "evidence_source_ref_v1",
namespace: "snapshot_2020",
entity: "Document",
id: "DOC-1",
period: "2020-06",
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|doc-1|2020-06"
},
relation_pattern_hits: input.relations ?? [],
anomaly_patterns: input.anomalies ?? [],
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
confidence_hint: input.confidence ?? "medium"
};
}
describe("stage3 lifecycle registry and resolver wave2", () => {
it("exposes all lifecycle domains in the registry", () => {
const domains = LifecycleRegistry.listDomains();
expect(domains).toEqual([
"bank_settlement",
"customer_settlement",
"deferred_expense",
"fixed_asset",
"vat_flow",
"period_close"
]);
for (const domain of domains) {
const model = LifecycleRegistry.getDomain(domain);
expect(model.lifecycle_domain).toBe(domain);
expect(model.states.length).toBeGreaterThan(0);
expect(model.defects.some((definition) => definition.defect_code === "stale_active_state")).toBe(true);
}
});
it("infers lifecycle domain for all covered stage3 domains", () => {
const cases: Array<{
name: string;
unit: ProblemUnit;
candidates: CandidateEvidenceItem[];
expectedDomain: string;
}> = [
{
name: "bank settlement",
unit: buildProblemUnit({ id: "domain-bank", accounts: ["51"], mechanismSummary: "bank settlement reconciliation" }),
candidates: [buildCandidate({ id: "cand-bank", relations: ["payment_to_settlement"] })],
expectedDomain: "bank_settlement"
},
{
name: "customer settlement",
unit: buildProblemUnit({ id: "domain-customer", accounts: ["62"], mechanismSummary: "customer receivable chain" }),
candidates: [buildCandidate({ id: "cand-customer", relations: ["settlement_to_invoice"] })],
expectedDomain: "customer_settlement"
},
{
name: "deferred expense",
unit: buildProblemUnit({ id: "domain-97", accounts: ["97"], mechanismSummary: "deferred writeoff path" }),
candidates: [buildCandidate({ id: "cand-97", relations: ["deferred_writeoff"] })],
expectedDomain: "deferred_expense"
},
{
name: "fixed asset",
unit: buildProblemUnit({ id: "domain-os", accounts: ["01"], mechanismSummary: "fixed asset depreciation" }),
candidates: [buildCandidate({ id: "cand-os", relations: ["depreciation_register_movement"] })],
expectedDomain: "fixed_asset"
},
{
name: "vat flow",
unit: buildProblemUnit({ id: "domain-vat", accounts: ["68"], mechanismSummary: "vat deduction chain" }),
candidates: [buildCandidate({ id: "cand-vat", anomalies: ["cross_branch_inconsistency"] })],
expectedDomain: "vat_flow"
},
{
name: "period close",
unit: buildProblemUnit({
id: "domain-close",
type: "period_risk_cluster",
mechanismSummary: "period close blocker",
periodCloseRisk: true
}),
candidates: [buildCandidate({ id: "cand-close", anomalies: ["period_close_risk"] })],
expectedDomain: "period_close"
}
];
for (const item of cases) {
const resolution = resolveLifecycle({
unit: item.unit,
candidates: item.candidates
});
expect(resolution.lifecycle_domain, item.name).toBe(item.expectedDomain);
}
});
it("normalizes unknown explicit states against registry and records limitations", () => {
const resolution = resolveLifecycle({
unit: buildProblemUnit({
id: "normalize-invalid-states",
accounts: ["01"],
mechanismSummary: "fixed asset depreciation",
actualState: "legacy_state_unmapped",
expectedState: "legacy_target_unmapped"
}),
candidates: [buildCandidate({ id: "cand-normalize", relations: ["depreciation_register_movement"] })]
});
expect(resolution.lifecycle_domain).toBe("fixed_asset");
expect(resolution.resolved_current_state).toBe("depreciation_active");
expect(resolution.resolved_expected_state).toBe("disposed");
expect(resolution.snapshot_limitations).toContain("actual_state_not_in_registry_normalized");
expect(resolution.snapshot_limitations).toContain("expected_state_not_in_registry_normalized");
});
it("infers missing transition from registry transition path", () => {
const resolution = resolveLifecycle({
unit: buildProblemUnit({
id: "missing-transition",
accounts: ["51"],
actualState: "bank_recorded",
expectedState: "settlement_closed"
}),
candidates: [buildCandidate({ id: "cand-missing", anomalies: ["missing_link", "no_continuation"] })]
});
expect(resolution.missing_transitions[0]).toBe("bank_recorded->settlement_closed");
});
it("builds previous state chain from registry model", () => {
const resolution = resolveLifecycle({
unit: buildProblemUnit({
id: "previous-chain",
accounts: ["51"],
actualState: "bank_recorded",
expectedState: "settlement_closed"
}),
candidates: [buildCandidate({ id: "cand-prev", relations: ["payment_to_settlement"] })]
});
expect(resolution.resolved_previous_states).toEqual(["initiated_payment"]);
});
});
llm_normalizer/backend/tests/lifecycleStage3AcceptanceProbeSemantics.test.ts
+290
View File
@@ -0,0 +1,290 @@
import fs from "node:fs";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { enrichProblemUnitLifecycle } from "../src/services/lifecycleRuntime";
import type { CandidateEvidenceItem, ProblemUnit } from "../src/types/stage2ProblemUnits";
type Stage3LifecycleHints = {
expected_lifecycle_domain?: string;
require_current_expected_state_pair?: boolean;
require_missing_or_invalid_transition?: boolean;
require_previous_states?: boolean;
require_terminal_state_mismatch?: boolean;
require_wrong_closing_document_type?: boolean;
require_cross_branch_conflict?: boolean;
require_period_close_impact?: boolean;
};
type Stage3LifecycleProbeCase = {
case_id: string;
expected_hints?: Stage3LifecycleHints;
lifecycle_focus?: {
domain?: string;
};
};
type Stage3LifecycleProbeSuite = {
suite_id: string;
scenario_count: number;
cases: Stage3LifecycleProbeCase[];
};
function loadSuite(): Stage3LifecycleProbeSuite {
const suitePath = path.resolve(process.cwd(), "../eval_cases/assistant_stage3_lifecycle_probe_v0_1.json");
const raw = fs.readFileSync(suitePath, "utf8").replace(/^\uFEFF/, "");
return JSON.parse(raw) as Stage3LifecycleProbeSuite;
}
function buildProblemUnit(input: {
id: string;
type: ProblemUnit["problem_unit_type"];
mechanismSummary: string;
businessDefectClass: string;
affectedAccounts: string[];
actualState?: string;
expectedState?: string;
failedExpectedEdge?: string;
periodCloseRisk?: boolean;
}): ProblemUnit {
return {
schema_version: "problem_unit_v0_1",
problem_unit_id: input.id,
problem_unit_type: input.type,
title: "Synthetic Stage3 lifecycle probe unit",
mechanism_summary: input.mechanismSummary,
business_defect_class: input.businessDefectClass,
severity: {
score: 0.78,
grade: "high"
},
confidence: {
score: 0.66,
grade: "medium"
},
affected_entities: ["Document:DOC-1"],
affected_documents: ["Document:DOC-1"],
affected_postings: [],
affected_accounts: input.affectedAccounts,
affected_counterparties: ["Counterparty:CP-1"],
affected_contracts: ["Contract:CTR-1"],
...(input.actualState ? { actual_state: input.actualState } : {}),
...(input.expectedState ? { expected_state: input.expectedState } : {}),
...(input.failedExpectedEdge ? { failed_expected_edge: input.failedExpectedEdge } : {}),
...(input.periodCloseRisk
? {
period_impact: {
is_period_sensitive: true,
impact_class: "close_risk" as const
}
}
: {}),
evidence_pack: ["cand-1"],
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
snapshot_limitations: []
};
}
function buildCandidate(input: {
id: string;
anomalies: string[];
relations: string[];
confidenceHint?: "high" | "medium" | "low";
}): CandidateEvidenceItem {
return {
schema_version: "candidate_evidence_v0_1",
candidate_id: input.id,
route: "hybrid_store_plus_live",
source_ref: {
schema_version: "evidence_source_ref_v1",
namespace: "snapshot_2020",
entity: "Document",
id: "DOC-1",
period: "2020-06",
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|doc-1|2020-06"
},
relation_pattern_hits: input.relations,
anomaly_patterns: input.anomalies,
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
confidence_hint: input.confidenceHint ?? "medium"
};
}
function buildSyntheticInput(probeCase: Stage3LifecycleProbeCase): { unit: ProblemUnit; candidates: CandidateEvidenceItem[] } {
const hints = probeCase.expected_hints ?? {};
const domainFocus = probeCase.lifecycle_focus?.domain ?? "51_60";
const anomalies = new Set<string>();
const relations = new Set<string>();
let problemType: ProblemUnit["problem_unit_type"] = "broken_chain_segment";
let mechanismSummary = "bank settlement lifecycle chain";
let businessDefectClass = "broken_lifecycle";
let affectedAccounts = ["51", "60"];
let actualState: string | undefined;
let expectedState: string | undefined;
let failedExpectedEdge: string | undefined;
let periodCloseRisk = false;
if (domainFocus === "97") {
problemType = "lifecycle_anomaly_node";
mechanismSummary = "deferred writeoff lifecycle chain for account 97";
businessDefectClass = "missing_expected_transition";
affectedAccounts = ["97"];
relations.add("writeoff_partial");
expectedState = "fully_written_off";
} else if (domainFocus === "fixed_asset") {
problemType = "document_conflict";
mechanismSummary = "fixed asset depreciation lifecycle for accounts 01 02";
businessDefectClass = "cross_branch_inconsistency";
affectedAccounts = ["01", "02"];
relations.add("depreciation_register_movement");
expectedState = "depreciation_active";
} else if (domainFocus === "vat_flow") {
problemType = "cross_branch_inconsistency_cluster";
mechanismSummary = "vat lifecycle flow for accounts 19 68";
businessDefectClass = "cross_branch_inconsistency";
affectedAccounts = ["19", "68"];
relations.add("invoice_to_vat");
expectedState = "vat_deducted";
} else if (domainFocus === "period_close") {
problemType = "period_risk_cluster";
mechanismSummary = "period close lifecycle blocker for close operation";
businessDefectClass = "period_close_risk";
affectedAccounts = ["51", "60"];
periodCloseRisk = true;
expectedState = "close_completed";
} else {
relations.add("payment_to_settlement");
expectedState = "settlement_closed";
}
if (hints.require_missing_or_invalid_transition) {
anomalies.add("missing_link");
anomalies.add("no_continuation");
failedExpectedEdge = "expected_transition_not_observed";
}
if (hints.require_wrong_closing_document_type) {
anomalies.add("wrong_document_type");
anomalies.add("posting_mismatch");
}
if (hints.require_cross_branch_conflict) {
anomalies.add("cross_branch_inconsistency");
}
if (hints.require_period_close_impact) {
anomalies.add("period_close_risk");
periodCloseRisk = true;
}
if (hints.require_previous_states) {
actualState = domainFocus === "97" ? "partially_written_off" : "bank_recorded";
if (!expectedState) {
expectedState = domainFocus === "97" ? "fully_written_off" : "settlement_closed";
}
}
if (hints.require_terminal_state_mismatch) {
if (!actualState) {
if (domainFocus === "97") actualState = "partially_written_off";
else if (domainFocus === "fixed_asset") actualState = "depreciation_active";
else if (domainFocus === "vat_flow") actualState = "vat_registered";
else actualState = "bank_recorded";
}
if (domainFocus === "fixed_asset") expectedState = "disposed";
else if (domainFocus === "vat_flow") expectedState = "vat_deducted";
else if (domainFocus === "97") expectedState = "fully_written_off";
else expectedState = "settlement_closed";
}
const unit = buildProblemUnit({
id: `probe-${probeCase.case_id.toLowerCase()}`,
type: problemType,
mechanismSummary,
businessDefectClass,
affectedAccounts,
actualState,
expectedState,
failedExpectedEdge,
periodCloseRisk
});
const candidates = [
buildCandidate({
id: `cand-${probeCase.case_id.toLowerCase()}`,
anomalies: Array.from(anomalies),
relations: Array.from(relations)
})
];
return {
unit,
candidates
};
}
describe("stage3 lifecycle probe semantics", () => {
it("validates lifecycle acceptance targets on synthetic runtime inputs", () => {
const suite = loadSuite();
expect(suite.suite_id).toBe("assistant_stage3_lifecycle_probe");
expect(suite.scenario_count).toBe(suite.cases.length);
for (const probeCase of suite.cases) {
const hints = probeCase.expected_hints ?? {};
const { unit, candidates } = buildSyntheticInput(probeCase);
const enriched = enrichProblemUnitLifecycle({ unit, candidates });
if (typeof hints.expected_lifecycle_domain === "string" && hints.expected_lifecycle_domain.length > 0) {
expect(enriched.lifecycle_domain, `${probeCase.case_id}: expected lifecycle domain`).toBe(hints.expected_lifecycle_domain);
}
if (hints.require_current_expected_state_pair) {
expect(typeof enriched.current_lifecycle_state, `${probeCase.case_id}: current state`).toBe("string");
expect(typeof enriched.expected_lifecycle_state, `${probeCase.case_id}: expected state`).toBe("string");
}
if (hints.require_missing_or_invalid_transition) {
expect(
Boolean(enriched.missing_transition || enriched.invalid_transition),
`${probeCase.case_id}: missing/invalid transition`
).toBe(true);
}
if (hints.require_previous_states) {
const previousStates = Array.isArray(enriched.lifecycle_resolution?.resolved_previous_states)
? enriched.lifecycle_resolution?.resolved_previous_states
: [];
expect(previousStates.length, `${probeCase.case_id}: resolved_previous_states`).toBeGreaterThan(0);
}
if (hints.require_wrong_closing_document_type) {
const defect = String(enriched.lifecycle_defect_type ?? "");
expect(["misclosed_state", "invalid_transition"].includes(defect), `${probeCase.case_id}: wrong close defect`).toBe(true);
}
if (hints.require_cross_branch_conflict) {
expect(enriched.lifecycle_defect_type, `${probeCase.case_id}: cross-branch defect`).toBe("cross_branch_state_conflict");
}
if (hints.require_terminal_state_mismatch) {
const defect = String(enriched.lifecycle_defect_type ?? "");
const current = String(enriched.current_lifecycle_state ?? "");
const expected = String(enriched.expected_lifecycle_state ?? "");
const hasMismatchSignal =
defect === "premature_terminal_state" ||
defect === "misclosed_state" ||
defect === "orphan_intermediate_state" ||
(current.length > 0 && expected.length > 0 && current !== expected);
expect(hasMismatchSignal, `${probeCase.case_id}: terminal mismatch signal`).toBe(true);
}
if (hints.require_period_close_impact) {
const hasPeriodCloseImpact = Array.isArray(enriched.lifecycle_ranking_basis)
? enriched.lifecycle_ranking_basis.includes("period_close_impact")
: false;
expect(hasPeriodCloseImpact || enriched.lifecycle_domain === "period_close", `${probeCase.case_id}: period close impact`).toBe(true);
}
}
});
});