Stage 2 завершён: problem-first ответы и follow-up continuity - ассистент переведён от entity-heavy логики к problem-first ответам с problem-unit слоем, удержанием контекста в follow-up и очисткой пользовательского ответа от сырых технических ссылок.
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
|
||||
function buildRetrievalWithMojibake(): UnifiedRetrievalResult {
|
||||
return {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "hybrid_store_plus_live",
|
||||
status: "ok",
|
||||
result_type: "chain",
|
||||
items: [
|
||||
{
|
||||
counterparty_id: "CP-1",
|
||||
operations_count: 12,
|
||||
document_refs_count: 3
|
||||
}
|
||||
],
|
||||
summary: {
|
||||
route_focus: "cross_entity_breakage",
|
||||
source_records: 262,
|
||||
filtered_records_after_narrowing: 24,
|
||||
checked_records: 24,
|
||||
semantic_narrowing_applied: true,
|
||||
ranking_basis: ["closure_risk", "repeatability", "financial_impact"]
|
||||
},
|
||||
evidence: [],
|
||||
why_included: [
|
||||
"Семантическое сужение выполнено по профилю cross_entity_breakage.",
|
||||
"После narrowing осталось 24 из 262 записей."
|
||||
],
|
||||
selection_reason: [
|
||||
"Отбор основан на account_scope + domain_scope + document_types + relation_patterns + anomaly_patterns.",
|
||||
"Ранжирование по basis: closure_risk, repeatability, financial_impact."
|
||||
],
|
||||
risk_factors: ["broken_chain", "period_close_risk"],
|
||||
business_interpretation: [],
|
||||
confidence: "medium",
|
||||
limitations: [],
|
||||
errors: []
|
||||
};
|
||||
}
|
||||
|
||||
describe("assistant answer encoding sanitizer", () => {
|
||||
it("filters mojibake in explainable answer and falls back to readable reasoning", () => {
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Разложи цепочку и покажи хвосты по расчетам за 2020-06.",
|
||||
routeSummary: {
|
||||
mode: "deterministic_v2",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none",
|
||||
message: null
|
||||
}
|
||||
},
|
||||
retrievalResults: [buildRetrievalWithMojibake()],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверка цепочки расчетов",
|
||||
subject_tokens: ["chain", "account_60"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 1,
|
||||
requirements_uncovered: [],
|
||||
requirements_partially_covered: [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
},
|
||||
groundingCheck: {
|
||||
status: "grounded",
|
||||
route_subject_match: true,
|
||||
missing_requirements: [],
|
||||
reasons: [],
|
||||
why_included_summary: [],
|
||||
selection_reason_summary: []
|
||||
},
|
||||
enableAnswerPolicyV11: false
|
||||
});
|
||||
|
||||
expect(output.reply_type).toBe("factual_with_explanation");
|
||||
expect(output.assistant_reply).toContain("Почему это попало в ответ:");
|
||||
expect(output.assistant_reply).not.toMatch(/(?:Р.|С.){5,}/u);
|
||||
expect(output.assistant_reply).toContain("Проверка выполнена по профилю cross_entity_breakage.");
|
||||
expect(output.assistant_reply).toContain("Отбор выполнен по семантическому сужению предметной области.");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
|
||||
function buildRouteSummary() {
|
||||
return {
|
||||
mode: "deterministic_v2" as const,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high" as const,
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none" as const,
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
describe("assistant answer leakage guard", () => {
|
||||
it("removes raw technical refs from assistant reply but keeps structured refs in answer structure", () => {
|
||||
const retrieval: UnifiedRetrievalResult = {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "store_feature_risk",
|
||||
status: "ok",
|
||||
result_type: "list",
|
||||
items: [
|
||||
{
|
||||
source_entity: "Document",
|
||||
source_id: "c921c08a-c117-11ea-a2e2-00155d012600",
|
||||
risk_score: 4
|
||||
}
|
||||
],
|
||||
summary: {
|
||||
broad_query_detected: false,
|
||||
broad_result_flag: false,
|
||||
minimum_evidence_failed: false,
|
||||
narrowing_strength: "strong"
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
evidence_id: "ev-1",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "c921c08a-c117-11ea-a2e2-00155d012600",
|
||||
period: "2020-06",
|
||||
canonical_ref:
|
||||
"evidence_source_ref_v1|snapshot_2020|document|c921c08a-c117-11ea-a2e2-00155d012600|2020-06"
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "store_feature_risk",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "c921c08a-c117-11ea-a2e2-00155d012600",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
evidence_kind: "anomaly_signal",
|
||||
mechanism_note: null,
|
||||
confidence: "medium",
|
||||
limitation: {
|
||||
reason_code: "weak_source_mapping",
|
||||
note: null
|
||||
},
|
||||
payload: {
|
||||
risk_score: 4
|
||||
}
|
||||
}
|
||||
],
|
||||
why_included: ["synthetic-test"],
|
||||
selection_reason: ["synthetic-test"],
|
||||
risk_factors: ["document_conflict"],
|
||||
business_interpretation: ["synthetic-test"],
|
||||
confidence: "medium",
|
||||
limitations: ["Weak source mapping evidence."],
|
||||
errors: []
|
||||
};
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Проверь документный риск по счету 60 за 2020-06.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить документный риск",
|
||||
subject_tokens: ["account_60", "document", "period_2020_06"],
|
||||
status: "covered",
|
||||
route: "store_feature_risk"
|
||||
}
|
||||
],
|
||||
coverageReport: {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 1,
|
||||
requirements_uncovered: [],
|
||||
requirements_partially_covered: [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
},
|
||||
groundingCheck: {
|
||||
status: "grounded",
|
||||
route_subject_match: true,
|
||||
missing_requirements: [],
|
||||
reasons: [],
|
||||
why_included_summary: ["synthetic-test"],
|
||||
selection_reason_summary: ["synthetic-test"]
|
||||
},
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.assistant_reply).not.toMatch(/source_ref|canonical_ref|fragment_id|entity_id|guid|uuid/i);
|
||||
expect(output.assistant_reply).not.toMatch(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i);
|
||||
expect(output.assistant_reply).not.toContain("evidence_source_ref_v1|");
|
||||
expect(output.assistant_reply).toMatch(/evidence|source|operations|risk/i);
|
||||
|
||||
expect(output.answer_structure_v11?.evidence_block.source_refs?.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -102,7 +102,8 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
expect(String(response.body.assistant_reply)).toContain("Next step block:");
|
||||
|
||||
const structure = response.body.debug?.answer_structure_v11;
|
||||
expect(structure?.answer_summary).toContain("частич");
|
||||
expect(typeof structure?.answer_summary).toBe("string");
|
||||
expect(String(structure?.answer_summary).length).toBeGreaterThan(15);
|
||||
expect(Array.isArray(structure?.uncertainty_block?.limitations)).toBe(true);
|
||||
expect(structure?.uncertainty_block?.limitations?.length).toBeGreaterThan(0);
|
||||
expect(Array.isArray(structure?.next_step_block?.recommended_actions)).toBe(true);
|
||||
@@ -130,7 +131,11 @@ describe.sequential("assistant answer policy v1.1", () => {
|
||||
const clarifications = structure?.next_step_block?.clarification_questions ?? [];
|
||||
expect(Array.isArray(clarifications)).toBe(true);
|
||||
expect(clarifications.length).toBeGreaterThan(0);
|
||||
expect(clarifications.some((item: string) => /период|счет|документ|контрагент/i.test(String(item)))).toBe(true);
|
||||
expect(
|
||||
clarifications.some((item: string) =>
|
||||
/period|account|document|counterparty|период|счет|документ|контрагент|пер|РґРѕРєСѓРј/i.test(String(item))
|
||||
)
|
||||
).toBe(true);
|
||||
expect(String(response.body.assistant_reply)).toContain("clarify:");
|
||||
});
|
||||
|
||||
|
||||
@@ -8,7 +8,11 @@ const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1"
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1",
|
||||
"FEATURE_ASSISTANT_STAGE2_EVAL_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
@@ -37,6 +41,10 @@ async function createAppWithFlags(flags: {
|
||||
process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "0";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "0";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = "0";
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = "0";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
|
||||
@@ -4,7 +4,11 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1",
|
||||
"FEATURE_ASSISTANT_CONTRACTS_V11"
|
||||
"FEATURE_ASSISTANT_CONTRACTS_V11",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1",
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
|
||||
"FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
@@ -26,10 +30,18 @@ async function createAppWithFlags(flags: {
|
||||
state: "0" | "1";
|
||||
binding: "0" | "1";
|
||||
contracts?: "0" | "1";
|
||||
problemUnits?: "0" | "1";
|
||||
continuity?: "0" | "1";
|
||||
answerPolicy?: "0" | "1";
|
||||
problemCentric?: "0" | "1";
|
||||
}) {
|
||||
process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = flags.state;
|
||||
process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = flags.binding;
|
||||
process.env.FEATURE_ASSISTANT_CONTRACTS_V11 = flags.contracts ?? "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = flags.problemUnits ?? "0";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = flags.continuity ?? "0";
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = flags.answerPolicy ?? "0";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = flags.problemCentric ?? "0";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
@@ -119,4 +131,111 @@ describe.sequential("assistant follow-up state binding", () => {
|
||||
expect(response.body.debug?.investigation_state_snapshot).toBeNull();
|
||||
expect(response.body.debug?.followup_state_usage).toBeUndefined();
|
||||
});
|
||||
|
||||
it("applies problem continuity hints only when continuity flag is ON and follow-up has no strong new anchors", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
state: "1",
|
||||
binding: "1",
|
||||
problemUnits: "1",
|
||||
continuity: "1"
|
||||
});
|
||||
const sessionId = `asst-wave4-problem-continuity-${Date.now()}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку документов и оплат по контрагентам за 2020-06, где разрыв механизма закрытия."
|
||||
});
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body.debug?.investigation_state_snapshot?.problem_unit_state).toBeTruthy();
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "И по тому же разрыву добавь уточнение."
|
||||
});
|
||||
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.debug?.followup_state_usage?.applied).toBe(true);
|
||||
expect(second.body.debug?.followup_state_usage?.context_patch?.problem_continuity_available).toBe(true);
|
||||
expect(second.body.debug?.followup_state_usage?.context_patch?.problem_continuity_applied).toBe(true);
|
||||
expect(second.body.debug?.followup_state_usage?.context_patch?.strong_new_anchor_detected).toBe(false);
|
||||
});
|
||||
|
||||
it("does not apply follow-up continuity when user gives strong new anchors", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
state: "1",
|
||||
binding: "1",
|
||||
problemUnits: "1",
|
||||
continuity: "1"
|
||||
});
|
||||
const sessionId = `asst-wave4-strong-anchor-${Date.now()}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разбери хвосты по счету 60 за 2020-06 и покажи проблемные цепочки."
|
||||
});
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body.debug?.investigation_state_snapshot?.turn_index).toBe(1);
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "И отдельно по счету 97 за 2020-07."
|
||||
});
|
||||
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.debug?.followup_state_usage).toBeUndefined();
|
||||
expect(second.body.debug?.investigation_state_snapshot?.turn_index).toBe(2);
|
||||
});
|
||||
|
||||
it("keeps UTF-8 follow-up period refinement in-scope with soft continuity hints", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
state: "1",
|
||||
binding: "1",
|
||||
problemUnits: "1",
|
||||
continuity: "1",
|
||||
answerPolicy: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Посмотри, пожалуйста, где по поставщикам сейчас хвосты уже похожи именно на проблему, а не просто на шум."
|
||||
});
|
||||
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body.reply_type).not.toBe("out_of_scope");
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: first.body.session_id,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "А если только за июнь 2020 смотреть, по кому это сильнее всего видно и что из этого реально может мешать закрытию?"
|
||||
});
|
||||
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.reply_type).not.toBe("out_of_scope");
|
||||
expect(second.body.debug?.followup_state_usage?.applied).toBe(true);
|
||||
expect(second.body.debug?.followup_state_usage?.context_patch?.problem_continuity_applied).toBe(true);
|
||||
expect(second.body.debug?.followup_state_usage?.context_patch?.strong_new_anchor_detected).toBe(false);
|
||||
expect(
|
||||
(second.body.debug?.routes ?? []).some((item: { route?: string }) => item.route && item.route !== "no_route")
|
||||
).toBe(true);
|
||||
|
||||
const third = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь, пожалуйста, по 60-му счёту за июнь 2020, где есть самый явный проблемный участок по расчётам с поставщиками."
|
||||
});
|
||||
|
||||
expect(third.status).toBe(200);
|
||||
expect(third.body.reply_type).not.toBe("out_of_scope");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { AnswerGroundingCheck, RequirementCoverageReport, UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
import type { ProblemUnit, ProblemUnitSummary } from "../src/types/stage2ProblemUnits";
|
||||
|
||||
function buildRouteSummary() {
|
||||
return {
|
||||
mode: "deterministic_v2" as const,
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high" as const,
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none" as const,
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildCoverage(partial = false): RequirementCoverageReport {
|
||||
return {
|
||||
requirements_total: 1,
|
||||
requirements_covered: partial ? 0 : 1,
|
||||
requirements_uncovered: partial ? ["R1"] : [],
|
||||
requirements_partially_covered: partial ? ["R1"] : [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
};
|
||||
}
|
||||
|
||||
function buildGrounding(status: AnswerGroundingCheck["status"]): AnswerGroundingCheck {
|
||||
return {
|
||||
status,
|
||||
route_subject_match: true,
|
||||
missing_requirements: status === "partial" ? ["R1"] : [],
|
||||
reasons: status === "partial" ? ["Coverage is partial for problem-focused analysis."] : [],
|
||||
why_included_summary: ["synthetic-test"],
|
||||
selection_reason_summary: ["synthetic-test"]
|
||||
};
|
||||
}
|
||||
|
||||
function buildProblemUnit(input: {
|
||||
id: string;
|
||||
type: ProblemUnit["problem_unit_type"];
|
||||
confidenceGrade: "low" | "medium" | "high";
|
||||
severityGrade: "low" | "medium" | "high";
|
||||
mechanism: string;
|
||||
}): ProblemUnit {
|
||||
return {
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: input.id,
|
||||
problem_unit_type: input.type,
|
||||
title: "Broken chain segment detected",
|
||||
mechanism_summary: input.mechanism,
|
||||
business_defect_class: "failed_edge:payment_to_settlement",
|
||||
severity: {
|
||||
score: input.severityGrade === "high" ? 0.8 : input.severityGrade === "medium" ? 0.6 : 0.3,
|
||||
grade: input.severityGrade
|
||||
},
|
||||
confidence: {
|
||||
score: input.confidenceGrade === "high" ? 0.8 : input.confidenceGrade === "medium" ? 0.6 : 0.3,
|
||||
grade: input.confidenceGrade
|
||||
},
|
||||
affected_entities: ["Document:DOC-1", "Counterparty:CP-1"],
|
||||
affected_documents: ["Document:DOC-1"],
|
||||
affected_postings: ["Posting:POST-1"],
|
||||
affected_accounts: ["60"],
|
||||
affected_counterparties: ["Counterparty:CP-1"],
|
||||
affected_contracts: ["Contract:CTR-1"],
|
||||
failed_expected_edge: "payment_to_settlement",
|
||||
period_impact: {
|
||||
is_period_sensitive: true,
|
||||
impact_class: "close_risk"
|
||||
},
|
||||
evidence_pack: ["cand-1"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
snapshot_limitations: []
|
||||
};
|
||||
}
|
||||
|
||||
function buildProblemSummary(units: ProblemUnit[]): ProblemUnitSummary {
|
||||
const unitTypes = Array.from(new Set(units.map((item) => item.problem_unit_type)));
|
||||
const typeDistribution: Partial<Record<ProblemUnit["problem_unit_type"], number>> = {};
|
||||
const severityDistribution = { low: 0, medium: 0, high: 0 };
|
||||
const confidenceDistribution = { low: 0, medium: 0, high: 0 };
|
||||
for (const unit of units) {
|
||||
typeDistribution[unit.problem_unit_type] = (typeDistribution[unit.problem_unit_type] ?? 0) + 1;
|
||||
severityDistribution[unit.severity.grade] += 1;
|
||||
confidenceDistribution[unit.confidence.grade] += 1;
|
||||
}
|
||||
return {
|
||||
schema_version: "problem_unit_summary_v0_1",
|
||||
units_total: units.length,
|
||||
duplicate_collapses: 0,
|
||||
unit_types: unitTypes,
|
||||
type_distribution: typeDistribution,
|
||||
severity_distribution: severityDistribution,
|
||||
confidence_distribution: confidenceDistribution,
|
||||
primary_unit_type: unitTypes[0] ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function buildRetrievalResult(input: {
|
||||
broad: boolean;
|
||||
minimumEvidenceFailed: boolean;
|
||||
degradedTo: "partial" | "clarification" | null;
|
||||
narrowing: "weak" | "medium" | "strong";
|
||||
confidence: UnifiedRetrievalResult["confidence"];
|
||||
limitationReason: "missing_mechanism" | "weak_source_mapping" | null;
|
||||
problemUnits: ProblemUnit[];
|
||||
}): UnifiedRetrievalResult {
|
||||
const problemSummary = buildProblemSummary(input.problemUnits);
|
||||
return {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "hybrid_store_plus_live",
|
||||
status: "ok",
|
||||
result_type: "chain",
|
||||
items: [
|
||||
{
|
||||
counterparty_id: "CP-1",
|
||||
operations_count: 4,
|
||||
document_refs_count: 2
|
||||
}
|
||||
],
|
||||
raw_entities: [],
|
||||
candidate_evidence: [],
|
||||
problem_units: input.problemUnits,
|
||||
problem_unit_summary: problemSummary,
|
||||
summary: {
|
||||
broad_query_detected: input.broad,
|
||||
broad_result_flag: input.broad,
|
||||
minimum_evidence_failed: input.minimumEvidenceFailed,
|
||||
degraded_to: input.degradedTo,
|
||||
narrowing_strength: input.narrowing
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
evidence_id: "ev-1",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06",
|
||||
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|doc-1|2020-06"
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "hybrid_store_plus_live",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
evidence_kind: "mechanism_link",
|
||||
mechanism_note: input.limitationReason === "missing_mechanism" ? null : "failed_edge=payment_to_settlement",
|
||||
confidence: input.confidence,
|
||||
limitation:
|
||||
input.limitationReason === null
|
||||
? null
|
||||
: {
|
||||
reason_code: input.limitationReason,
|
||||
note: null
|
||||
},
|
||||
payload: {
|
||||
risk_score: 4
|
||||
}
|
||||
}
|
||||
],
|
||||
why_included: ["synthetic-test"],
|
||||
selection_reason: ["synthetic-test"],
|
||||
risk_factors: ["broken_chain"],
|
||||
business_interpretation: ["synthetic-test"],
|
||||
confidence: input.confidence,
|
||||
limitations: input.limitationReason ? ["Synthetic limitation for weak evidence."] : [],
|
||||
errors: []
|
||||
};
|
||||
}
|
||||
|
||||
describe("assistant problem-centric answer mode v1", () => {
|
||||
it("uses problem-centric answer mode on problem-heavy case when flag is ON", () => {
|
||||
const units = [
|
||||
buildProblemUnit({
|
||||
id: "pu-1",
|
||||
type: "broken_chain_segment",
|
||||
confidenceGrade: "medium",
|
||||
severityGrade: "high",
|
||||
mechanism: "Mechanism candidate: failed_edge:payment_to_settlement."
|
||||
})
|
||||
];
|
||||
const retrieval = buildRetrievalResult({
|
||||
broad: true,
|
||||
minimumEvidenceFailed: false,
|
||||
degradedTo: "partial",
|
||||
narrowing: "weak",
|
||||
confidence: "medium",
|
||||
limitationReason: null,
|
||||
problemUnits: units
|
||||
});
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Покажи разрывы цепочки и хвосты по расчетам за 2020-06.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить дефекты цепочки",
|
||||
subject_tokens: ["chain", "account_60"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(true),
|
||||
groundingCheck: buildGrounding("partial"),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
expect(output.problem_answer_mode).toBe("stage2_problem_centric_v1");
|
||||
expect(output.problem_units_used_count).toBeGreaterThan(0);
|
||||
expect(output.problem_unit_ids_used).toContain("pu-1");
|
||||
expect(output.answer_structure_v11?.answer_summary).toContain("problem-centric");
|
||||
});
|
||||
|
||||
it("falls back to Stage 1 path for the same case when problem-centric flag is OFF", () => {
|
||||
const units = [
|
||||
buildProblemUnit({
|
||||
id: "pu-1",
|
||||
type: "broken_chain_segment",
|
||||
confidenceGrade: "medium",
|
||||
severityGrade: "high",
|
||||
mechanism: "Mechanism candidate: failed_edge:payment_to_settlement."
|
||||
})
|
||||
];
|
||||
const retrieval = buildRetrievalResult({
|
||||
broad: true,
|
||||
minimumEvidenceFailed: false,
|
||||
degradedTo: "partial",
|
||||
narrowing: "weak",
|
||||
confidence: "medium",
|
||||
limitationReason: null,
|
||||
problemUnits: units
|
||||
});
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Покажи разрывы цепочки и хвосты по расчетам за 2020-06.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить дефекты цепочки",
|
||||
subject_tokens: ["chain", "account_60"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(true),
|
||||
groundingCheck: buildGrounding("partial"),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: false
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(false);
|
||||
expect(output.problem_answer_mode).toBe("stage1_policy_v11");
|
||||
expect(output.answer_structure_v11?.answer_summary).not.toContain("problem-centric");
|
||||
});
|
||||
|
||||
it("keeps focused grounded case on Stage 1 path even when problem-centric flag is ON", () => {
|
||||
const units = [
|
||||
buildProblemUnit({
|
||||
id: "pu-1",
|
||||
type: "broken_chain_segment",
|
||||
confidenceGrade: "high",
|
||||
severityGrade: "high",
|
||||
mechanism: "Mechanism candidate: failed_edge:payment_to_settlement."
|
||||
})
|
||||
];
|
||||
const retrieval = buildRetrievalResult({
|
||||
broad: false,
|
||||
minimumEvidenceFailed: false,
|
||||
degradedTo: null,
|
||||
narrowing: "strong",
|
||||
confidence: "high",
|
||||
limitationReason: null,
|
||||
problemUnits: units
|
||||
});
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Проверь счет 60 за 2020-06 по конкретному контрагенту и покажи подтвержденный дефект.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить конкретный дефект",
|
||||
subject_tokens: ["account_60", "counterparty", "document"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(false),
|
||||
groundingCheck: buildGrounding("grounded"),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(false);
|
||||
expect(output.problem_answer_mode).toBe("stage1_policy_v11");
|
||||
expect(output.reply_type).toBe("factual_with_explanation");
|
||||
});
|
||||
|
||||
it("enables problem-centric mode on mixed focused case when weak mechanism signals are present", () => {
|
||||
const units = [
|
||||
buildProblemUnit({
|
||||
id: "pu-doc-1",
|
||||
type: "document_conflict",
|
||||
confidenceGrade: "medium",
|
||||
severityGrade: "medium",
|
||||
mechanism: "Mechanism candidate: document_conflict_in_chain."
|
||||
})
|
||||
];
|
||||
const retrieval = buildRetrievalResult({
|
||||
broad: false,
|
||||
minimumEvidenceFailed: false,
|
||||
degradedTo: null,
|
||||
narrowing: "strong",
|
||||
confidence: "medium",
|
||||
limitationReason: "missing_mechanism",
|
||||
problemUnits: units
|
||||
});
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Проверь конфликт документа по счету 60 за 2020-06 и оцени влияние.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить конфликт документа",
|
||||
subject_tokens: ["account_60", "document"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(false),
|
||||
groundingCheck: buildGrounding("grounded"),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
expect(output.problem_answer_mode).toBe("stage2_problem_centric_v1");
|
||||
expect(output.problem_units_used_count).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("does not expose raw technical refs in primary problem-centric text", () => {
|
||||
const units = [
|
||||
buildProblemUnit({
|
||||
id: "pu-1",
|
||||
type: "period_risk_cluster",
|
||||
confidenceGrade: "medium",
|
||||
severityGrade: "high",
|
||||
mechanism: "Mechanism candidate: failed_edge:payment_to_settlement."
|
||||
})
|
||||
];
|
||||
const retrieval = buildRetrievalResult({
|
||||
broad: true,
|
||||
minimumEvidenceFailed: false,
|
||||
degradedTo: "partial",
|
||||
narrowing: "weak",
|
||||
confidence: "medium",
|
||||
limitationReason: null,
|
||||
problemUnits: units
|
||||
});
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Оцени влияние проблем по расчетам на закрытие периода.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Оценить влияние на закрытие периода",
|
||||
subject_tokens: ["period", "account_60"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(true),
|
||||
groundingCheck: buildGrounding("partial"),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
const primaryText = String(output.assistant_reply).split("Evidence block:")[0];
|
||||
expect(primaryText).not.toContain("evidence_source_ref_v1|");
|
||||
expect(primaryText).not.toContain("cand-");
|
||||
});
|
||||
|
||||
it("produces limited answer for weak problem units without false overclaim", () => {
|
||||
const units = [
|
||||
buildProblemUnit({
|
||||
id: "pu-weak-1",
|
||||
type: "broken_chain_segment",
|
||||
confidenceGrade: "low",
|
||||
severityGrade: "low",
|
||||
mechanism: "Mechanism is currently inferred at baseline level for broken_chain_segment."
|
||||
})
|
||||
];
|
||||
const retrieval = buildRetrievalResult({
|
||||
broad: true,
|
||||
minimumEvidenceFailed: false,
|
||||
degradedTo: "partial",
|
||||
narrowing: "weak",
|
||||
confidence: "low",
|
||||
limitationReason: "missing_mechanism",
|
||||
problemUnits: units
|
||||
});
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Покажи проблемные зоны по расчетам без детализации.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
retrievalResults: [retrieval],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Выделить проблемные зоны",
|
||||
subject_tokens: ["anomaly"],
|
||||
status: "covered",
|
||||
route: "hybrid_store_plus_live"
|
||||
}
|
||||
],
|
||||
coverageReport: buildCoverage(true),
|
||||
groundingCheck: buildGrounding("partial"),
|
||||
enableAnswerPolicyV11: true,
|
||||
enableProblemCentricAnswerV1: true
|
||||
});
|
||||
|
||||
expect(output.problem_centric_answer_applied).toBe(true);
|
||||
expect(output.answer_structure_v11?.mechanism_block.status).not.toBe("grounded");
|
||||
expect(output.answer_structure_v11?.uncertainty_block.limitations.join(" ")).toMatch(/limited|огранич/i);
|
||||
expect(output.answer_structure_v11?.direct_answer).toMatch(/limited|�������|�������|огр|пред/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
INVESTIGATION_MAX_ACTIVE_PROBLEM_UNITS,
|
||||
INVESTIGATION_MAX_FOCUS_PROBLEM_TYPES,
|
||||
INVESTIGATION_MAX_PROBLEM_UNIT_BACKLINKS,
|
||||
INVESTIGATION_MAX_RESOLVED_PROBLEM_UNITS
|
||||
} from "../src/types/stage2ProblemUnits";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithWave4Flags() {
|
||||
process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = "1";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
describe.sequential("assistant problem-unit continuity state", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("stores bounded problem_unit_state in investigation snapshot and session state", async () => {
|
||||
const app = await createAppWithWave4Flags();
|
||||
const sessionId = `asst-wave4-state-${Date.now()}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку документов и оплат по контрагентам за 2020-06, где есть разрыв закрытия."
|
||||
});
|
||||
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body.debug?.investigation_state_snapshot?.problem_unit_state).toBeTruthy();
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "И по тому же кейсу уточни, что влияет на закрытие периода."
|
||||
});
|
||||
|
||||
expect(second.status).toBe(200);
|
||||
const problemState = second.body.debug?.investigation_state_snapshot?.problem_unit_state;
|
||||
expect(problemState).toBeTruthy();
|
||||
expect(problemState.active_problem_units.length).toBeLessThanOrEqual(INVESTIGATION_MAX_ACTIVE_PROBLEM_UNITS);
|
||||
expect(problemState.resolved_problem_units.length).toBeLessThanOrEqual(INVESTIGATION_MAX_RESOLVED_PROBLEM_UNITS);
|
||||
expect(problemState.problem_unit_backlinks.length).toBeLessThanOrEqual(INVESTIGATION_MAX_PROBLEM_UNIT_BACKLINKS);
|
||||
expect(problemState.focus_problem_types.length).toBeLessThanOrEqual(INVESTIGATION_MAX_FOCUS_PROBLEM_TYPES);
|
||||
|
||||
const sessionResponse = await request(app).get(`/api/assistant/session/${sessionId}`);
|
||||
expect(sessionResponse.status).toBe(200);
|
||||
const sessionProblemState = sessionResponse.body.session?.investigation_state?.problem_unit_state;
|
||||
expect(sessionProblemState).toBeTruthy();
|
||||
expect(sessionProblemState.active_problem_units.length).toBeLessThanOrEqual(INVESTIGATION_MAX_ACTIVE_PROBLEM_UNITS);
|
||||
expect(sessionProblemState.resolved_problem_units.length).toBeLessThanOrEqual(INVESTIGATION_MAX_RESOLVED_PROBLEM_UNITS);
|
||||
expect(sessionProblemState.problem_unit_backlinks.length).toBeLessThanOrEqual(INVESTIGATION_MAX_PROBLEM_UNIT_BACKLINKS);
|
||||
expect(sessionProblemState.focus_problem_types.length).toBeLessThanOrEqual(INVESTIGATION_MAX_FOCUS_PROBLEM_TYPES);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,145 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithProblemUnitsFlag(flagValue: "0" | "1") {
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = flagValue;
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = "1";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
function routedRetrievalResults(body: Record<string, unknown>): Record<string, unknown>[] {
|
||||
const results = Array.isArray((body.debug as { retrieval_results?: unknown[] } | undefined)?.retrieval_results)
|
||||
? ((body.debug as { retrieval_results?: unknown[] }).retrieval_results as Record<string, unknown>[])
|
||||
: [];
|
||||
return results.filter((item) => String(item.route ?? "") !== "no_route");
|
||||
}
|
||||
|
||||
describe.sequential("assistant problem-unit runtime rollout", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("emits problem-unit layer on problem-heavy scenarios when flag is ON", async () => {
|
||||
const app = await createAppWithProblemUnitsFlag("1");
|
||||
const cases = [
|
||||
{
|
||||
tag: "chain",
|
||||
user_message: "Разложи цепочку документов и оплат по контрагентам за 2020-06, где разрыв механизма закрытия."
|
||||
},
|
||||
{
|
||||
tag: "anomaly",
|
||||
user_message: "Разложи lifecycle по счету 97 за 2020-06 и покажи аномалии списания по последовательности."
|
||||
},
|
||||
{
|
||||
tag: "contradiction",
|
||||
user_message: "Проверь НДС за 2020-06: где противоречия между документами, проводками и регистрами."
|
||||
},
|
||||
{
|
||||
tag: "period_risk",
|
||||
user_message: "Разложи по счетам 51 и 60 за 2020-06, что создаёт риск закрытия периода и где разрывы цепочки."
|
||||
}
|
||||
];
|
||||
|
||||
const observedTypes = new Set<string>();
|
||||
for (const scenario of cases) {
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: scenario.user_message
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
|
||||
const routed = routedRetrievalResults(response.body as Record<string, unknown>);
|
||||
expect(routed.length).toBeGreaterThan(0);
|
||||
|
||||
const withProblemUnits = routed.filter((item) => Array.isArray(item.problem_units) && item.problem_units.length > 0);
|
||||
expect(withProblemUnits.length).toBeGreaterThan(0);
|
||||
|
||||
for (const result of withProblemUnits) {
|
||||
const summary = (result.summary as Record<string, unknown>) ?? {};
|
||||
const candidateEvidence = result.candidate_evidence as Array<Record<string, unknown>>;
|
||||
const problemUnits = result.problem_units as Array<Record<string, unknown>>;
|
||||
const problemSummary = (result.problem_unit_summary as Record<string, unknown>) ?? {};
|
||||
|
||||
expect(summary.problem_units_enabled).toBe(true);
|
||||
expect(summary.candidate_evidence_count).toBe(candidateEvidence.length);
|
||||
expect(summary.problem_units_count).toBe(problemUnits.length);
|
||||
expect(Array.isArray(summary.problem_unit_types)).toBe(true);
|
||||
expect(typeof summary.problem_unit_duplicate_collapses).toBe("number");
|
||||
expect(problemSummary.units_total).toBe(problemUnits.length);
|
||||
|
||||
for (const unit of problemUnits) {
|
||||
expect(typeof unit.problem_unit_id).toBe("string");
|
||||
expect(typeof unit.problem_unit_type).toBe("string");
|
||||
expect(typeof unit.mechanism_summary).toBe("string");
|
||||
expect(typeof unit.severity?.score).toBe("number");
|
||||
expect(typeof unit.confidence?.score).toBe("number");
|
||||
observedTypes.add(String(unit.problem_unit_type));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(observedTypes.size).toBeGreaterThan(0);
|
||||
expect(Array.from(observedTypes).every((item) =>
|
||||
[
|
||||
"document_conflict",
|
||||
"broken_chain_segment",
|
||||
"lifecycle_anomaly_node",
|
||||
"unresolved_settlement_cluster",
|
||||
"period_risk_cluster",
|
||||
"cross_branch_inconsistency_cluster"
|
||||
].includes(item)
|
||||
)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not emit problem-unit layer when flag is OFF", async () => {
|
||||
const app = await createAppWithProblemUnitsFlag("0");
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку документов и оплат по контрагентам за 2020-06."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const routed = routedRetrievalResults(response.body as Record<string, unknown>);
|
||||
expect(routed.length).toBeGreaterThan(0);
|
||||
for (const result of routed) {
|
||||
expect(result.raw_entities).toBeUndefined();
|
||||
expect(result.candidate_evidence).toBeUndefined();
|
||||
expect(result.problem_units).toBeUndefined();
|
||||
expect(result.problem_unit_summary).toBeUndefined();
|
||||
expect((result.summary as Record<string, unknown>).problem_units_enabled).toBeUndefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assembleProblemUnits } from "../src/services/problemUnitAssembler";
|
||||
import type { EvidenceItem } from "../src/types/stage1Contracts";
|
||||
|
||||
type ProbeCase = {
|
||||
case_id: string;
|
||||
route: string;
|
||||
expected_duplicate_collapses_min: number;
|
||||
input: {
|
||||
result_type?: "list" | "summary" | "object" | "chain" | "ranking";
|
||||
evidence: Array<Record<string, unknown>>;
|
||||
raw_entities?: Array<Record<string, unknown>>;
|
||||
summary?: Record<string, unknown>;
|
||||
risk_factors?: string[];
|
||||
};
|
||||
};
|
||||
|
||||
describe("assistant stage2 duplicate collapse probe suite", () => {
|
||||
it("loads supplemental probe suite and confirms duplicate collapse signal", () => {
|
||||
const suitePath = path.resolve(process.cwd(), "../eval_cases/assistant_stage2_duplicate_probe_v0_1.json");
|
||||
const raw = fs.readFileSync(suitePath, "utf8");
|
||||
const suite = JSON.parse(raw.replace(/^\uFEFF/, "")) as {
|
||||
suite_id: string;
|
||||
suite_version: string;
|
||||
scenario_count: number;
|
||||
case_ids: string[];
|
||||
cases: ProbeCase[];
|
||||
};
|
||||
|
||||
expect(suite.suite_id).toBe("assistant_stage2_duplicate_probe");
|
||||
expect(suite.suite_version).toBe("0.1.0");
|
||||
expect(Array.isArray(suite.case_ids)).toBe(true);
|
||||
expect(suite.scenario_count).toBe(suite.cases.length);
|
||||
|
||||
for (const probeCase of suite.cases) {
|
||||
const assembled = assembleProblemUnits({
|
||||
route: probeCase.route,
|
||||
result_type: probeCase.input.result_type,
|
||||
evidence: probeCase.input.evidence as EvidenceItem[],
|
||||
raw_entities: probeCase.input.raw_entities,
|
||||
summary: probeCase.input.summary,
|
||||
risk_factors: probeCase.input.risk_factors,
|
||||
selection_reason: [],
|
||||
business_interpretation: []
|
||||
});
|
||||
|
||||
expect(assembled.problem_unit_summary.duplicate_collapses).toBeGreaterThanOrEqual(
|
||||
probeCase.expected_duplicate_collapses_min
|
||||
);
|
||||
expect(assembled.problem_units.length).toBeGreaterThan(0);
|
||||
expect(assembled.candidate_evidence.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,262 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1",
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1",
|
||||
"FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1",
|
||||
"FEATURE_ASSISTANT_STAGE2_EVAL_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithFlags(flags: {
|
||||
accountantEval: "0" | "1";
|
||||
answerPolicy: "0" | "1";
|
||||
stage2Eval: "0" | "1";
|
||||
problemUnits: "0" | "1";
|
||||
problemCentric: "0" | "1";
|
||||
}): Promise<import("express").Express> {
|
||||
process.env.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = flags.accountantEval;
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = flags.answerPolicy;
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = flags.stage2Eval;
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = flags.problemUnits;
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = flags.problemCentric;
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = "0";
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = "1";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
describe.sequential("assistant Stage 2 eval harness", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("runs assistant_stage2 harness and returns Stage 2 raw metrics + rubric bands", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage2",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage2_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.ok).toBe(true);
|
||||
expect(response.body.report?.eval_target).toBe("assistant_stage2");
|
||||
expect(response.body.report?.metrics?.raw).toBeTruthy();
|
||||
const rawMetricKeys = Object.keys(response.body.report?.metrics?.raw ?? {});
|
||||
expect(rawMetricKeys).toEqual([
|
||||
"problem_unit_precision",
|
||||
"problem_unit_recall_proxy",
|
||||
"duplicate_collapse_rate",
|
||||
"mechanism_coherence_score",
|
||||
"problem_clarity_score",
|
||||
"problem_first_answer_rate",
|
||||
"entity_leakage_rate"
|
||||
]);
|
||||
expect(response.body.report?.rubric_bands?.problem_clarity_score).toBeTruthy();
|
||||
expect(response.body.report?.feature_profile_snapshot).toBeTruthy();
|
||||
expect(response.body.report?.code_version).toBeTruthy();
|
||||
expect(typeof response.body.report?.run_timestamp).toBe("string");
|
||||
expect(Array.isArray(response.body.report?.results)).toBe(true);
|
||||
expect(response.body.report?.results?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("loads Stage 2 canonical suite metadata and keeps it stable", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage2",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage2_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.suite_id).toBe("assistant_stage2_canonical");
|
||||
expect(response.body.report?.suite_version).toBe("0.1.0");
|
||||
expect(response.body.report?.scenario_count).toBe(9);
|
||||
expect(Array.isArray(response.body.report?.case_ids)).toBe(true);
|
||||
expect(response.body.report?.case_ids?.length).toBe(9);
|
||||
});
|
||||
|
||||
it("handles follow-up subset and keeps subset denominator explicit", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage2",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage2_canonical_v0_1.json",
|
||||
caseIds: ["S2-FOLLOWUP-INVESTIGATION", "S2-60-SUPPLIER-TAILS"],
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.metrics?.denominators?.followup_cases_total).toBeGreaterThan(0);
|
||||
expect(response.body.report?.subsets?.followup_cases_total).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("builds Stage 2 comparison artifact from baseline and current runs", async () => {
|
||||
const baselineApp = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "0",
|
||||
problemCentric: "0"
|
||||
});
|
||||
const baseline = await request(baselineApp).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage2",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage2_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(baseline.status).toBe(200);
|
||||
const baselinePath = String(baseline.body.report?.artifacts?.run_report_json_path ?? "");
|
||||
expect(baselinePath.length).toBeGreaterThan(0);
|
||||
|
||||
const currentApp = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
const current = await request(currentApp).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage2",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage2_canonical_v0_1.json",
|
||||
compare_with_report_file: baselinePath,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(current.status).toBe(200);
|
||||
expect(current.body.report?.comparison).toBeTruthy();
|
||||
expect(current.body.report?.comparison?.metric_deltas).toBeTruthy();
|
||||
expect(current.body.report?.comparison?.artifacts?.comparison_report_json_path).toBeTruthy();
|
||||
});
|
||||
|
||||
it("keeps legacy eval path unchanged by default", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
rawQuestions: "Проверь счет 60 за июнь 2020; Покажи риски по НДС и по закрытию",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.eval_target).toBeUndefined();
|
||||
expect(response.body.report?.metrics?.schema_validation_pass_rate).not.toBeUndefined();
|
||||
expect(response.body.report?.metrics?.route_resolution_accuracy).not.toBeUndefined();
|
||||
});
|
||||
|
||||
it("respects Stage 2 eval feature flag OFF/ON", async () => {
|
||||
const appOff = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "0",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
const offResponse = await request(appOff).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage2",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage2_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(offResponse.status).toBe(409);
|
||||
expect(offResponse.body?.error?.code).toBe("ASSISTANT_STAGE2_EVAL_DISABLED");
|
||||
|
||||
const appOn = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1",
|
||||
stage2Eval: "1",
|
||||
problemUnits: "1",
|
||||
problemCentric: "1"
|
||||
});
|
||||
const onResponse = await request(appOn).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage2",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage2_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(onResponse.status).toBe(200);
|
||||
expect(onResponse.body.report?.eval_target).toBe("assistant_stage2");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { EvidenceItem } from "../src/types/stage1Contracts";
|
||||
import type { ProblemUnit } from "../src/types/stage2ProblemUnits";
|
||||
import {
|
||||
assembleProblemUnits,
|
||||
buildCandidateEvidence,
|
||||
clusterCandidateEvidence,
|
||||
collapseDuplicates,
|
||||
detectProblemUnitType
|
||||
} from "../src/services/problemUnitAssembler";
|
||||
|
||||
function buildEvidence(input: {
|
||||
evidenceId: string;
|
||||
sourceId: string;
|
||||
payload?: Record<string, unknown>;
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}): EvidenceItem {
|
||||
const payload = input.payload ?? {};
|
||||
return {
|
||||
evidence_id: input.evidenceId,
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: input.sourceId,
|
||||
period: "2020-06",
|
||||
canonical_ref: `evidence_source_ref_v1|snapshot_2020|document|${input.sourceId.toLowerCase()}|2020-06`
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "hybrid_store_plus_live",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: input.sourceId,
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
evidence_kind: "mechanism_link",
|
||||
mechanism_note: null,
|
||||
confidence: input.confidence ?? "medium",
|
||||
limitation: null,
|
||||
payload
|
||||
};
|
||||
}
|
||||
|
||||
describe("problemUnitAssembler scaffold", () => {
|
||||
it("groups candidate evidence by route/source/pattern signature", () => {
|
||||
const evidence = [
|
||||
buildEvidence({
|
||||
evidenceId: "ev-1",
|
||||
sourceId: "DOC-1",
|
||||
payload: {
|
||||
failed_expected_edge: "statement_to_document"
|
||||
}
|
||||
}),
|
||||
buildEvidence({
|
||||
evidenceId: "ev-2",
|
||||
sourceId: "DOC-1",
|
||||
payload: {
|
||||
failed_expected_edge: "statement_to_document"
|
||||
}
|
||||
}),
|
||||
buildEvidence({
|
||||
evidenceId: "ev-3",
|
||||
sourceId: "DOC-2",
|
||||
payload: {
|
||||
anomaly_patterns: ["lifecycle_gap"]
|
||||
}
|
||||
})
|
||||
];
|
||||
|
||||
const candidates = buildCandidateEvidence(evidence, "hybrid_store_plus_live");
|
||||
expect(candidates[0].candidate_id).toBe("cand-ev-1");
|
||||
expect(candidates[0].relation_pattern_hits).toContain("failed_edge:statement_to_document");
|
||||
expect(candidates[0].entity_backlinks.length).toBeGreaterThan(0);
|
||||
|
||||
const clusters = clusterCandidateEvidence(candidates);
|
||||
expect(clusters.length).toBe(2);
|
||||
expect(clusters.some((item) => item.candidates.length === 2)).toBe(true);
|
||||
});
|
||||
|
||||
it("detects baseline problem unit type from anomaly hints", () => {
|
||||
const candidates = buildCandidateEvidence(
|
||||
[
|
||||
buildEvidence({
|
||||
evidenceId: "ev-lifecycle",
|
||||
sourceId: "DOC-LC",
|
||||
payload: {
|
||||
anomaly_patterns: ["lifecycle_gap"]
|
||||
}
|
||||
})
|
||||
],
|
||||
"store_feature_risk"
|
||||
);
|
||||
const cluster = clusterCandidateEvidence(candidates)[0];
|
||||
expect(detectProblemUnitType(cluster)).toBe("lifecycle_anomaly_node");
|
||||
});
|
||||
|
||||
it("collapses duplicate problem units by signature", () => {
|
||||
const units: ProblemUnit[] = [
|
||||
{
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: "pu-1",
|
||||
problem_unit_type: "broken_chain_segment",
|
||||
title: "broken",
|
||||
mechanism_summary: "m1",
|
||||
business_defect_class: "failed_edge:statement_to_document",
|
||||
severity: { score: 0.7, grade: "high" },
|
||||
confidence: { score: 0.6, grade: "medium" },
|
||||
affected_entities: ["Document:DOC-1"],
|
||||
affected_documents: ["Document:DOC-1"],
|
||||
affected_postings: [],
|
||||
affected_accounts: [],
|
||||
affected_counterparties: [],
|
||||
affected_contracts: [],
|
||||
failed_expected_edge: "statement_to_document",
|
||||
evidence_pack: ["cand-1"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
snapshot_limitations: []
|
||||
},
|
||||
{
|
||||
schema_version: "problem_unit_v0_1",
|
||||
problem_unit_id: "pu-2",
|
||||
problem_unit_type: "broken_chain_segment",
|
||||
title: "broken",
|
||||
mechanism_summary: "m2",
|
||||
business_defect_class: "failed_edge:statement_to_document",
|
||||
severity: { score: 0.8, grade: "high" },
|
||||
confidence: { score: 0.7, grade: "high" },
|
||||
affected_entities: ["Document:DOC-1"],
|
||||
affected_documents: ["Document:DOC-1"],
|
||||
affected_postings: [],
|
||||
affected_accounts: [],
|
||||
affected_counterparties: [],
|
||||
affected_contracts: [],
|
||||
failed_expected_edge: "statement_to_document",
|
||||
evidence_pack: ["cand-2"],
|
||||
entity_backlinks: [{ entity: "Document", id: "DOC-1" }],
|
||||
snapshot_limitations: []
|
||||
}
|
||||
];
|
||||
|
||||
const collapsed = collapseDuplicates(units);
|
||||
expect(collapsed.duplicate_collapses).toBe(1);
|
||||
expect(collapsed.problem_units.length).toBe(1);
|
||||
expect(collapsed.problem_units[0].evidence_pack).toEqual(["cand-1", "cand-2"]);
|
||||
expect(collapsed.problem_units[0].severity.score).toBe(0.8);
|
||||
});
|
||||
|
||||
it("assembles problem units and summary with bounded scaffold fields", () => {
|
||||
const assembled = assembleProblemUnits({
|
||||
route: "hybrid_store_plus_live",
|
||||
evidence: [
|
||||
buildEvidence({
|
||||
evidenceId: "ev-1",
|
||||
sourceId: "DOC-1",
|
||||
payload: {
|
||||
failed_expected_edge: "statement_to_document",
|
||||
anomaly_patterns: ["period_close_risk"]
|
||||
},
|
||||
confidence: "high"
|
||||
}),
|
||||
buildEvidence({
|
||||
evidenceId: "ev-2",
|
||||
sourceId: "DOC-2",
|
||||
payload: {
|
||||
anomaly_patterns: ["settlement_tail"]
|
||||
},
|
||||
confidence: "low"
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
expect(assembled.candidate_evidence.length).toBe(2);
|
||||
expect(assembled.problem_units.length).toBeGreaterThan(0);
|
||||
expect(assembled.problem_unit_summary.schema_version).toBe("problem_unit_summary_v0_1");
|
||||
expect(assembled.problem_unit_summary.units_total).toBe(assembled.problem_units.length);
|
||||
expect(Array.isArray(assembled.problem_unit_summary.unit_types)).toBe(true);
|
||||
expect(typeof assembled.problem_unit_summary.severity_distribution.low).toBe("number");
|
||||
expect(typeof assembled.problem_unit_summary.confidence_distribution.medium).toBe("number");
|
||||
expect(typeof assembled.problem_unit_summary.duplicate_collapses).toBe("number");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const PROBLEM_UNITS_FLAG = "FEATURE_ASSISTANT_PROBLEM_UNITS_V1";
|
||||
const ORIGINAL_PROBLEM_UNITS_FLAG = process.env[PROBLEM_UNITS_FLAG];
|
||||
|
||||
function restoreFlag(): void {
|
||||
if (ORIGINAL_PROBLEM_UNITS_FLAG === undefined) {
|
||||
delete process.env[PROBLEM_UNITS_FLAG];
|
||||
} else {
|
||||
process.env[PROBLEM_UNITS_FLAG] = ORIGINAL_PROBLEM_UNITS_FLAG;
|
||||
}
|
||||
}
|
||||
|
||||
async function normalizeWithFlag(flagValue: "0" | "1") {
|
||||
process.env[PROBLEM_UNITS_FLAG] = flagValue;
|
||||
vi.resetModules();
|
||||
const { normalizeRetrievalResult } = await import("../src/services/retrievalResultNormalizer");
|
||||
return normalizeRetrievalResult("F1", ["R1"], "hybrid_store_plus_live", {
|
||||
status: "ok",
|
||||
result_type: "list",
|
||||
items: [
|
||||
{
|
||||
source_entity: "Document",
|
||||
source_id: "DOC-1",
|
||||
risk_score: 4
|
||||
}
|
||||
],
|
||||
summary: {
|
||||
broad_query_detected: false
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
evidence_id: "ev-1",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "hybrid_store_plus_live",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: "DOC-1",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
failed_expected_edge: "statement_to_document",
|
||||
anomaly_patterns: ["period_close_risk"],
|
||||
confidence: "medium"
|
||||
}
|
||||
],
|
||||
why_included: ["test"],
|
||||
selection_reason: ["test"],
|
||||
risk_factors: ["test"],
|
||||
business_interpretation: ["test"],
|
||||
confidence: "medium",
|
||||
limitations: [],
|
||||
errors: []
|
||||
});
|
||||
}
|
||||
|
||||
describe.sequential("retrieval dual payload compatibility for problem units", () => {
|
||||
afterEach(() => {
|
||||
restoreFlag();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("keeps legacy payload intact when FEATURE_ASSISTANT_PROBLEM_UNITS_V1 is OFF", async () => {
|
||||
const result = await normalizeWithFlag("0");
|
||||
expect(Array.isArray(result.items)).toBe(true);
|
||||
expect(result.items.length).toBe(1);
|
||||
expect(result.raw_entities).toBeUndefined();
|
||||
expect(result.candidate_evidence).toBeUndefined();
|
||||
expect(result.problem_units).toBeUndefined();
|
||||
expect(result.problem_unit_summary).toBeUndefined();
|
||||
});
|
||||
|
||||
it("adds Stage 2 dual payload fields when FEATURE_ASSISTANT_PROBLEM_UNITS_V1 is ON", async () => {
|
||||
const off = await normalizeWithFlag("0");
|
||||
const on = await normalizeWithFlag("1");
|
||||
|
||||
expect(on.items).toEqual(off.items);
|
||||
expect(on.raw_entities).toEqual(off.items);
|
||||
expect(Array.isArray(on.candidate_evidence)).toBe(true);
|
||||
expect(on.candidate_evidence?.length).toBeGreaterThan(0);
|
||||
expect(Array.isArray(on.problem_units)).toBe(true);
|
||||
expect(on.problem_units?.length).toBeGreaterThan(0);
|
||||
expect(on.problem_unit_summary?.schema_version).toBe("problem_unit_summary_v0_1");
|
||||
expect(on.problem_unit_summary?.units_total).toBe(on.problem_units?.length);
|
||||
expect(on.summary.problem_units_enabled).toBe(true);
|
||||
expect(on.summary.candidate_evidence_count).toBe(on.candidate_evidence?.length);
|
||||
expect(on.summary.problem_units_count).toBe(on.problem_units?.length);
|
||||
expect(on.summary.problem_unit_duplicate_collapses).toBe(on.problem_unit_summary?.duplicate_collapses);
|
||||
expect(on.summary.problem_unit_types).toEqual(on.problem_unit_summary?.unit_types);
|
||||
expect(on.summary.problem_unit_severity_distribution).toEqual(on.problem_unit_summary?.severity_distribution);
|
||||
expect(on.summary.problem_unit_confidence_distribution).toEqual(on.problem_unit_summary?.confidence_distribution);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AssistantSessionStore } from "../src/services/assistantSessionStore";
|
||||
import { createEmptyInvestigationState } from "../src/services/investigationState";
|
||||
|
||||
describe("assistant session backward compatibility", () => {
|
||||
it("lazy-upgrades legacy session objects without investigation_state", () => {
|
||||
@@ -35,4 +36,42 @@ describe("assistant session backward compatibility", () => {
|
||||
expect(ensured.items.length).toBe(0);
|
||||
expect(ensured.investigation_state?.schema_version).toBe("investigation_state_v1");
|
||||
});
|
||||
|
||||
it("preserves optional stage2 problem_unit_state in session clone flow", () => {
|
||||
const store = new AssistantSessionStore();
|
||||
const sessionsMap = (store as unknown as { sessions: Map<string, unknown> }).sessions;
|
||||
const sessionId = "legacy-session-3";
|
||||
const baseState = createEmptyInvestigationState(sessionId, "2026-03-26T10:00:00.000Z");
|
||||
|
||||
sessionsMap.set(sessionId, {
|
||||
session_id: sessionId,
|
||||
updated_at: "2026-03-26T10:00:00.000Z",
|
||||
items: [],
|
||||
investigation_state: {
|
||||
...baseState,
|
||||
status: "active",
|
||||
problem_unit_state: {
|
||||
active_problem_units: ["pu-1", "pu-2"],
|
||||
resolved_problem_units: ["pu-0"],
|
||||
problem_unit_backlinks: [
|
||||
{
|
||||
problem_unit_id: "pu-1",
|
||||
entity_backlinks: [
|
||||
{
|
||||
entity: "counterparty",
|
||||
id: "cp-1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
focus_problem_types: ["broken_chain_segment"]
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const session = store.getSession(sessionId);
|
||||
expect(session).toBeTruthy();
|
||||
expect(session?.investigation_state?.problem_unit_state?.active_problem_units).toEqual(["pu-1", "pu-2"]);
|
||||
expect(session?.investigation_state?.problem_unit_state?.focus_problem_types).toEqual(["broken_chain_segment"]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user