Initial import NDC_1C
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { composeAssistantAnswer } from "../src/services/answerComposer";
|
||||
import type { UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithFlags(flags: {
|
||||
answerPolicy: "0" | "1";
|
||||
broad: "0" | "1";
|
||||
evidenceGate: "0" | "1";
|
||||
antiGeneric: "0" | "1";
|
||||
}) {
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = flags.answerPolicy;
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = flags.broad;
|
||||
process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = flags.evidenceGate;
|
||||
process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = flags.antiGeneric;
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
function firstRoutedResult(body: Record<string, unknown>): Record<string, unknown> | null {
|
||||
const retrieval = Array.isArray((body.debug as { retrieval_results?: unknown[] } | undefined)?.retrieval_results)
|
||||
? ((body.debug as { retrieval_results?: unknown[] }).retrieval_results as Record<string, unknown>[])
|
||||
: [];
|
||||
return retrieval.find((item) => String(item.route ?? "") !== "no_route") ?? null;
|
||||
}
|
||||
|
||||
describe.sequential("assistant answer policy v1.1", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("keeps focused grounded answer direct and useful", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь счет 97 за 2020-06 по документам и выдели отклонения."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).toBe("factual_with_explanation");
|
||||
expect(String(response.body.assistant_reply)).toContain("Answer summary:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Direct answer:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Mechanism block:");
|
||||
|
||||
const structure = response.body.debug?.answer_structure_v11;
|
||||
expect(structure?.mechanism_block).toBeTruthy();
|
||||
expect(["grounded", "limited", "unresolved"]).toContain(structure?.mechanism_block?.status);
|
||||
|
||||
const routed = firstRoutedResult(response.body);
|
||||
const summary = (routed?.summary as Record<string, unknown>) ?? {};
|
||||
expect(summary.minimum_evidence_failed).not.toBe(true);
|
||||
});
|
||||
|
||||
it("renders broad partial answer with explicit limitations and concrete next steps", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Покажи в целом общую картину и топ рисков по документам за июнь 2020."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).toBe("partial_coverage");
|
||||
expect(String(response.body.assistant_reply)).toContain("Uncertainty block:");
|
||||
expect(String(response.body.assistant_reply)).toContain("Next step block:");
|
||||
|
||||
const structure = response.body.debug?.answer_structure_v11;
|
||||
expect(structure?.answer_summary).toContain("частич");
|
||||
expect(Array.isArray(structure?.uncertainty_block?.limitations)).toBe(true);
|
||||
expect(structure?.uncertainty_block?.limitations?.length).toBeGreaterThan(0);
|
||||
expect(Array.isArray(structure?.next_step_block?.recommended_actions)).toBe(true);
|
||||
expect(structure?.next_step_block?.recommended_actions?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("uses domain-specific clarification prompts when support is insufficient", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Что не так по документ #123?"
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).toBe("clarification_required");
|
||||
|
||||
const structure = response.body.debug?.answer_structure_v11;
|
||||
const clarifications = structure?.next_step_block?.clarification_questions ?? [];
|
||||
expect(Array.isArray(clarifications)).toBe(true);
|
||||
expect(clarifications.length).toBeGreaterThan(0);
|
||||
expect(clarifications.some((item: string) => /период|счет|документ|контрагент/i.test(String(item)))).toBe(true);
|
||||
expect(String(response.body.assistant_reply)).toContain("clarify:");
|
||||
});
|
||||
|
||||
it("does not fabricate mechanism when mechanism_note is unresolved", () => {
|
||||
const retrievalResult: UnifiedRetrievalResult = {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "store_feature_risk",
|
||||
status: "ok",
|
||||
result_type: "list",
|
||||
items: [{ source_entity: "Document", source_id: "doc-weak-1" }],
|
||||
summary: {
|
||||
broad_query_detected: false,
|
||||
broad_result_flag: false,
|
||||
minimum_evidence_failed: false,
|
||||
narrowing_strength: "strong"
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
evidence_id: "ev-weak",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "doc-weak-1",
|
||||
period: "2020-06",
|
||||
canonical_ref: "evidence_source_ref_v1|snapshot_2020|document|doc-weak-1|2020-06"
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "store_feature_risk",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: "doc-weak-1",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
evidence_kind: "anomaly_signal",
|
||||
mechanism_note: null,
|
||||
confidence: "low",
|
||||
limitation: {
|
||||
reason_code: "missing_mechanism",
|
||||
note: "Mechanism could not be resolved."
|
||||
},
|
||||
payload: {
|
||||
risk_score: 1
|
||||
}
|
||||
}
|
||||
],
|
||||
why_included: ["synthetic-test"],
|
||||
selection_reason: ["synthetic-test"],
|
||||
risk_factors: [],
|
||||
business_interpretation: [],
|
||||
confidence: "low",
|
||||
limitations: ["Weak mechanism evidence."],
|
||||
errors: []
|
||||
};
|
||||
|
||||
const output = composeAssistantAnswer({
|
||||
userMessage: "Проверь риск по документу doc-weak-1 за 2020-06.",
|
||||
routeSummary: {
|
||||
mode: "deterministic_v2",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [],
|
||||
fallback: {
|
||||
type: "none",
|
||||
message: null
|
||||
}
|
||||
},
|
||||
retrievalResults: [retrievalResult],
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить риск документа",
|
||||
subject_tokens: ["документ"],
|
||||
status: "covered",
|
||||
route: "store_feature_risk"
|
||||
}
|
||||
],
|
||||
coverageReport: {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 1,
|
||||
requirements_uncovered: [],
|
||||
requirements_partially_covered: [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
},
|
||||
groundingCheck: {
|
||||
status: "grounded",
|
||||
route_subject_match: true,
|
||||
missing_requirements: [],
|
||||
reasons: [],
|
||||
why_included_summary: ["synthetic-test"],
|
||||
selection_reason_summary: ["synthetic-test"]
|
||||
},
|
||||
enableAnswerPolicyV11: true
|
||||
});
|
||||
|
||||
expect(output.answer_structure_v11?.mechanism_block?.status).toBe("unresolved");
|
||||
expect(output.answer_structure_v11?.mechanism_block?.mechanism_notes).toEqual([]);
|
||||
expect(output.answer_structure_v11?.mechanism_block?.limitation_reason_codes).toContain("missing_mechanism");
|
||||
expect(output.assistant_reply).toContain("mechanism_note is intentionally omitted");
|
||||
});
|
||||
|
||||
it("preserves legacy reply path when policy flag is OFF", async () => {
|
||||
const appLegacy = await createAppWithFlags({
|
||||
answerPolicy: "0",
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const legacy = await request(appLegacy).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь счет 97 за 2020-06 по документам и выдели отклонения."
|
||||
});
|
||||
|
||||
expect(legacy.status).toBe(200);
|
||||
expect(String(legacy.body.assistant_reply)).not.toContain("Answer summary:");
|
||||
|
||||
const appPolicy = await createAppWithFlags({
|
||||
answerPolicy: "1",
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const policy = await request(appPolicy).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь счет 97 за 2020-06 по документам и выдели отклонения."
|
||||
});
|
||||
|
||||
expect(policy.status).toBe(200);
|
||||
expect(String(policy.body.assistant_reply)).toContain("Answer summary:");
|
||||
expect(String(policy.body.assistant_reply)).not.toBe(String(legacy.body.assistant_reply));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithFlags(flags: {
|
||||
broad: "0" | "1";
|
||||
evidenceGate: "0" | "1";
|
||||
antiGeneric: "0" | "1";
|
||||
}) {
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = flags.broad;
|
||||
process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = flags.evidenceGate;
|
||||
process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = flags.antiGeneric;
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
function firstRoutedResult(body: Record<string, unknown>): Record<string, unknown> | null {
|
||||
const retrieval = Array.isArray((body.debug as { retrieval_results?: unknown[] } | undefined)?.retrieval_results)
|
||||
? ((body.debug as { retrieval_results?: unknown[] }).retrieval_results as Record<string, unknown>[])
|
||||
: [];
|
||||
return retrieval.find((item) => String(item.route ?? "") !== "no_route") ?? null;
|
||||
}
|
||||
|
||||
describe.sequential("assistant broad guard", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("keeps focused queries from degrading under broad guard", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь НДС по счету 19 за 2020-06 и рискованные записи по документам."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const routed = firstRoutedResult(response.body);
|
||||
expect(routed).toBeTruthy();
|
||||
|
||||
const summary = (routed?.summary as Record<string, unknown>) ?? {};
|
||||
expect(summary.broad_guard_applied).toBe(false);
|
||||
expect(summary.minimum_evidence_failed).toBe(false);
|
||||
expect(response.body.reply_type).not.toBe("clarification_required");
|
||||
});
|
||||
|
||||
it("degrades broad ranking output to partial instead of deceptively strong factual", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Покажи в целом общую картину и топ рисков по документам за июнь 2020."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const routed = firstRoutedResult(response.body);
|
||||
expect(routed).toBeTruthy();
|
||||
expect(routed?.route).toBe("batch_refresh_then_store");
|
||||
|
||||
const summary = (routed?.summary as Record<string, unknown>) ?? {};
|
||||
expect(summary.broad_guard_applied).toBe(true);
|
||||
expect(summary.minimum_evidence_failed).toBe(true);
|
||||
expect(summary.anti_generic_guard_applied).toBe(true);
|
||||
expect(summary.broad_result_flag).toBe(true);
|
||||
expect(["partial_coverage", "clarification_required"]).toContain(String(response.body.reply_type));
|
||||
expect(response.body.reply_type).not.toBe("factual_with_explanation");
|
||||
});
|
||||
|
||||
it("returns clarification when broad query has insufficient support", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
broad: "1",
|
||||
evidenceGate: "1",
|
||||
antiGeneric: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Что не так по документ #123?"
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const routed = firstRoutedResult(response.body);
|
||||
expect(routed).toBeTruthy();
|
||||
expect(routed?.route).toBe("live_mcp_drilldown");
|
||||
|
||||
const summary = (routed?.summary as Record<string, unknown>) ?? {};
|
||||
expect(summary.broad_guard_applied).toBe(true);
|
||||
expect(summary.minimum_evidence_failed).toBe(true);
|
||||
expect(summary.broad_result_flag).toBe(true);
|
||||
expect(summary.degraded_to).toBe("clarification");
|
||||
expect(response.body.reply_type).toBe("clarification_required");
|
||||
});
|
||||
|
||||
it("supports legacy behavior when broad guard flags are OFF", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
broad: "0",
|
||||
evidenceGate: "0",
|
||||
antiGeneric: "0"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Покажи в целом общую картину и топ рисков по документам за июнь 2020."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const routed = firstRoutedResult(response.body);
|
||||
expect(routed).toBeTruthy();
|
||||
|
||||
const summary = (routed?.summary as Record<string, unknown>) ?? {};
|
||||
expect(summary.broad_guard_applied).toBeUndefined();
|
||||
expect(summary.minimum_evidence_failed).toBeUndefined();
|
||||
expect(summary.anti_generic_guard_applied).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { RouteHintSummary } from "../src/types/normalizer";
|
||||
import type { UnifiedRetrievalResult } from "../src/types/assistant";
|
||||
import {
|
||||
ACCOUNTANT_SCORING_RUBRIC_V01,
|
||||
INVESTIGATION_MAX_EVIDENCE_REFS,
|
||||
INVESTIGATION_MAX_UNCERTAINTIES
|
||||
} from "../src/types/stage1Contracts";
|
||||
import { createEmptyInvestigationState, updateInvestigationState } from "../src/services/investigationState";
|
||||
|
||||
function buildRouteSummary(): RouteHintSummary {
|
||||
return {
|
||||
mode: "deterministic_v2",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
planner: {
|
||||
total_fragments: 1,
|
||||
in_scope_fragments: 1,
|
||||
out_of_scope_fragments: 0,
|
||||
discarded_fragments: 0,
|
||||
contains_multiple_tasks: false
|
||||
},
|
||||
decisions: [
|
||||
{
|
||||
fragment_id: "F1",
|
||||
domain_relevance: "in_scope",
|
||||
business_scope: "company_specific_accounting",
|
||||
candidate_labels: ["anomaly_probe"],
|
||||
decision_flags: {
|
||||
has_multi_entity_scope: false,
|
||||
asks_for_chain_explanation: false,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: true,
|
||||
asks_for_anomaly_scan: true,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: true,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
route: "store_feature_risk",
|
||||
reason: "test-route"
|
||||
}
|
||||
],
|
||||
fallback: {
|
||||
type: "none",
|
||||
message: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildRetrievalResult(evidenceCount: number): UnifiedRetrievalResult {
|
||||
return {
|
||||
fragment_id: "F1",
|
||||
requirement_ids: ["R1"],
|
||||
route: "store_feature_risk",
|
||||
status: "ok",
|
||||
result_type: "list",
|
||||
items: [],
|
||||
summary: {},
|
||||
evidence: Array.from({ length: evidenceCount }, (_, index) => ({
|
||||
evidence_id: `ev-${index + 1}`,
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
source_ref: {
|
||||
schema_version: "evidence_source_ref_v1",
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: `doc-${index + 1}`,
|
||||
period: "2020-06",
|
||||
canonical_ref: `evidence_source_ref_v1|snapshot_2020|document|doc-${index + 1}|2020-06`
|
||||
},
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "store_feature_risk",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "document",
|
||||
id: `doc-${index + 1}`,
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: index
|
||||
}
|
||||
},
|
||||
evidence_kind: "anomaly_signal",
|
||||
mechanism_note: "Risk signal",
|
||||
confidence: "medium",
|
||||
limitation: null,
|
||||
payload: { risk_score: 2 }
|
||||
})),
|
||||
why_included: [],
|
||||
selection_reason: [],
|
||||
risk_factors: [],
|
||||
business_interpretation: [],
|
||||
confidence: "high",
|
||||
limitations: ["Need period clarification"],
|
||||
errors: []
|
||||
};
|
||||
}
|
||||
|
||||
describe("stage1 contract scaffolding", () => {
|
||||
it("provides rubric v0.1 for accountant-facing metrics", () => {
|
||||
const metricNames = Object.keys(ACCOUNTANT_SCORING_RUBRIC_V01);
|
||||
expect(metricNames).toEqual([
|
||||
"retrieval_differentiation_rate",
|
||||
"generic_explanation_rate",
|
||||
"accountant_actionability_score",
|
||||
"false_confidence_rate",
|
||||
"broad_answer_rate",
|
||||
"mechanism_specificity_score",
|
||||
"followup_context_retention_score"
|
||||
]);
|
||||
for (const metric of metricNames) {
|
||||
const bands = ACCOUNTANT_SCORING_RUBRIC_V01[metric as keyof typeof ACCOUNTANT_SCORING_RUBRIC_V01];
|
||||
expect(bands.some((item) => item.score === 0)).toBe(true);
|
||||
expect(bands.some((item) => item.score === 5)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("updates investigation_state with bounded fields", () => {
|
||||
const initial = createEmptyInvestigationState("asst-contract-test", "2026-03-25T10:00:00.000Z");
|
||||
const updated = updateInvestigationState({
|
||||
previous: initial,
|
||||
timestamp: "2026-03-25T10:01:00.000Z",
|
||||
questionId: "msg-1",
|
||||
userMessage: "Prover schet 97 za 2020-06 i podsveti risk.",
|
||||
routeSummary: buildRouteSummary(),
|
||||
requirements: [
|
||||
{
|
||||
requirement_id: "R1",
|
||||
source_fragment_id: "F1",
|
||||
requirement_text: "Проверить счет 97",
|
||||
subject_tokens: ["счет_97"],
|
||||
status: "covered",
|
||||
route: "store_feature_risk"
|
||||
}
|
||||
],
|
||||
coverageReport: {
|
||||
requirements_total: 1,
|
||||
requirements_covered: 1,
|
||||
requirements_uncovered: [],
|
||||
requirements_partially_covered: [],
|
||||
clarification_needed_for: [],
|
||||
out_of_scope_requirements: []
|
||||
},
|
||||
retrievalResults: [buildRetrievalResult(40)],
|
||||
replyType: "factual_with_explanation"
|
||||
});
|
||||
|
||||
expect(updated.turn_index).toBe(1);
|
||||
expect(updated.status).toBe("active");
|
||||
expect(updated.focus.period).toBe("2020-06");
|
||||
expect(updated.focus.primary_accounts).toContain("97");
|
||||
expect(updated.evidence_refs.length).toBeLessThanOrEqual(INVESTIGATION_MAX_EVIDENCE_REFS);
|
||||
expect(updated.open_uncertainties.length).toBeLessThanOrEqual(INVESTIGATION_MAX_UNCERTAINTIES);
|
||||
expect(updated.query_mode_hint).toBe("direct_answer");
|
||||
expect(updated.followup_context?.referenced_requirement_ids).toEqual(["R1"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,260 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import request from "supertest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ASSISTANT_SESSIONS_DIR } from "../src/config";
|
||||
import { createApp } from "../src/server";
|
||||
|
||||
describe("assistant mode API", () => {
|
||||
it("processes message and returns assistant response with debug payload", async () => {
|
||||
const app = createApp();
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Prover schet 97 i podsveti riskovye zony."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.ok).toBe(true);
|
||||
expect(typeof response.body.session_id).toBe("string");
|
||||
expect(typeof response.body.assistant_reply).toBe("string");
|
||||
expect(typeof response.body.reply_type).toBe("string");
|
||||
expect(response.body.conversation_item?.role).toBe("assistant");
|
||||
expect(response.body.conversation_item?.reply_type).toBe(response.body.reply_type);
|
||||
expect(response.body.debug?.trace_id).toBeTypeOf("string");
|
||||
expect(Array.isArray(response.body.debug?.routes)).toBe(true);
|
||||
expect(Array.isArray(response.body.debug?.requirements_extracted)).toBe(true);
|
||||
expect(typeof response.body.debug?.coverage_report?.requirements_total).toBe("number");
|
||||
expect(typeof response.body.debug?.answer_grounding_check?.status).toBe("string");
|
||||
expect(Array.isArray(response.body.debug?.retrieval_status)).toBe(true);
|
||||
expect(Array.isArray(response.body.debug?.retrieval_results)).toBe(true);
|
||||
expect(Array.isArray(response.body.conversation)).toBe(true);
|
||||
expect(response.body.conversation.length).toBe(2);
|
||||
});
|
||||
|
||||
it("keeps session-scoped history and returns it via session endpoint", async () => {
|
||||
const app = createApp();
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Sdelai proverku po postavshchikam."
|
||||
});
|
||||
expect(first.status).toBe(200);
|
||||
const sessionId = String(first.body.session_id);
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Dobav proverku po periodu 2020-06."
|
||||
});
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.session_id).toBe(sessionId);
|
||||
|
||||
const session = await request(app).get(`/api/assistant/session/${sessionId}`);
|
||||
expect(session.status).toBe(200);
|
||||
expect(session.body.ok).toBe(true);
|
||||
expect(session.body.session?.session_id).toBe(sessionId);
|
||||
expect(Array.isArray(session.body.session?.items)).toBe(true);
|
||||
expect(session.body.session.items.length).toBe(4);
|
||||
});
|
||||
|
||||
it("executes factual retrieval for routed fragments", async () => {
|
||||
const app = createApp();
|
||||
|
||||
const riskResponse = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь НДС и рискованные записи по документам."
|
||||
});
|
||||
|
||||
expect(riskResponse.status).toBe(200);
|
||||
expect(Array.isArray(riskResponse.body.debug?.retrieval_results)).toBe(true);
|
||||
expect(riskResponse.body.debug.retrieval_results.length).toBeGreaterThan(0);
|
||||
expect(riskResponse.body.debug.retrieval_results.some((item: { route?: string }) => item.route === "store_feature_risk")).toBe(true);
|
||||
expect(riskResponse.body.debug.retrieval_results.some((item: { status?: string }) => item.status === "ok")).toBe(true);
|
||||
expect(typeof riskResponse.body.reply_type).toBe("string");
|
||||
expect(["factual_with_explanation", "partial_coverage"]).toContain(riskResponse.body.reply_type);
|
||||
expect(String(riskResponse.body.assistant_reply)).toContain("Почему это попало в ответ");
|
||||
|
||||
const chainResponse = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку документов и оплат по контрагентам."
|
||||
});
|
||||
|
||||
expect(chainResponse.status).toBe(200);
|
||||
expect(Array.isArray(chainResponse.body.debug?.retrieval_results)).toBe(true);
|
||||
expect(chainResponse.body.debug.retrieval_results.some((item: { route?: string }) => item.route === "hybrid_store_plus_live")).toBe(true);
|
||||
const answerStructure = chainResponse.body.debug?.answer_structure_v11;
|
||||
const evidenceBlock = answerStructure?.evidence_block;
|
||||
if (Array.isArray(evidenceBlock?.evidence_ids) && evidenceBlock.evidence_ids.length > 0) {
|
||||
expect(Array.isArray(evidenceBlock.claim_evidence_links)).toBe(true);
|
||||
expect(typeof evidenceBlock.claim_evidence_links[0]?.claim_ref).toBe("string");
|
||||
expect(Array.isArray(evidenceBlock.claim_evidence_links[0]?.evidence_ids)).toBe(true);
|
||||
}
|
||||
expect(String(chainResponse.body.assistant_reply)).toContain("Основание отбора");
|
||||
});
|
||||
|
||||
it("keeps in-domain translit queries in scope and routed", async () => {
|
||||
const app = createApp();
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Prover schet 60 za 2020-06, gde taili postavshikov i kakie dokumenty ne zakryvayut oplaty."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).not.toBe("out_of_scope");
|
||||
expect(response.body.debug?.route_summary?.message_in_scope).toBe(true);
|
||||
expect(Array.isArray(response.body.debug?.routes)).toBe(true);
|
||||
expect(response.body.debug?.routes.some((item: { route?: string }) => item.route !== "no_route")).toBe(true);
|
||||
expect(Array.isArray(response.body.debug?.retrieval_results)).toBe(true);
|
||||
expect(response.body.debug?.retrieval_results.some((item: { status?: string }) => item.status === "ok")).toBe(true);
|
||||
});
|
||||
|
||||
it("avoids false route mismatch when supported evidence exists for bounded answer", async () => {
|
||||
const app = createApp();
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message:
|
||||
"Покажи хвосты поставщиков по счету 60 за 2020-06 и выдели, где проблема уже похожа на разрыв цепочки документов."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).not.toBe("route_mismatch_blocked");
|
||||
expect(response.body.debug?.answer_grounding_check?.status).not.toBe("route_mismatch_blocked");
|
||||
expect(["partial", "grounded"]).toContain(String(response.body.debug?.answer_grounding_check?.status));
|
||||
expect(response.body.reply_type).toBe("partial_coverage");
|
||||
});
|
||||
|
||||
it("blocks answer when critical domain token is not grounded", async () => {
|
||||
const app = createApp();
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь основные средства и рискованные записи."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.reply_type).toBe("route_mismatch_blocked");
|
||||
expect(response.body.debug?.answer_grounding_check?.status).toBe("route_mismatch_blocked");
|
||||
expect(response.body.debug?.answer_grounding_check?.route_subject_match).toBe(false);
|
||||
expect(Array.isArray(response.body.debug?.answer_grounding_check?.reasons)).toBe(true);
|
||||
expect(String(response.body.assistant_reply)).toContain("предмет результата не совпал");
|
||||
});
|
||||
|
||||
it("applies semantic narrowing profile for hybrid retrieval without GUID", async () => {
|
||||
const app = createApp();
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку по 51 и 60 счетам: где закрытие не тем документом."
|
||||
});
|
||||
expect(first.status).toBe(200);
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку по банку: где выписка, документ и проводка живут отдельно и повторяется паттерн."
|
||||
});
|
||||
expect(second.status).toBe(200);
|
||||
|
||||
const firstHybrid = (first.body.debug?.retrieval_results ?? []).find((item: { route?: string }) => item.route === "hybrid_store_plus_live");
|
||||
const secondHybrid = (second.body.debug?.retrieval_results ?? []).find((item: { route?: string }) => item.route === "hybrid_store_plus_live");
|
||||
|
||||
expect(firstHybrid).toBeTruthy();
|
||||
expect(secondHybrid).toBeTruthy();
|
||||
|
||||
const firstSummary = (firstHybrid as { summary?: Record<string, unknown> }).summary ?? {};
|
||||
const secondSummary = (secondHybrid as { summary?: Record<string, unknown> }).summary ?? {};
|
||||
|
||||
expect(firstSummary.semantic_narrowing_applied).toBe(true);
|
||||
expect(typeof firstSummary.source_records).toBe("number");
|
||||
expect(typeof firstSummary.filtered_records_after_narrowing).toBe("number");
|
||||
expect(Number(firstSummary.filtered_records_after_narrowing)).toBeLessThan(Number(firstSummary.source_records));
|
||||
|
||||
const firstProfile = firstSummary.semantic_profile as Record<string, unknown>;
|
||||
const secondProfile = secondSummary.semantic_profile as Record<string, unknown>;
|
||||
expect(firstProfile).toBeTruthy();
|
||||
expect(secondProfile).toBeTruthy();
|
||||
|
||||
expect(Array.isArray(firstProfile.account_scope)).toBe(true);
|
||||
expect((firstProfile.account_scope as string[]).includes("51")).toBe(true);
|
||||
expect((firstProfile.account_scope as string[]).includes("60")).toBe(true);
|
||||
|
||||
expect(Array.isArray(firstProfile.anomaly_patterns)).toBe(true);
|
||||
expect(Array.isArray(secondProfile.anomaly_patterns)).toBe(true);
|
||||
expect((firstProfile.anomaly_patterns as string[]).includes("wrong_document_type")).toBe(true);
|
||||
expect((secondProfile.anomaly_patterns as string[]).includes("repeated_anomaly")).toBe(true);
|
||||
});
|
||||
|
||||
it("writes one persistent JSON log file per session", async () => {
|
||||
const app = createApp();
|
||||
const sessionId = `asst-test-${Date.now()}-${Math.floor(Math.random() * 10000)}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Проверь НДС."
|
||||
});
|
||||
expect(first.status).toBe(200);
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
message: "Разложи цепочку документов по контрагентам."
|
||||
});
|
||||
expect(second.status).toBe(200);
|
||||
|
||||
const logPath = path.resolve(ASSISTANT_SESSIONS_DIR, `${sessionId}.json`);
|
||||
expect(fs.existsSync(logPath)).toBe(true);
|
||||
|
||||
const logPayload = JSON.parse(fs.readFileSync(logPath, "utf-8")) as {
|
||||
schema_version: string;
|
||||
session_id: string;
|
||||
counters: {
|
||||
total_messages: number;
|
||||
user_messages: number;
|
||||
assistant_messages: number;
|
||||
};
|
||||
turns: Array<{
|
||||
human_block: string;
|
||||
human_readable: {
|
||||
question_raw: string;
|
||||
question_understood: string;
|
||||
decomposition: string[];
|
||||
answer: string;
|
||||
};
|
||||
}>;
|
||||
conversation: unknown[];
|
||||
};
|
||||
|
||||
expect(logPayload.schema_version).toBe("assistant_session_log_v1");
|
||||
expect(logPayload.session_id).toBe(sessionId);
|
||||
expect(logPayload.counters.total_messages).toBe(4);
|
||||
expect(logPayload.counters.user_messages).toBe(2);
|
||||
expect(logPayload.counters.assistant_messages).toBe(2);
|
||||
expect(Array.isArray(logPayload.turns)).toBe(true);
|
||||
expect(logPayload.turns.length).toBe(2);
|
||||
expect(logPayload.turns[0].human_block).toContain("Вопрос:");
|
||||
expect(logPayload.turns[0].human_block).toContain("Понято как:");
|
||||
expect(logPayload.turns[0].human_block).toContain("Декомпозиция:");
|
||||
expect(logPayload.turns[0].human_block).toContain("Ответ:");
|
||||
expect(Array.isArray(logPayload.turns[0].human_readable.decomposition)).toBe(true);
|
||||
expect(Array.isArray(logPayload.conversation)).toBe(true);
|
||||
expect(logPayload.conversation.length).toBe(4);
|
||||
|
||||
const sameSessionFiles = fs.readdirSync(ASSISTANT_SESSIONS_DIR).filter((item) => item === `${sessionId}.json`);
|
||||
expect(sameSessionFiles.length).toBe(1);
|
||||
|
||||
fs.unlinkSync(logPath);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,227 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1",
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithFlags(flags: {
|
||||
accountantEval: "0" | "1";
|
||||
answerPolicy: "0" | "1";
|
||||
}): Promise<import("express").Express> {
|
||||
process.env.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1 = flags.accountantEval;
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = flags.answerPolicy;
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = "1";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
describe.sequential("assistant Stage 1 eval harness", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("runs assistant_stage1 harness and returns raw metrics + rubric bands", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage1",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage1_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.ok).toBe(true);
|
||||
expect(response.body.report?.eval_target).toBe("assistant_stage1");
|
||||
expect(response.body.report?.metrics?.raw).toBeTruthy();
|
||||
const rawMetricKeys = Object.keys(response.body.report?.metrics?.raw ?? {});
|
||||
expect(rawMetricKeys).toEqual([
|
||||
"retrieval_differentiation_rate",
|
||||
"generic_explanation_rate",
|
||||
"accountant_actionability_score",
|
||||
"false_confidence_rate",
|
||||
"broad_answer_rate",
|
||||
"mechanism_specificity_score",
|
||||
"followup_context_retention_score"
|
||||
]);
|
||||
expect(response.body.report?.rubric_bands?.generic_explanation_rate).toBeTruthy();
|
||||
expect(response.body.report?.feature_profile_snapshot).toBeTruthy();
|
||||
expect(response.body.report?.code_version).toBeTruthy();
|
||||
expect(typeof response.body.report?.run_timestamp).toBe("string");
|
||||
expect(Array.isArray(response.body.report?.results)).toBe(true);
|
||||
expect(response.body.report?.results?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("loads canonical suite metadata and keeps it stable", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage1",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage1_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.suite_id).toBe("assistant_stage1_canonical");
|
||||
expect(response.body.report?.suite_version).toBe("0.1.0");
|
||||
expect(response.body.report?.scenario_count).toBe(9);
|
||||
expect(Array.isArray(response.body.report?.case_ids)).toBe(true);
|
||||
expect(response.body.report?.case_ids?.length).toBe(9);
|
||||
});
|
||||
|
||||
it("handles follow-up cases as dedicated subset", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage1",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage1_canonical_v0_1.json",
|
||||
caseIds: ["S1-FOLLOWUP-INVESTIGATION", "S1-60-SUPPLIER-TAILS"],
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.subsets?.followup_cases_total).toBeGreaterThan(0);
|
||||
expect(response.body.report?.metrics?.raw?.followup_context_retention_score).not.toBeNull();
|
||||
});
|
||||
|
||||
it("builds comparison artifact from baseline and current runs", async () => {
|
||||
const baselineApp = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "0"
|
||||
});
|
||||
const baseline = await request(baselineApp).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage1",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage1_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(baseline.status).toBe(200);
|
||||
const baselinePath = String(baseline.body.report?.artifacts?.run_report_json_path ?? "");
|
||||
expect(baselinePath.length).toBeGreaterThan(0);
|
||||
|
||||
const currentApp = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1"
|
||||
});
|
||||
const current = await request(currentApp).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage1",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage1_canonical_v0_1.json",
|
||||
compare_with_report_file: baselinePath,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(current.status).toBe(200);
|
||||
expect(current.body.report?.comparison).toBeTruthy();
|
||||
expect(current.body.report?.comparison?.metric_deltas).toBeTruthy();
|
||||
expect(current.body.report?.comparison?.artifacts?.comparison_report_json_path).toBeTruthy();
|
||||
});
|
||||
|
||||
it("keeps legacy eval path unchanged by default", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
rawQuestions: "Проверь счет 60 за июнь 2020; Покажи риски по счету 97",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.report?.eval_target).toBeUndefined();
|
||||
expect(response.body.report?.metrics?.schema_validation_pass_rate).not.toBeUndefined();
|
||||
expect(response.body.report?.metrics?.route_resolution_accuracy).not.toBeUndefined();
|
||||
});
|
||||
|
||||
it("respects accountant eval feature flag OFF/ON", async () => {
|
||||
const appOff = await createAppWithFlags({
|
||||
accountantEval: "0",
|
||||
answerPolicy: "1"
|
||||
});
|
||||
const offResponse = await request(appOff).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage1",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage1_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(offResponse.status).toBe(409);
|
||||
expect(offResponse.body?.error?.code).toBe("ASSISTANT_STAGE1_EVAL_DISABLED");
|
||||
|
||||
const appOn = await createAppWithFlags({
|
||||
accountantEval: "1",
|
||||
answerPolicy: "1"
|
||||
});
|
||||
const onResponse = await request(appOn).post("/api/eval/run").send({
|
||||
eval_target: "assistant_stage1",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "assistant_stage1_canonical_v0_1.json",
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
expect(onResponse.status).toBe(200);
|
||||
expect(onResponse.body.report?.eval_target).toBe("assistant_stage1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,122 @@
|
||||
import request from "supertest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const FLAG_KEYS = [
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1",
|
||||
"FEATURE_ASSISTANT_CONTRACTS_V11"
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_FLAGS: Record<string, string | undefined> = Object.fromEntries(
|
||||
FLAG_KEYS.map((key) => [key, process.env[key]])
|
||||
);
|
||||
|
||||
function restoreFlags(): void {
|
||||
for (const key of FLAG_KEYS) {
|
||||
const original = ORIGINAL_FLAGS[key];
|
||||
if (original === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = original;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createAppWithFlags(flags: {
|
||||
state: "0" | "1";
|
||||
binding: "0" | "1";
|
||||
contracts?: "0" | "1";
|
||||
}) {
|
||||
process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 = flags.state;
|
||||
process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 = flags.binding;
|
||||
process.env.FEATURE_ASSISTANT_CONTRACTS_V11 = flags.contracts ?? "1";
|
||||
vi.resetModules();
|
||||
const { createApp } = await import("../src/server");
|
||||
return createApp();
|
||||
}
|
||||
|
||||
describe.sequential("assistant follow-up state binding", () => {
|
||||
afterEach(() => {
|
||||
restoreFlags();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("applies investigation_state binding in follow-up flow when flags are ON", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
state: "1",
|
||||
binding: "1"
|
||||
});
|
||||
const sessionId = `asst-wave2-on-${Date.now()}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку документов по контрагентам."
|
||||
});
|
||||
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body.debug?.followup_state_usage).toBeUndefined();
|
||||
expect(first.body.debug?.investigation_state_snapshot?.turn_index).toBe(1);
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "И по периоду 2020-06."
|
||||
});
|
||||
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.debug?.followup_state_usage?.applied).toBe(true);
|
||||
expect(second.body.debug?.followup_state_usage?.context_patch?.business_context_from_state).toBe(true);
|
||||
expect(second.body.debug?.followup_state_usage?.state_turn_index).toBe(1);
|
||||
expect(
|
||||
(second.body.debug?.routes ?? []).some((item: { route?: string }) => item.route && item.route !== "no_route")
|
||||
).toBe(true);
|
||||
expect(second.body.debug?.investigation_state_snapshot?.turn_index).toBe(2);
|
||||
});
|
||||
|
||||
it("does not apply follow-up binding when binding flag is OFF", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
state: "1",
|
||||
binding: "0"
|
||||
});
|
||||
const sessionId = `asst-wave2-off-${Date.now()}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Разложи цепочку документов по контрагентам."
|
||||
});
|
||||
expect(first.status).toBe(200);
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "И по периоду 2020-06."
|
||||
});
|
||||
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.debug?.followup_state_usage).toBeUndefined();
|
||||
expect((second.body.debug?.routes ?? []).every((item: { route?: string }) => item.route === "no_route")).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps legacy-like behavior when investigation state flag is OFF", async () => {
|
||||
const app = await createAppWithFlags({
|
||||
state: "0",
|
||||
binding: "1"
|
||||
});
|
||||
|
||||
const response = await request(app).post("/api/assistant/message").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "И по периоду 2020-06."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.debug?.investigation_state_snapshot).toBeNull();
|
||||
expect(response.body.debug?.followup_state_usage).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import request from "supertest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createApp } from "../src/server";
|
||||
|
||||
describe("POST /api/eval/run", () => {
|
||||
it("runs v2 eval using inline rawQuestions batch", async () => {
|
||||
const app = createApp();
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
useMock: true
|
||||
},
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
rawQuestions:
|
||||
"Проверь хвосты по поставщикам и разложи цепочку; Как вообще по ФСБУ; Покажи топ рисков за июнь 2020"
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.ok).toBe(true);
|
||||
expect(response.body.report?.schema_version).toBe("v2_0_2");
|
||||
expect(response.body.report?.cases_total).toBe(3);
|
||||
expect(typeof response.body.report?.metrics?.schema_validation_pass_rate).toBe("number");
|
||||
expect(response.body.report?.metrics?.route_resolution_accuracy).not.toBeUndefined();
|
||||
expect(response.body.report?.metrics?.execution_state_consistency_rate).not.toBeUndefined();
|
||||
expect(Array.isArray(response.body.report?.results)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,189 @@
|
||||
import type { NormalizedQueryV1, NormalizedQueryV2, NormalizedQueryV2_0_1, NormalizedQueryV2_0_2 } from "../src/types/normalizer";
|
||||
|
||||
export function normalizedFixture(): NormalizedQueryV1 {
|
||||
return {
|
||||
schema_version: "normalized_query_v1",
|
||||
user_question_raw: "По каким поставщикам не бьются взаиморасчеты?",
|
||||
normalized_question: "Показать поставщиков с расхождениями по взаиморасчетам.",
|
||||
intent_class: "cross_entity",
|
||||
business_problem_type: "reconciliation",
|
||||
domain_entities: ["контрагент", "документ", "проводка"],
|
||||
accounts_mentioned: ["60"],
|
||||
documents_mentioned: ["поступление", "списание"],
|
||||
registers_mentioned: ["взаиморасчеты"],
|
||||
period_scope: {
|
||||
type: "inferred",
|
||||
value: "2020-06",
|
||||
confidence: "medium"
|
||||
},
|
||||
requires: {
|
||||
needs_cross_entity_join: true,
|
||||
needs_causal_chain: true,
|
||||
needs_exact_object_trace: false,
|
||||
needs_ranking: false,
|
||||
needs_anomaly_summary: false,
|
||||
needs_runtime_truth: false,
|
||||
needs_period_cut: true,
|
||||
needs_evidence: true
|
||||
},
|
||||
expected_output_shape: "reconciliation_report",
|
||||
route_hint: "hybrid_store_plus_live",
|
||||
ambiguities: [],
|
||||
confidence: {
|
||||
overall: "medium",
|
||||
intent_class: "high",
|
||||
route_hint: "medium"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizedFixtureV2(): NormalizedQueryV2 {
|
||||
return {
|
||||
schema_version: "normalized_query_v2",
|
||||
user_message_raw: "Проверь по поставщикам хвосты и отдельно скажи, что не относится к данным компании.",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "medium",
|
||||
contains_multiple_tasks: true,
|
||||
fragments: [
|
||||
{
|
||||
fragment_id: "F1",
|
||||
raw_fragment_text: "Проверь по поставщикам хвосты",
|
||||
normalized_fragment_text: "Проверить хвосты по поставщикам",
|
||||
domain_relevance: "in_scope",
|
||||
business_scope: "company_specific_accounting",
|
||||
entity_hints: ["поставщик", "взаиморасчеты"],
|
||||
account_hints: ["60"],
|
||||
document_hints: ["документ"],
|
||||
register_hints: ["остатки"],
|
||||
time_scope: {
|
||||
type: "missing",
|
||||
value: null,
|
||||
confidence: "low"
|
||||
},
|
||||
flags: {
|
||||
has_multi_entity_scope: true,
|
||||
asks_for_chain_explanation: true,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: false,
|
||||
asks_for_anomaly_scan: true,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: true,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
candidate_labels: ["cross_entity", "anomaly_probe"],
|
||||
confidence: "medium"
|
||||
}
|
||||
],
|
||||
discarded_fragments: [
|
||||
{
|
||||
raw_fragment_text: "короче",
|
||||
reason: "noise_or_too_short"
|
||||
}
|
||||
],
|
||||
global_notes: {
|
||||
needs_clarification: true,
|
||||
clarification_reason: "Не указан период."
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizedFixtureV2_0_1(): NormalizedQueryV2_0_1 {
|
||||
return {
|
||||
schema_version: "normalized_query_v2_0_1",
|
||||
user_message_raw: "Проверь, что висит по 97 и где есть подозрительные хвосты.",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
contains_multiple_tasks: false,
|
||||
fragments: [
|
||||
{
|
||||
fragment_id: "F1",
|
||||
raw_fragment_text: "Проверь, что висит по 97 и где есть подозрительные хвосты",
|
||||
normalized_fragment_text: "Проверить зависшие записи по 97 и подозрительные хвосты",
|
||||
domain_relevance: "in_scope",
|
||||
business_scope: "company_specific_accounting",
|
||||
entity_hints: ["рбп"],
|
||||
account_hints: ["97"],
|
||||
document_hints: [],
|
||||
register_hints: ["остатки"],
|
||||
time_scope: {
|
||||
type: "missing",
|
||||
value: null,
|
||||
confidence: "low"
|
||||
},
|
||||
flags: {
|
||||
has_multi_entity_scope: false,
|
||||
asks_for_chain_explanation: false,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: true,
|
||||
asks_for_anomaly_scan: true,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: false,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
candidate_labels: ["rule_based_account_control", "anomaly_probe"],
|
||||
confidence: "high",
|
||||
execution_readiness: "executable_with_soft_assumptions",
|
||||
clarification_reason: null,
|
||||
soft_assumption_used: ["problem_scan_mode_enabled"]
|
||||
}
|
||||
],
|
||||
discarded_fragments: [],
|
||||
global_notes: {
|
||||
needs_clarification: false,
|
||||
clarification_reason: null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizedFixtureV2_0_2(): NormalizedQueryV2_0_2 {
|
||||
return {
|
||||
schema_version: "normalized_query_v2_0_2",
|
||||
user_message_raw: "Проверь зависшие истории по 97 и подсвети рискованные участки.",
|
||||
message_in_scope: true,
|
||||
scope_confidence: "high",
|
||||
contains_multiple_tasks: false,
|
||||
fragments: [
|
||||
{
|
||||
fragment_id: "F1",
|
||||
raw_fragment_text: "Проверь зависшие истории по 97 и подсвети рискованные участки",
|
||||
normalized_fragment_text: "Проверить зависшие истории по 97 и рискованные участки",
|
||||
domain_relevance: "in_scope",
|
||||
business_scope: "company_specific_accounting",
|
||||
entity_hints: ["рбп"],
|
||||
account_hints: ["97"],
|
||||
document_hints: [],
|
||||
register_hints: ["остатки"],
|
||||
time_scope: {
|
||||
type: "missing",
|
||||
value: null,
|
||||
confidence: "low"
|
||||
},
|
||||
flags: {
|
||||
has_multi_entity_scope: false,
|
||||
asks_for_chain_explanation: false,
|
||||
asks_for_ranking_or_top: false,
|
||||
asks_for_period_summary: false,
|
||||
asks_for_rule_check: true,
|
||||
asks_for_anomaly_scan: true,
|
||||
asks_for_exact_object_trace: false,
|
||||
asks_for_evidence: false,
|
||||
mentions_period_close_context: false
|
||||
},
|
||||
candidate_labels: ["rule_based_account_control", "anomaly_probe"],
|
||||
confidence: "high",
|
||||
execution_readiness: "executable_with_soft_assumptions",
|
||||
clarification_reason: null,
|
||||
soft_assumption_used: ["problem_scan_mode_enabled"],
|
||||
route_status: "routed",
|
||||
no_route_reason: null
|
||||
}
|
||||
],
|
||||
discarded_fragments: [],
|
||||
global_notes: {
|
||||
needs_clarification: false,
|
||||
clarification_reason: null
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import request from "supertest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createApp } from "../src/server";
|
||||
import { INVESTIGATION_MAX_EVIDENCE_REFS, INVESTIGATION_MAX_UNCERTAINTIES } from "../src/types/stage1Contracts";
|
||||
|
||||
describe("investigation_state flow scaffolding", () => {
|
||||
it("keeps bounded investigation_state across follow-up turns", async () => {
|
||||
const app = createApp();
|
||||
const sessionId = `asst-wave1-${Date.now()}`;
|
||||
|
||||
const first = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Prover schet 97 i riskovye zony za 2020-06."
|
||||
});
|
||||
|
||||
expect(first.status).toBe(200);
|
||||
expect(first.body.debug?.investigation_state_snapshot?.schema_version).toBe("investigation_state_v1");
|
||||
expect(first.body.debug?.answer_structure_v11?.schema_version).toBe("answer_structure_v1_1");
|
||||
expect(first.body.debug?.followup_state_usage).toBeUndefined();
|
||||
|
||||
const evidenceResult = (first.body.debug?.retrieval_results ?? []).find(
|
||||
(item: { evidence?: unknown[] }) => Array.isArray(item.evidence) && item.evidence.length > 0
|
||||
) as { evidence?: Array<{ pointer?: { source?: { entity?: string } } }> } | undefined;
|
||||
|
||||
if (evidenceResult?.evidence?.length) {
|
||||
expect(typeof evidenceResult.evidence[0].pointer?.source?.entity).toBe("string");
|
||||
}
|
||||
|
||||
const second = await request(app).post("/api/assistant/message").send({
|
||||
session_id: sessionId,
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
user_message: "Dobav proverku po postavshchikam i utochni nezakrytye trebovaniya."
|
||||
});
|
||||
|
||||
expect(second.status).toBe(200);
|
||||
expect(second.body.debug?.investigation_state_snapshot?.turn_index).toBe(2);
|
||||
expect(second.body.debug?.followup_state_usage?.applied).toBe(true);
|
||||
|
||||
const sessionResponse = await request(app).get(`/api/assistant/session/${sessionId}`);
|
||||
expect(sessionResponse.status).toBe(200);
|
||||
|
||||
const investigationState = sessionResponse.body.session?.investigation_state;
|
||||
expect(investigationState).toBeTruthy();
|
||||
expect(investigationState.turn_index).toBe(2);
|
||||
expect(Array.isArray(investigationState.evidence_refs)).toBe(true);
|
||||
expect(Array.isArray(investigationState.open_uncertainties)).toBe(true);
|
||||
expect(investigationState.evidence_refs.length).toBeLessThanOrEqual(INVESTIGATION_MAX_EVIDENCE_REFS);
|
||||
expect(investigationState.open_uncertainties.length).toBeLessThanOrEqual(INVESTIGATION_MAX_UNCERTAINTIES);
|
||||
expect(typeof investigationState.question_id).toBe("string");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
import request from "supertest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createApp } from "../src/server";
|
||||
|
||||
describe("POST /api/normalize", () => {
|
||||
it("returns normalized v1 payload in mock mode", async () => {
|
||||
const app = createApp();
|
||||
const response = await request(app).post("/api/normalize").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v1_1_2_1",
|
||||
userQuestion: "По каким поставщикам не бьются взаиморасчеты по 60 счету?"
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.trace_id).toBeTypeOf("string");
|
||||
expect(response.body.schema_version).toBe("v1");
|
||||
expect(response.body.validation?.passed).toBe(true);
|
||||
expect(response.body.normalized?.schema_version).toBe("normalized_query_v1");
|
||||
});
|
||||
|
||||
it("returns normalized v2 payload in mock mode", async () => {
|
||||
const app = createApp();
|
||||
const response = await request(app).post("/api/normalize").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2",
|
||||
userQuestion: "Проверь хвосты по поставщикам и отдельно все, что не относится к данным компании."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.trace_id).toBeTypeOf("string");
|
||||
expect(response.body.schema_version).toBe("v2");
|
||||
expect(response.body.validation?.passed).toBe(true);
|
||||
expect(response.body.normalized?.schema_version).toBe("normalized_query_v2");
|
||||
expect(Array.isArray(response.body.normalized?.fragments)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns normalized v2.0.1 payload in mock mode with execution_readiness", async () => {
|
||||
const app = createApp();
|
||||
const response = await request(app).post("/api/normalize").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_1",
|
||||
userQuestion: "Покажи, что висит по 97 и что выглядит подозрительно."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.trace_id).toBeTypeOf("string");
|
||||
expect(response.body.schema_version).toBe("v2_0_1");
|
||||
expect(response.body.validation?.passed).toBe(true);
|
||||
expect(response.body.normalized?.schema_version).toBe("normalized_query_v2_0_1");
|
||||
expect(Array.isArray(response.body.normalized?.fragments)).toBe(true);
|
||||
expect(response.body.normalized?.fragments?.[0]?.execution_readiness).toBeTypeOf("string");
|
||||
});
|
||||
|
||||
it("returns normalized v2.0.2 payload in mock mode with route_status and no_route_reason", async () => {
|
||||
const app = createApp();
|
||||
const response = await request(app).post("/api/normalize").send({
|
||||
useMock: true,
|
||||
promptVersion: "normalizer_v2_0_2",
|
||||
userQuestion: "Проверь 97 и покажи, где логика учета выглядит подозрительно."
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.trace_id).toBeTypeOf("string");
|
||||
expect(response.body.schema_version).toBe("v2_0_2");
|
||||
expect(response.body.validation?.passed).toBe(true);
|
||||
expect(response.body.normalized?.schema_version).toBe("normalized_query_v2_0_2");
|
||||
expect(Array.isArray(response.body.normalized?.fragments)).toBe(true);
|
||||
expect(response.body.normalized?.fragments?.[0]?.route_status).toBeTypeOf("string");
|
||||
expect(response.body.normalized?.fragments?.[0]?.no_route_reason).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildPromptBundle, listBuiltinPromptPresets, loadDefaultPrompts } from "../src/services/promptBuilder";
|
||||
|
||||
describe("promptBuilder", () => {
|
||||
it("loads default prompts", () => {
|
||||
const defaults = loadDefaultPrompts();
|
||||
expect(defaults.systemPrompt.length).toBeGreaterThan(20);
|
||||
expect(defaults.developerPrompt.length).toBeGreaterThan(20);
|
||||
expect(defaults.domainPrompt.length).toBeGreaterThan(20);
|
||||
});
|
||||
|
||||
it("exposes v1, v1.1, v1.1.1, v1.1.2, v1.1.2.1, v2, v2.0.1 and v2.0.2 builtin presets", () => {
|
||||
const presets = listBuiltinPromptPresets();
|
||||
const versions = presets.map((item) => item.prompt_version);
|
||||
expect(versions).toContain("normalizer_v1");
|
||||
expect(versions).toContain("normalizer_v1_1");
|
||||
expect(versions).toContain("normalizer_v1_1_1");
|
||||
expect(versions).toContain("normalizer_v1_1_2");
|
||||
expect(versions).toContain("normalizer_v1_1_2_1");
|
||||
expect(versions).toContain("normalizer_v2");
|
||||
expect(versions).toContain("normalizer_v2_0_1");
|
||||
expect(versions).toContain("normalizer_v2_0_2");
|
||||
});
|
||||
|
||||
it("merges user prompt values", () => {
|
||||
const bundle = buildPromptBundle({
|
||||
systemPrompt: "S",
|
||||
developerPrompt: "D",
|
||||
domainPrompt: "N",
|
||||
schemaNotes: "schema",
|
||||
fewShotExamples: "fewshot"
|
||||
});
|
||||
expect(bundle.systemPrompt).toBe("S");
|
||||
expect(bundle.developerPrompt).toBe("D");
|
||||
expect(bundle.domainPrompt).toBe("N");
|
||||
expect(bundle.combinedDeveloperPrompt.includes("fewshot")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const ENRICHMENT_FLAG = "FEATURE_ASSISTANT_EVIDENCE_ENRICHMENT_V1";
|
||||
const ORIGINAL_FLAG_VALUE = process.env[ENRICHMENT_FLAG];
|
||||
|
||||
function restoreFlag(): void {
|
||||
if (ORIGINAL_FLAG_VALUE === undefined) {
|
||||
delete process.env[ENRICHMENT_FLAG];
|
||||
} else {
|
||||
process.env[ENRICHMENT_FLAG] = ORIGINAL_FLAG_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
async function normalizeSingleEvidence(flagValue: "0" | "1", evidenceRecord: Record<string, unknown>) {
|
||||
process.env[ENRICHMENT_FLAG] = flagValue;
|
||||
vi.resetModules();
|
||||
const { normalizeRetrievalResult } = await import("../src/services/retrievalResultNormalizer");
|
||||
return normalizeRetrievalResult("F1", ["R1"], "store_feature_risk", {
|
||||
status: "ok",
|
||||
result_type: "list",
|
||||
items: [],
|
||||
summary: {},
|
||||
evidence: [evidenceRecord],
|
||||
why_included: [],
|
||||
selection_reason: [],
|
||||
risk_factors: [],
|
||||
business_interpretation: [],
|
||||
confidence: "medium",
|
||||
limitations: [],
|
||||
errors: []
|
||||
});
|
||||
}
|
||||
|
||||
describe.sequential("retrieval evidence enrichment", () => {
|
||||
afterEach(() => {
|
||||
restoreFlag();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("builds deterministic canonical source_ref from pointer/source", async () => {
|
||||
const rawEvidence = {
|
||||
evidence_id: "ev-1",
|
||||
claim_ref: "requirement:R1",
|
||||
source_type: "retrieval_item",
|
||||
pointer: {
|
||||
fragment_id: "F1",
|
||||
route: "store_feature_risk",
|
||||
source: {
|
||||
namespace: "snapshot_2020",
|
||||
entity: "Document",
|
||||
id: "DOC-42",
|
||||
period: "2020-06"
|
||||
},
|
||||
locator: {
|
||||
field_path: "risk_score",
|
||||
item_index: 0
|
||||
}
|
||||
},
|
||||
risk_score: 3
|
||||
};
|
||||
|
||||
const first = await normalizeSingleEvidence("1", rawEvidence);
|
||||
const second = await normalizeSingleEvidence("1", rawEvidence);
|
||||
|
||||
expect(first.evidence[0].source_ref.canonical_ref).toBe(second.evidence[0].source_ref.canonical_ref);
|
||||
expect(first.evidence[0].source_ref.schema_version).toBe("evidence_source_ref_v1");
|
||||
expect(first.evidence[0].source_ref.namespace).toBe("snapshot_2020");
|
||||
expect(first.evidence[0].source_ref.entity).toBe("Document");
|
||||
expect(first.evidence[0].source_ref.id).toBe("DOC-42");
|
||||
});
|
||||
|
||||
it("uses honest weak-evidence fallback when mechanism is not reliable", async () => {
|
||||
const result = await normalizeSingleEvidence("1", {
|
||||
evidence_id: "ev-weak",
|
||||
source_entity: "DocumentJournal",
|
||||
source_id: "doc-weak-1",
|
||||
risk_score: 2
|
||||
});
|
||||
|
||||
expect(result.evidence[0].mechanism_note).toBeNull();
|
||||
expect(result.evidence[0].limitation?.reason_code).toBe("missing_mechanism");
|
||||
expect(result.evidence[0].confidence).toBe("low");
|
||||
});
|
||||
|
||||
it("maps explicit limitation to reason-coded structure", async () => {
|
||||
const result = await normalizeSingleEvidence("1", {
|
||||
evidence_id: "ev-limited",
|
||||
source_entity: "DocumentJournal",
|
||||
source_id: "doc-limited-1",
|
||||
limitation: "Snapshot-only evidence."
|
||||
});
|
||||
|
||||
expect(result.evidence[0].limitation?.reason_code).toBe("snapshot_only");
|
||||
expect(result.evidence[0].limitation?.note).toBe("Snapshot-only evidence.");
|
||||
});
|
||||
|
||||
it("keeps legacy inferred mechanism when enrichment flag is OFF", async () => {
|
||||
const result = await normalizeSingleEvidence("0", {
|
||||
evidence_id: "ev-legacy",
|
||||
source_entity: "DocumentJournal",
|
||||
source_id: "doc-legacy-1",
|
||||
risk_score: 2
|
||||
});
|
||||
|
||||
expect(typeof result.evidence[0].mechanism_note).toBe("string");
|
||||
expect(result.evidence[0].mechanism_note).toContain("Anomaly signal inferred");
|
||||
expect(result.evidence[0].limitation).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { toRouteHintSummary, toRouterInput } from "../src/services/routeHintAdapter";
|
||||
import { normalizedFixture, normalizedFixtureV2, normalizedFixtureV2_0_1, normalizedFixtureV2_0_2 } from "./fixtures";
|
||||
|
||||
describe("routeHintAdapter", () => {
|
||||
it("builds v1 route hint summary", () => {
|
||||
const summary = toRouteHintSummary(normalizedFixture());
|
||||
expect(summary.mode).toBe("legacy_v1");
|
||||
if (summary.mode !== "legacy_v1") {
|
||||
throw new Error("Expected legacy_v1 summary");
|
||||
}
|
||||
expect(summary.route_hint).toBe("hybrid_store_plus_live");
|
||||
expect(summary.decision_flags.needs_cross_entity_join).toBe(true);
|
||||
expect(summary.entities.accounts_mentioned).toEqual(["60"]);
|
||||
});
|
||||
|
||||
it("builds v2 deterministic route simulation", () => {
|
||||
const summary = toRouteHintSummary(normalizedFixtureV2());
|
||||
expect(summary.mode).toBe("deterministic_v2");
|
||||
if (summary.mode !== "deterministic_v2") {
|
||||
throw new Error("Expected deterministic_v2 summary");
|
||||
}
|
||||
expect(summary.planner.total_fragments).toBe(1);
|
||||
expect(summary.decisions[0]?.route).toBe("hybrid_store_plus_live");
|
||||
});
|
||||
|
||||
it("builds router input contract for v1", () => {
|
||||
const routerInput = toRouterInput(normalizedFixture());
|
||||
expect(routerInput.route_hint).toBe("hybrid_store_plus_live");
|
||||
expect(routerInput.intent_class).toBe("cross_entity");
|
||||
});
|
||||
|
||||
it("keeps v2.0.1 soft assumptions executable in deterministic routing", () => {
|
||||
const summary = toRouteHintSummary(normalizedFixtureV2_0_1());
|
||||
expect(summary.mode).toBe("deterministic_v2");
|
||||
if (summary.mode !== "deterministic_v2") {
|
||||
throw new Error("Expected deterministic_v2 summary");
|
||||
}
|
||||
expect(summary.fallback.type).toBe("none");
|
||||
expect(summary.decisions[0]?.execution_readiness).toBe("executable_with_soft_assumptions");
|
||||
expect(summary.decisions[0]?.route).toBe("store_feature_risk");
|
||||
});
|
||||
|
||||
it("uses explicit v2.0.2 route_status/no_route_reason contract", () => {
|
||||
const summary = toRouteHintSummary(normalizedFixtureV2_0_2());
|
||||
expect(summary.mode).toBe("deterministic_v2");
|
||||
if (summary.mode !== "deterministic_v2") {
|
||||
throw new Error("Expected deterministic_v2 summary");
|
||||
}
|
||||
expect(summary.decisions[0]?.route_status).toBe("routed");
|
||||
expect(summary.decisions[0]?.no_route_reason).toBeNull();
|
||||
expect(summary.decisions[0]?.route).toBe("store_feature_risk");
|
||||
|
||||
const routerInput = toRouterInput(normalizedFixtureV2_0_2());
|
||||
const first = (routerInput.fragments as Array<Record<string, unknown>>)[0];
|
||||
expect(first.route_status).toBe("routed");
|
||||
expect(first.no_route_reason).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { validateNormalized } from "../src/services/schemaValidator";
|
||||
import { normalizedFixture, normalizedFixtureV2, normalizedFixtureV2_0_1, normalizedFixtureV2_0_2 } from "./fixtures";
|
||||
|
||||
describe("schemaValidator", () => {
|
||||
it("passes valid normalized payload", () => {
|
||||
const result = validateNormalized(normalizedFixture(), "v1");
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("fails invalid payload", () => {
|
||||
const invalid = { ...normalizedFixture(), route_hint: "unknown_route" };
|
||||
const result = validateNormalized(invalid, "v1");
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.errors.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("passes valid normalized v2 payload", () => {
|
||||
const result = validateNormalized(normalizedFixtureV2(), "v2");
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("fails invalid v2 payload", () => {
|
||||
const invalid = { ...normalizedFixtureV2(), schema_version: "normalized_query_v1" };
|
||||
const result = validateNormalized(invalid, "v2");
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.errors.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("passes valid normalized v2.0.1 payload", () => {
|
||||
const result = validateNormalized(normalizedFixtureV2_0_1(), "v2_0_1");
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("passes valid normalized v2.0.2 payload", () => {
|
||||
const result = validateNormalized(normalizedFixtureV2_0_2(), "v2_0_2");
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AssistantSessionStore } from "../src/services/assistantSessionStore";
|
||||
|
||||
describe("assistant session backward compatibility", () => {
|
||||
it("lazy-upgrades legacy session objects without investigation_state", () => {
|
||||
const store = new AssistantSessionStore();
|
||||
const sessionsMap = (store as unknown as { sessions: Map<string, unknown> }).sessions;
|
||||
const sessionId = "legacy-session-1";
|
||||
|
||||
sessionsMap.set(sessionId, {
|
||||
session_id: sessionId,
|
||||
updated_at: "2026-03-25T10:00:00.000Z",
|
||||
items: []
|
||||
});
|
||||
|
||||
const session = store.getSession(sessionId);
|
||||
expect(session).toBeTruthy();
|
||||
expect(session?.session_id).toBe(sessionId);
|
||||
expect(Array.isArray(session?.items)).toBe(true);
|
||||
expect(session?.investigation_state?.schema_version).toBe("investigation_state_v1");
|
||||
});
|
||||
|
||||
it("normalizes malformed legacy sessions with missing items array", () => {
|
||||
const store = new AssistantSessionStore();
|
||||
const sessionsMap = (store as unknown as { sessions: Map<string, unknown> }).sessions;
|
||||
const sessionId = "legacy-session-2";
|
||||
|
||||
sessionsMap.set(sessionId, {
|
||||
session_id: sessionId
|
||||
});
|
||||
|
||||
const ensured = store.ensureSession(sessionId);
|
||||
expect(ensured.session_id).toBe(sessionId);
|
||||
expect(Array.isArray(ensured.items)).toBe(true);
|
||||
expect(ensured.items.length).toBe(0);
|
||||
expect(ensured.investigation_state?.schema_version).toBe("investigation_state_v1");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user