Этап 4 / Волна 10: корректировка settlement-кейса — доменная фиксация синтеза, честное покрытие, удержание фокуса / Этап 4 / Волна 11: бизнес-якоря, доменное заземление и устранение утечки дебага
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { AssistantDataLayer } from "../src/services/assistantDataLayer.ts";
|
||||
|
||||
type DomainPrefix = "SET" | "VAT" | "CLS";
|
||||
interface RegressionCase {
|
||||
case_id: string;
|
||||
expected_prefix: DomainPrefix;
|
||||
query: string;
|
||||
}
|
||||
|
||||
function buildRecord(input: {
|
||||
id: string;
|
||||
account: string;
|
||||
period: string;
|
||||
description: string;
|
||||
unknownLinks?: number;
|
||||
withCounterparty?: boolean;
|
||||
zeroGuid?: boolean;
|
||||
}): Record<string, unknown> {
|
||||
const attributes: Record<string, unknown> = {
|
||||
Recorder: `${input.id}-REC`,
|
||||
Period: input.period,
|
||||
Description: input.description,
|
||||
Account: input.account,
|
||||
"trace@navigationLinkUrl": `/trace/${input.id}`
|
||||
};
|
||||
if (input.zeroGuid) {
|
||||
attributes.LinkGuid = "00000000-0000-0000-0000-000000000000";
|
||||
}
|
||||
|
||||
const links: Array<Record<string, unknown>> = [
|
||||
{
|
||||
relation: "document_refers_to_document",
|
||||
target_entity: "Document",
|
||||
target_id: `${input.id}-DOC-LINK`,
|
||||
source_field: "Recorder"
|
||||
}
|
||||
];
|
||||
if (input.withCounterparty !== false) {
|
||||
links.push({
|
||||
relation: "document_has_counterparty",
|
||||
target_entity: "Counterparty",
|
||||
target_id: `${input.id}-CP`,
|
||||
source_field: "Counterparty"
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
source_entity: "Document",
|
||||
source_id: input.id,
|
||||
display_name: input.id,
|
||||
unknown_link_count: input.unknownLinks ?? 1,
|
||||
problem_flags: ["risk_marker"],
|
||||
attributes,
|
||||
links
|
||||
};
|
||||
}
|
||||
|
||||
function createDataset() {
|
||||
const settlements = [
|
||||
buildRecord({ id: "SET-PC-1", account: "60", period: "2020-06-10T00:00:00", description: "supplier payment recorded but settlement chain is still open account 60" }),
|
||||
buildRecord({ id: "SET-PC-2", account: "62", period: "2020-06-11T00:00:00", description: "customer settlement tail payment to settlement relation broken account 62" }),
|
||||
buildRecord({ id: "SET-DOC-1", account: "60", period: "2020-06-20T00:00:00", description: "bank statement linked to settlement document payment chain account 60" }),
|
||||
buildRecord({ id: "SET-DOC-2", account: "62", period: "2020-06-21T00:00:00", description: "customer payment linked to settlement closure account 62" }),
|
||||
buildRecord({ id: "SET-KF-1", account: "60", period: "2020-06-22T00:00:00", description: "settlement key field record account 60 payment" })
|
||||
];
|
||||
|
||||
const vat = [
|
||||
buildRecord({ id: "VAT-PC-1", account: "68", period: "2020-06-12T00:00:00", description: "vat invoice linked to register and purchase book account 68" }),
|
||||
buildRecord({ id: "VAT-PC-2", account: "19", period: "2020-06-13T00:00:00", description: "vat source document present but invoice to vat link is broken account 19" }),
|
||||
buildRecord({ id: "VAT-NDS-1", account: "68", period: "2020-06-23T00:00:00", description: "vat register entry book generation deduction posted" }),
|
||||
buildRecord({ id: "VAT-NDS-2", account: "19", period: "2020-06-24T00:00:00", description: "invoice to vat register chain for deduction account 19" }),
|
||||
buildRecord({ id: "VAT-KF-1", account: "68", period: "2020-06-25T00:00:00", description: "vat key field invoice register linkage account 68" })
|
||||
];
|
||||
|
||||
const close = [
|
||||
buildRecord({ id: "CLS-PC-1", account: "20", period: "2020-06-14T00:00:00", description: "period close costs accumulated but allocation rules unresolved account 20" }),
|
||||
buildRecord({ id: "CLS-PC-2", account: "44", period: "2020-06-15T00:00:00", description: "month close operation runs with residuals not zero account 44" }),
|
||||
buildRecord({ id: "CLS-DOC-1", account: "20", period: "2020-06-26T00:00:00", description: "period close costs allocation writeoff account 20" }),
|
||||
buildRecord({ id: "CLS-DOC-2", account: "44", period: "2020-06-27T00:00:00", description: "month close residuals explained allocation account 44" }),
|
||||
buildRecord({ id: "CLS-KF-1", account: "20", period: "2020-06-28T00:00:00", description: "period close key field account 20 allocation" })
|
||||
];
|
||||
|
||||
const mixed = [
|
||||
buildRecord({ id: "MIX-PC-1", account: "68", period: "2020-12-31T00:00:00", description: "bank settlement vat mixed conflict record", zeroGuid: true }),
|
||||
buildRecord({ id: "MIX-NDS-1", account: "60", period: "2020-12-30T00:00:00", description: "mixed nds and settlement overlap record", zeroGuid: true }),
|
||||
buildRecord({ id: "MIX-DOC-1", account: "68", period: "2020-12-29T00:00:00", description: "mixed document with vat settlement and bank signals", zeroGuid: true }),
|
||||
buildRecord({ id: "MIX-KF-1", account: "44", period: "2020-12-28T00:00:00", description: "mixed key field with period close and vat overlap", zeroGuid: true })
|
||||
];
|
||||
|
||||
return {
|
||||
keyFields: [settlements[4], vat[4], close[4], mixed[3]],
|
||||
problemCases: [mixed[0], vat[0], settlements[0], close[0], settlements[1], vat[1], close[1]],
|
||||
journals: [close[2], close[3], settlements[3]],
|
||||
ndsRegisters: [mixed[1], vat[2], vat[3]],
|
||||
docs: [mixed[2], settlements[2], settlements[3], vat[2], vat[3], close[2], close[3]]
|
||||
};
|
||||
}
|
||||
|
||||
function createSnapshotRoot(dataset: any): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "assistant-wave5-debug-"));
|
||||
const write = (fileName: string, records: Array<Record<string, unknown>>) => {
|
||||
fs.writeFileSync(path.resolve(root, fileName), JSON.stringify({ records }, null, 2), "utf-8");
|
||||
};
|
||||
write("09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json", dataset.keyFields);
|
||||
write("03_snapshot_fragment_problem_cases.json", dataset.problemCases);
|
||||
write("07_samples_DocumentJournals.json", dataset.journals);
|
||||
write("08_samples_NDS_registers.json", dataset.ndsRegisters);
|
||||
write("04_samples_SpisanieSRaschetnogoScheta.json", dataset.docs);
|
||||
write("05_samples_RealizaciyaTovarovUslug.json", []);
|
||||
write("06_samples_PostuplenieTovarovUslug.json", []);
|
||||
return root;
|
||||
}
|
||||
|
||||
function resolvePrefixFromId(sourceId: string): DomainPrefix | "OTHER" {
|
||||
if (sourceId.startsWith("SET")) return "SET";
|
||||
if (sourceId.startsWith("VAT")) return "VAT";
|
||||
if (sourceId.startsWith("CLS")) return "CLS";
|
||||
return "OTHER";
|
||||
}
|
||||
|
||||
const SETTLEMENT_QUERIES = [
|
||||
"Show why payment recorded but settlement for account 60 is still open.",
|
||||
"Account 62: payment posted, settlement closure is missing.",
|
||||
"Find settlement tails for account 60 where payment did not close chain.",
|
||||
"Bank and settlements 60/62: where link to settlement is broken.",
|
||||
"Why does account 60 keep open settlement after payment record.",
|
||||
"Account 62 settlement problem: payment done, closure not reached.",
|
||||
"Detect symptom where payment exists but settlement remains open on 60.",
|
||||
"Find lifecycle gap in payment to settlement for account 62.",
|
||||
"60-62 settlement chain has residual tail after payment.",
|
||||
"Locate unresolved settlement after bank payment on account 60."
|
||||
];
|
||||
|
||||
const VAT_QUERIES = [
|
||||
"VAT check: source document exists but invoice link is missing on account 68.",
|
||||
"Account 19 VAT chain: document to register to book is broken.",
|
||||
"Find VAT symptom where invoice linked but book entry was not generated.",
|
||||
"Show VAT lifecycle gaps for account 68 in document-register-book flow.",
|
||||
"VAT deduction issue on 19: source document present but deduction not posted.",
|
||||
"Find broken invoice to VAT register relation for account 68.",
|
||||
"VAT problem-first: document exists, register is present, book entry missing.",
|
||||
"Locate VAT residual issue where deduction chain is incomplete on 19.",
|
||||
"VAT 68: invoice and register mismatch in purchase/sales book.",
|
||||
"Detect VAT symptom with broken doc-register-book chain for account 68."
|
||||
];
|
||||
|
||||
const CLOSE_QUERIES = [
|
||||
"Month close: costs on accounts 20 and 44 are not allocated, residuals remain.",
|
||||
"Period close problem for 20/44: allocation rules unresolved.",
|
||||
"Find close lifecycle gap where costs accumulated but close operation fails 20 44.",
|
||||
"Account 20 and 44 month close symptom: residuals are not zero.",
|
||||
"Show period close issue when costs are accumulated but not distributed 20/44.",
|
||||
"Close operation run for 20 and 44 leaves unexplained residuals.",
|
||||
"Detect month close break in costs allocation chain on 20/44.",
|
||||
"Period close 20-44: allocation exists but residual tail remains.",
|
||||
"Find cost close mismatch: costs accumulated, close not completed 20 and 44.",
|
||||
"Month close domain check for accounts 20 and 44 with unresolved residuals."
|
||||
];
|
||||
|
||||
const cases: RegressionCase[] = [
|
||||
...SETTLEMENT_QUERIES.map((query, idx) => ({ case_id: `SET-${String(idx + 1).padStart(2, "0")}`, expected_prefix: "SET" as const, query })),
|
||||
...VAT_QUERIES.map((query, idx) => ({ case_id: `VAT-${String(idx + 1).padStart(2, "0")}`, expected_prefix: "VAT" as const, query })),
|
||||
...CLOSE_QUERIES.map((query, idx) => ({ case_id: `CLS-${String(idx + 1).padStart(2, "0")}`, expected_prefix: "CLS" as const, query }))
|
||||
];
|
||||
|
||||
const root = createSnapshotRoot(createDataset());
|
||||
const layer = new AssistantDataLayer(root);
|
||||
|
||||
let bad = 0;
|
||||
for (const c of cases) {
|
||||
const r = layer.executeRoute("store_feature_risk", c.query);
|
||||
const ids = (r.items as Array<any>).map((x) => String(x.source_id ?? ""));
|
||||
const top1 = ids[0] ?? "<empty>";
|
||||
const ok = top1 !== "<empty>" && resolvePrefixFromId(top1) === c.expected_prefix;
|
||||
if (!ok) {
|
||||
bad += 1;
|
||||
const guard = (r.summary as any)?.domain_purity_guard;
|
||||
console.log(`${c.case_id} FAIL top1=${top1} expected=${c.expected_prefix} status=${r.status}`);
|
||||
console.log(` query=${c.query}`);
|
||||
console.log(` ids=${ids.join(",")}`);
|
||||
console.log(` card=${guard?.domain_card_id} source_allowed=${guard?.source_selection_allowed} ranking_allowed=${guard?.ranking_allowed} promotion_allowed=${guard?.promotion_allowed}`);
|
||||
}
|
||||
}
|
||||
console.log(`bad=${bad}`);
|
||||
@@ -0,0 +1,67 @@
|
||||
import path from "path";
|
||||
import request from "supertest";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const originalLog = console.log;
|
||||
console.log = () => {};
|
||||
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = "1";
|
||||
|
||||
const { createApp } = await import("../src/server.ts");
|
||||
const app = createApp();
|
||||
|
||||
const baselinePath = path.resolve(
|
||||
process.cwd(),
|
||||
"../docs/runs/2026-03-27_Stage_04_Wave_07_P0_Eval_Harness_Formal_Product_Acceptance/artifacts/current_report.json"
|
||||
);
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_1.json",
|
||||
compare_with_report_file: baselinePath,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
console.log = originalLog;
|
||||
|
||||
if (response.status !== 200) {
|
||||
console.error(`status=${response.status}`);
|
||||
console.error(JSON.stringify(response.body, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const report = response.body.report;
|
||||
originalLog(
|
||||
JSON.stringify(
|
||||
{
|
||||
run_id: report?.run_id,
|
||||
verdict: report?.acceptance_gate?.verdict,
|
||||
metrics: report?.metrics?.raw,
|
||||
assertions: report?.assertions,
|
||||
run_report_json_path: report?.artifacts?.run_report_json_path,
|
||||
run_report_md_path: report?.artifacts?.run_report_md_path,
|
||||
comparison_json_path: report?.comparison?.artifacts?.comparison_report_json_path,
|
||||
comparison_md_path: report?.comparison?.artifacts?.comparison_report_md_path,
|
||||
comparison_verdict_delta: report?.comparison?.verdict_delta
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import path from "path";
|
||||
import request from "supertest";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const originalLog = console.log;
|
||||
console.log = () => {};
|
||||
|
||||
process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
|
||||
process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = "1";
|
||||
process.env.FEATURE_ASSISTANT_GRAPH_RUNTIME_V1 = "1";
|
||||
|
||||
const { createApp } = await import("../src/server.ts");
|
||||
const app = createApp();
|
||||
|
||||
const baselinePath = path.resolve(
|
||||
process.cwd(),
|
||||
"../docs/runs/2026-03-27_Stage_04_Wave_08_Route_Correctness_Recovery_Domain_Purity_Closure/artifacts/current_report.json"
|
||||
);
|
||||
|
||||
const response = await request(app).post("/api/eval/run").send({
|
||||
eval_target: "assistant_p0",
|
||||
useMock: true,
|
||||
mode: "single-pass-strict",
|
||||
caseSetFile: "p0_eval_corpus_v0_2.json",
|
||||
compare_with_report_file: baselinePath,
|
||||
normalizeConfig: {
|
||||
promptVersion: "normalizer_v2_0_2"
|
||||
}
|
||||
});
|
||||
|
||||
console.log = originalLog;
|
||||
|
||||
if (response.status !== 200) {
|
||||
console.error(`status=${response.status}`);
|
||||
console.error(JSON.stringify(response.body, null, 2));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const report = response.body.report;
|
||||
originalLog(
|
||||
JSON.stringify(
|
||||
{
|
||||
run_id: report?.run_id,
|
||||
acceptance_verdict: report?.acceptance_gate?.verdict,
|
||||
baseline_stability_verdict: report?.baseline_stability_gate?.verdict,
|
||||
metrics: report?.metrics?.raw,
|
||||
quality_gap_metrics: report?.quality_gap_metrics?.raw,
|
||||
assertions: report?.assertions,
|
||||
run_report_json_path: report?.artifacts?.run_report_json_path,
|
||||
run_report_md_path: report?.artifacts?.run_report_md_path,
|
||||
comparison_json_path: report?.comparison?.artifacts?.comparison_report_json_path,
|
||||
comparison_md_path: report?.comparison?.artifacts?.comparison_report_md_path,
|
||||
comparison_verdict_delta: report?.comparison?.verdict_delta
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user