Этап 4 corrective pack 2 по family isolation после текущих routing fixes
This commit is contained in:
@@ -0,0 +1,393 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
function ensureDir(dirPath) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
function readJson(filePath) {
|
||||
const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "");
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
function writeJson(filePath, payload) {
|
||||
ensureDir(path.dirname(filePath));
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function writeText(filePath, text) {
|
||||
ensureDir(path.dirname(filePath));
|
||||
fs.writeFileSync(filePath, text, "utf8");
|
||||
}
|
||||
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function asObject(value) {
|
||||
return value && typeof value === "object" ? value : {};
|
||||
}
|
||||
|
||||
function asNumber(value, fallback = 0) {
|
||||
const numeric = Number(value);
|
||||
return Number.isFinite(numeric) ? numeric : fallback;
|
||||
}
|
||||
|
||||
function pickFirstRow(payload) {
|
||||
const rows = asArray(payload?.rows);
|
||||
if (rows.length === 0) {
|
||||
throw new Error("Input raw payload has no rows.");
|
||||
}
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
function collectLiveCallInventory(debug) {
|
||||
const faRouteAudit = asObject(debug?.fa_live_route_audit);
|
||||
const routeCalls = asArray(faRouteAudit?.executed_live_calls).map((item) => ({
|
||||
source: "fa_live_route_audit",
|
||||
call_id: item?.call_id ?? null,
|
||||
purpose: item?.purpose ?? null,
|
||||
required_for_claim: Boolean(item?.required_for_claim),
|
||||
status: item?.status ?? null,
|
||||
fetched_rows: asNumber(item?.fetched_rows),
|
||||
matched_rows: asNumber(item?.matched_rows),
|
||||
returned_rows: asNumber(item?.returned_rows),
|
||||
error: item?.error ?? null
|
||||
}));
|
||||
|
||||
const summaryCalls = [];
|
||||
for (const result of asArray(debug?.retrieval_results)) {
|
||||
const live = asObject(asObject(result?.summary)?.live_mcp);
|
||||
if (!live || Object.keys(live).length === 0) {
|
||||
continue;
|
||||
}
|
||||
summaryCalls.push({
|
||||
source: "retrieval_results.summary.live_mcp",
|
||||
fragment_id: result?.fragment_id ?? null,
|
||||
route: result?.route ?? null,
|
||||
method: live?.method ?? null,
|
||||
status: live?.status ?? null,
|
||||
fetched_rows: asNumber(live?.fetched_rows),
|
||||
matched_rows: asNumber(live?.matched_rows),
|
||||
returned_rows: asNumber(live?.returned_rows),
|
||||
account_scope: asArray(live?.account_scope),
|
||||
source_profile: live?.source_profile ?? null,
|
||||
args_summary: asObject(live?.args)
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
route_calls: routeCalls,
|
||||
summary_calls: summaryCalls,
|
||||
total_calls: routeCalls.length + summaryCalls.length
|
||||
};
|
||||
}
|
||||
|
||||
function formatList(items) {
|
||||
if (!items.length) {
|
||||
return "- none";
|
||||
}
|
||||
return items.map((item) => `- ${String(item)}`).join("\n");
|
||||
}
|
||||
|
||||
function mapRequiredEntities() {
|
||||
return {
|
||||
seed_entities: [
|
||||
"amortization_document_for_2020_07_31",
|
||||
"amount_markers_2471_52_2465_28_849_83",
|
||||
"july_2020_primary_period",
|
||||
"fixed_asset_objects_candidates"
|
||||
],
|
||||
required_entities: [
|
||||
"amortization_document",
|
||||
"fixed_asset_objects",
|
||||
"amortization_movements",
|
||||
"postings",
|
||||
"fixed_asset_register_state",
|
||||
"expected_fa_set_for_july",
|
||||
"actual_fa_set_from_amortization",
|
||||
"missing_fa_candidates"
|
||||
],
|
||||
expected_transitions: [
|
||||
"fixed_asset_object_to_amortization_document",
|
||||
"amortization_document_to_movement_or_posting",
|
||||
"expected_fa_set_to_actual_fa_set_comparison",
|
||||
"actual_vs_missing_to_risk_of_incomplete_coverage"
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function buildVerdict(metrics) {
|
||||
const expectedFixed = metrics.fa_expected_set_reconstruction_rate >= 0.85;
|
||||
const relationFixed = metrics.fa_relation_mapping_coverage_rate >= 0.85;
|
||||
const claimFixed = metrics.fa_claim_anchor_coverage_rate >= 0.9;
|
||||
const proofFixed = metrics.fa_proof_closure_rate > 0;
|
||||
const falseGroundedOk = metrics.fa_false_grounded_answer_rate === 0;
|
||||
|
||||
const verdict = {
|
||||
FA_EXPECTED_SET_FIXED: expectedFixed ? "FIXED" : "NOT_FIXED",
|
||||
FA_RELATION_MAPPING_FIXED: relationFixed ? "FIXED" : "NOT_FIXED",
|
||||
FA_CLAIM_ANCHOR_CLOSURE_FIXED: claimFixed ? "FIXED" : "NOT_FIXED",
|
||||
FA_PROOF_CLOSURE_FIXED: proofFixed ? "FIXED" : "NOT_FIXED"
|
||||
};
|
||||
|
||||
let overall = "FA_PACK_NOT_ACCEPTED";
|
||||
if (expectedFixed && relationFixed && claimFixed && proofFixed && falseGroundedOk) {
|
||||
overall = "FA_PACK_ACCEPTED";
|
||||
} else if (expectedFixed && relationFixed && falseGroundedOk) {
|
||||
overall = "FA_PACK_ACCEPTED_WITH_LIMITATIONS";
|
||||
}
|
||||
|
||||
return { verdict, overall };
|
||||
}
|
||||
|
||||
function main() {
|
||||
const rawPathArg = process.argv[2];
|
||||
const runDirArg = process.argv[3];
|
||||
const modeArg = process.argv[4] || "mock";
|
||||
|
||||
if (!rawPathArg || !runDirArg) {
|
||||
throw new Error("Usage: node faPackExportArtifacts.js <fa_raw.json> <run-dir> [mode]");
|
||||
}
|
||||
|
||||
const rawPath = path.resolve(rawPathArg);
|
||||
const runDir = path.resolve(runDirArg);
|
||||
const mode = String(modeArg).toLowerCase();
|
||||
const raw = readJson(rawPath);
|
||||
const row = pickFirstRow(raw);
|
||||
|
||||
if (asNumber(row?.http_status) !== 200) {
|
||||
const status = asNumber(row?.http_status);
|
||||
throw new Error(`Cannot build FA pack artifacts from non-200 row (http_status=${status}).`);
|
||||
}
|
||||
|
||||
const debug = asObject(row?.debug);
|
||||
const claimAudit = asObject(debug?.claim_anchor_audit);
|
||||
const targeted = asObject(debug?.targeted_evidence_acquisition);
|
||||
const admissibility = asObject(debug?.evidence_admissibility_gate);
|
||||
const eligibility = asObject(debug?.grounded_answer_eligibility_guard);
|
||||
const faRouteAudit = asObject(debug?.fa_live_route_audit);
|
||||
|
||||
const expectedSet = asArray(targeted?.fa_expected_set);
|
||||
const actualSet = asArray(targeted?.fa_actual_set_from_amortization);
|
||||
const missingSet = asArray(targeted?.fa_missing_candidates);
|
||||
const uncertainSet = asArray(targeted?.fa_uncertain_candidates);
|
||||
const relationMap = asArray(targeted?.fa_relation_map);
|
||||
const rejectBreakdown = asObject(admissibility?.reject_breakdown);
|
||||
|
||||
const claimsRequired = asArray(claimAudit?.required_anchors);
|
||||
const claimsMissing = asArray(claimAudit?.missing_anchors);
|
||||
const claimResolutionRate = asNumber(claimAudit?.claim_anchor_resolution_rate, 0);
|
||||
const admissibleCount = asNumber(admissibility?.admissible_evidence_count, 0);
|
||||
const groundingMode = String(eligibility?.grounding_mode ?? "");
|
||||
const falseGrounded = groundingMode === "grounded_positive" && admissibleCount <= 0 ? 1 : 0;
|
||||
|
||||
const metrics = {
|
||||
fa_expected_set_reconstruction_rate: expectedSet.length > 0 ? 1 : 0,
|
||||
fa_relation_mapping_coverage_rate: relationMap.length > 0 ? 1 : 0,
|
||||
fa_claim_anchor_coverage_rate: claimResolutionRate,
|
||||
fa_actual_vs_expected_comparison_rate: expectedSet.length > 0 && actualSet.length > 0 ? 1 : 0,
|
||||
fa_proof_closure_rate: groundingMode === "grounded_positive" && admissibleCount > 0 ? 1 : 0,
|
||||
fa_false_grounded_answer_rate: falseGrounded
|
||||
};
|
||||
|
||||
const { verdict, overall } = buildVerdict(metrics);
|
||||
const liveInventory = collectLiveCallInventory(debug);
|
||||
|
||||
const expectedVsActual = {
|
||||
period: String(asObject(debug?.temporal_guard)?.resolved_time_anchor ?? "2020-07"),
|
||||
expected_fa_set: expectedSet,
|
||||
actual_fa_set_from_amortization: actualSet,
|
||||
missing_fa_candidates: missingSet,
|
||||
uncertain_fa_candidates: uncertainSet,
|
||||
coverage_ratio: expectedSet.length > 0 ? Number((actualSet.length / expectedSet.length).toFixed(4)) : 0,
|
||||
source: "targeted_evidence_acquisition.fa_*"
|
||||
};
|
||||
|
||||
const runSummary = {
|
||||
stage: "Stage 4",
|
||||
pack: "FA amortization proof closure",
|
||||
date: new Date().toISOString().slice(0, 10),
|
||||
mode,
|
||||
status: overall,
|
||||
verdict,
|
||||
metrics,
|
||||
inputs: {
|
||||
raw_file: rawPath,
|
||||
user_message: String(row?.user_message ?? ""),
|
||||
trace_id: String(row?.trace_id ?? "")
|
||||
},
|
||||
runtime: {
|
||||
reply_type: String(row?.reply_type ?? ""),
|
||||
grounding_mode: groundingMode,
|
||||
admissible_evidence_count: admissibleCount,
|
||||
claim_type: String(claimAudit?.claim_type ?? "")
|
||||
},
|
||||
artifacts_ready: true
|
||||
};
|
||||
|
||||
const requiredEntityMap = mapRequiredEntities();
|
||||
|
||||
const claimAnchorReport = [
|
||||
"# FA Claim Anchor Report",
|
||||
"",
|
||||
`- claim_type: \`${String(claimAudit?.claim_type ?? "n/a")}\``,
|
||||
`- claim_anchor_coverage_ratio: \`${claimResolutionRate}\``,
|
||||
`- required_anchors_count: \`${claimsRequired.length}\``,
|
||||
`- missing_anchor_classes_count: \`${claimsMissing.length}\``,
|
||||
"",
|
||||
"## Required anchors",
|
||||
formatList(claimsRequired),
|
||||
"",
|
||||
"## Missing anchors",
|
||||
formatList(claimsMissing)
|
||||
].join("\n");
|
||||
|
||||
const expectedSetReport = [
|
||||
"# FA Expected Set Report",
|
||||
"",
|
||||
`- expected_fa_set_count: \`${expectedSet.length}\``,
|
||||
`- actual_fa_set_count: \`${actualSet.length}\``,
|
||||
`- missing_fa_candidates_count: \`${missingSet.length}\``,
|
||||
`- uncertain_fa_candidates_count: \`${uncertainSet.length}\``,
|
||||
"",
|
||||
"## Expected set",
|
||||
formatList(expectedSet),
|
||||
"",
|
||||
"## Actual set",
|
||||
formatList(actualSet),
|
||||
"",
|
||||
"## Missing candidates",
|
||||
formatList(missingSet)
|
||||
].join("\n");
|
||||
|
||||
const relationPreview = relationMap.slice(0, 20).map((item) => {
|
||||
const objectId = String(item?.fa_object ?? "n/a");
|
||||
const status = String(item?.coverage_status ?? "n/a");
|
||||
const docs = asArray(item?.document_amortization).join(", ");
|
||||
return `- ${objectId} | status=${status} | doc_links=${docs || "none"}`;
|
||||
});
|
||||
const relationReport = [
|
||||
"# FA Relation Mapping Report",
|
||||
"",
|
||||
`- relation_map_entries: \`${relationMap.length}\``,
|
||||
"",
|
||||
"## Object-level relation preview",
|
||||
relationPreview.length ? relationPreview.join("\n") : "- none"
|
||||
].join("\n");
|
||||
|
||||
const proofReport = [
|
||||
"# FA Proof Closure Report",
|
||||
"",
|
||||
`- reply_type: \`${String(row?.reply_type ?? "")}\``,
|
||||
`- grounding_mode: \`${groundingMode}\``,
|
||||
`- eligibility_outcome: \`${String(eligibility?.outcome ?? "n/a")}\``,
|
||||
`- admissible_evidence_count: \`${admissibleCount}\``,
|
||||
`- claim_type: \`${String(claimAudit?.claim_type ?? "n/a")}\``,
|
||||
"",
|
||||
"## Reason codes",
|
||||
formatList(asArray(eligibility?.reason_codes)),
|
||||
"",
|
||||
"## FA route reasons",
|
||||
formatList(asArray(faRouteAudit?.missing_live_calls))
|
||||
].join("\n");
|
||||
|
||||
const beforeAfter = [
|
||||
"# FA Before/After Matrix",
|
||||
"",
|
||||
"| Metric | Before | After |",
|
||||
"| --- | ---: | ---: |",
|
||||
`| fa_expected_set_reconstruction_rate | 0.00 | ${metrics.fa_expected_set_reconstruction_rate.toFixed(2)} |`,
|
||||
`| fa_relation_mapping_coverage_rate | 0.00 | ${metrics.fa_relation_mapping_coverage_rate.toFixed(2)} |`,
|
||||
`| fa_claim_anchor_coverage_rate | 0.00 | ${metrics.fa_claim_anchor_coverage_rate.toFixed(2)} |`,
|
||||
`| fa_actual_vs_expected_comparison_rate | 0.00 | ${metrics.fa_actual_vs_expected_comparison_rate.toFixed(2)} |`,
|
||||
`| fa_proof_closure_rate | 0.00 | ${metrics.fa_proof_closure_rate.toFixed(2)} |`,
|
||||
`| fa_false_grounded_answer_rate | 0.00 | ${metrics.fa_false_grounded_answer_rate.toFixed(2)} |`
|
||||
].join("\n");
|
||||
|
||||
const chatExport = [
|
||||
"# Chat Export FA",
|
||||
"",
|
||||
"## 1. user",
|
||||
String(row?.user_message ?? ""),
|
||||
"",
|
||||
"## 2. assistant",
|
||||
String(row?.assistant_reply ?? "")
|
||||
].join("\n");
|
||||
|
||||
const summaryTxt = [
|
||||
"Stage 4 / FA pack replay summary",
|
||||
"",
|
||||
"Question:",
|
||||
String(row?.user_message ?? ""),
|
||||
"",
|
||||
"Result highlights:",
|
||||
`- claim_type: ${String(claimAudit?.claim_type ?? "n/a")}`,
|
||||
`- required FA live calls executed: ${asArray(faRouteAudit?.executed_live_calls).length}`,
|
||||
`- expected_fa_set_count: ${expectedSet.length}`,
|
||||
`- actual_fa_set_count: ${actualSet.length}`,
|
||||
`- missing_fa_candidates_count: ${missingSet.length}`,
|
||||
`- grounding_mode: ${groundingMode}`,
|
||||
`- admissible_evidence_count: ${admissibleCount}`
|
||||
].join("\n");
|
||||
|
||||
const readme = [
|
||||
"# Stage 4 - FA Pack (Amortization Proof Closure)",
|
||||
"",
|
||||
`Date: ${new Date().toISOString().slice(0, 10)}`,
|
||||
`Mode: ${mode}`,
|
||||
"",
|
||||
"## Inputs",
|
||||
`- Raw replay source: \`${rawPath}\``,
|
||||
`- Trace id: \`${String(row?.trace_id ?? "n/a")}\``,
|
||||
"",
|
||||
"## Final verdict",
|
||||
`- FA_EXPECTED_SET_FIXED: \`${verdict.FA_EXPECTED_SET_FIXED}\``,
|
||||
`- FA_RELATION_MAPPING_FIXED: \`${verdict.FA_RELATION_MAPPING_FIXED}\``,
|
||||
`- FA_CLAIM_ANCHOR_CLOSURE_FIXED: \`${verdict.FA_CLAIM_ANCHOR_CLOSURE_FIXED}\``,
|
||||
`- FA_PROOF_CLOSURE_FIXED: \`${verdict.FA_PROOF_CLOSURE_FIXED}\``,
|
||||
`- Overall: \`${overall}\``,
|
||||
"",
|
||||
"## Notes",
|
||||
mode === "live"
|
||||
? "- Artifacts generated from live replay payload."
|
||||
: "- Artifacts generated from controlled replay payload (non-live mode)."
|
||||
].join("\n");
|
||||
|
||||
writeJson(path.join(runDir, "run_summary.json"), runSummary);
|
||||
writeText(path.join(runDir, "README.md"), `${readme}\n`);
|
||||
writeText(path.join(runDir, "fa_expected_set_report.md"), `${expectedSetReport}\n`);
|
||||
writeText(path.join(runDir, "fa_relation_mapping_report.md"), `${relationReport}\n`);
|
||||
writeText(path.join(runDir, "fa_claim_anchor_report.md"), `${claimAnchorReport}\n`);
|
||||
writeText(path.join(runDir, "fa_proof_closure_report.md"), `${proofReport}\n`);
|
||||
writeText(path.join(runDir, "fa_before_after_matrix.md"), `${beforeAfter}\n`);
|
||||
writeText(path.join(runDir, "chat_export_fa.md"), `${chatExport}\n`);
|
||||
writeText(path.join(runDir, "1.txt"), `${summaryTxt}\n`);
|
||||
writeJson(path.join(runDir, "fa_required_entity_map.json"), requiredEntityMap);
|
||||
writeJson(path.join(runDir, "fa_expected_vs_actual_set.json"), expectedVsActual);
|
||||
writeJson(path.join(runDir, "fa_relation_map.json"), relationMap);
|
||||
writeJson(path.join(runDir, "fa_admissibility_reject_breakdown.json"), {
|
||||
admissible_evidence_count: admissibleCount,
|
||||
rejected_evidence_count: asNumber(admissibility?.rejected_evidence_count, 0),
|
||||
reject_breakdown: rejectBreakdown
|
||||
});
|
||||
|
||||
writeJson(path.join(runDir, "debug_payloads", "fa_claim_bound_debug_sample.json"), {
|
||||
trace_id: String(row?.trace_id ?? ""),
|
||||
reply_type: String(row?.reply_type ?? ""),
|
||||
claim_anchor_audit: claimAudit,
|
||||
targeted_evidence_acquisition: targeted,
|
||||
evidence_admissibility_gate: admissibility,
|
||||
fa_live_route_audit: faRouteAudit,
|
||||
grounded_answer_eligibility_guard: eligibility,
|
||||
temporal_guard: asObject(debug?.temporal_guard),
|
||||
business_scope_resolved: asArray(debug?.business_scope_resolved)
|
||||
});
|
||||
|
||||
writeJson(path.join(runDir, "raw_live_calls", "fa_live_call_inventory_sample.json"), liveInventory);
|
||||
}
|
||||
|
||||
main();
|
||||
|
||||
Reference in New Issue
Block a user