394 lines
14 KiB
JavaScript
394 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
|
|
function ensureDir(dirPath) {
|
|
fs.mkdirSync(dirPath, { recursive: true });
|
|
}
|
|
|
|
function readJson(filePath) {
|
|
const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "");
|
|
return JSON.parse(raw);
|
|
}
|
|
|
|
function writeJson(filePath, payload) {
|
|
ensureDir(path.dirname(filePath));
|
|
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
function writeText(filePath, text) {
|
|
ensureDir(path.dirname(filePath));
|
|
fs.writeFileSync(filePath, text, "utf8");
|
|
}
|
|
|
|
function asArray(value) {
|
|
return Array.isArray(value) ? value : [];
|
|
}
|
|
|
|
function asObject(value) {
|
|
return value && typeof value === "object" ? value : {};
|
|
}
|
|
|
|
function asNumber(value, fallback = 0) {
|
|
const numeric = Number(value);
|
|
return Number.isFinite(numeric) ? numeric : fallback;
|
|
}
|
|
|
|
function pickFirstRow(payload) {
|
|
const rows = asArray(payload?.rows);
|
|
if (rows.length === 0) {
|
|
throw new Error("Input raw payload has no rows.");
|
|
}
|
|
return rows[0];
|
|
}
|
|
|
|
function collectLiveCallInventory(debug) {
|
|
const faRouteAudit = asObject(debug?.fa_live_route_audit);
|
|
const routeCalls = asArray(faRouteAudit?.executed_live_calls).map((item) => ({
|
|
source: "fa_live_route_audit",
|
|
call_id: item?.call_id ?? null,
|
|
purpose: item?.purpose ?? null,
|
|
required_for_claim: Boolean(item?.required_for_claim),
|
|
status: item?.status ?? null,
|
|
fetched_rows: asNumber(item?.fetched_rows),
|
|
matched_rows: asNumber(item?.matched_rows),
|
|
returned_rows: asNumber(item?.returned_rows),
|
|
error: item?.error ?? null
|
|
}));
|
|
|
|
const summaryCalls = [];
|
|
for (const result of asArray(debug?.retrieval_results)) {
|
|
const live = asObject(asObject(result?.summary)?.live_mcp);
|
|
if (!live || Object.keys(live).length === 0) {
|
|
continue;
|
|
}
|
|
summaryCalls.push({
|
|
source: "retrieval_results.summary.live_mcp",
|
|
fragment_id: result?.fragment_id ?? null,
|
|
route: result?.route ?? null,
|
|
method: live?.method ?? null,
|
|
status: live?.status ?? null,
|
|
fetched_rows: asNumber(live?.fetched_rows),
|
|
matched_rows: asNumber(live?.matched_rows),
|
|
returned_rows: asNumber(live?.returned_rows),
|
|
account_scope: asArray(live?.account_scope),
|
|
source_profile: live?.source_profile ?? null,
|
|
args_summary: asObject(live?.args)
|
|
});
|
|
}
|
|
|
|
return {
|
|
route_calls: routeCalls,
|
|
summary_calls: summaryCalls,
|
|
total_calls: routeCalls.length + summaryCalls.length
|
|
};
|
|
}
|
|
|
|
function formatList(items) {
|
|
if (!items.length) {
|
|
return "- none";
|
|
}
|
|
return items.map((item) => `- ${String(item)}`).join("\n");
|
|
}
|
|
|
|
function mapRequiredEntities() {
|
|
return {
|
|
seed_entities: [
|
|
"amortization_document_for_2020_07_31",
|
|
"amount_markers_2471_52_2465_28_849_83",
|
|
"july_2020_primary_period",
|
|
"fixed_asset_objects_candidates"
|
|
],
|
|
required_entities: [
|
|
"amortization_document",
|
|
"fixed_asset_objects",
|
|
"amortization_movements",
|
|
"postings",
|
|
"fixed_asset_register_state",
|
|
"expected_fa_set_for_july",
|
|
"actual_fa_set_from_amortization",
|
|
"missing_fa_candidates"
|
|
],
|
|
expected_transitions: [
|
|
"fixed_asset_object_to_amortization_document",
|
|
"amortization_document_to_movement_or_posting",
|
|
"expected_fa_set_to_actual_fa_set_comparison",
|
|
"actual_vs_missing_to_risk_of_incomplete_coverage"
|
|
]
|
|
};
|
|
}
|
|
|
|
function buildVerdict(metrics) {
|
|
const expectedFixed = metrics.fa_expected_set_reconstruction_rate >= 0.85;
|
|
const relationFixed = metrics.fa_relation_mapping_coverage_rate >= 0.85;
|
|
const claimFixed = metrics.fa_claim_anchor_coverage_rate >= 0.9;
|
|
const proofFixed = metrics.fa_proof_closure_rate > 0;
|
|
const falseGroundedOk = metrics.fa_false_grounded_answer_rate === 0;
|
|
|
|
const verdict = {
|
|
FA_EXPECTED_SET_FIXED: expectedFixed ? "FIXED" : "NOT_FIXED",
|
|
FA_RELATION_MAPPING_FIXED: relationFixed ? "FIXED" : "NOT_FIXED",
|
|
FA_CLAIM_ANCHOR_CLOSURE_FIXED: claimFixed ? "FIXED" : "NOT_FIXED",
|
|
FA_PROOF_CLOSURE_FIXED: proofFixed ? "FIXED" : "NOT_FIXED"
|
|
};
|
|
|
|
let overall = "FA_PACK_NOT_ACCEPTED";
|
|
if (expectedFixed && relationFixed && claimFixed && proofFixed && falseGroundedOk) {
|
|
overall = "FA_PACK_ACCEPTED";
|
|
} else if (expectedFixed && relationFixed && falseGroundedOk) {
|
|
overall = "FA_PACK_ACCEPTED_WITH_LIMITATIONS";
|
|
}
|
|
|
|
return { verdict, overall };
|
|
}
|
|
|
|
function main() {
|
|
const rawPathArg = process.argv[2];
|
|
const runDirArg = process.argv[3];
|
|
const modeArg = process.argv[4] || "mock";
|
|
|
|
if (!rawPathArg || !runDirArg) {
|
|
throw new Error("Usage: node faPackExportArtifacts.js <fa_raw.json> <run-dir> [mode]");
|
|
}
|
|
|
|
const rawPath = path.resolve(rawPathArg);
|
|
const runDir = path.resolve(runDirArg);
|
|
const mode = String(modeArg).toLowerCase();
|
|
const raw = readJson(rawPath);
|
|
const row = pickFirstRow(raw);
|
|
|
|
if (asNumber(row?.http_status) !== 200) {
|
|
const status = asNumber(row?.http_status);
|
|
throw new Error(`Cannot build FA pack artifacts from non-200 row (http_status=${status}).`);
|
|
}
|
|
|
|
const debug = asObject(row?.debug);
|
|
const claimAudit = asObject(debug?.claim_anchor_audit);
|
|
const targeted = asObject(debug?.targeted_evidence_acquisition);
|
|
const admissibility = asObject(debug?.evidence_admissibility_gate);
|
|
const eligibility = asObject(debug?.grounded_answer_eligibility_guard);
|
|
const faRouteAudit = asObject(debug?.fa_live_route_audit);
|
|
|
|
const expectedSet = asArray(targeted?.fa_expected_set);
|
|
const actualSet = asArray(targeted?.fa_actual_set_from_amortization);
|
|
const missingSet = asArray(targeted?.fa_missing_candidates);
|
|
const uncertainSet = asArray(targeted?.fa_uncertain_candidates);
|
|
const relationMap = asArray(targeted?.fa_relation_map);
|
|
const rejectBreakdown = asObject(admissibility?.reject_breakdown);
|
|
|
|
const claimsRequired = asArray(claimAudit?.required_anchors);
|
|
const claimsMissing = asArray(claimAudit?.missing_anchors);
|
|
const claimResolutionRate = asNumber(claimAudit?.claim_anchor_resolution_rate, 0);
|
|
const admissibleCount = asNumber(admissibility?.admissible_evidence_count, 0);
|
|
const groundingMode = String(eligibility?.grounding_mode ?? "");
|
|
const falseGrounded = groundingMode === "grounded_positive" && admissibleCount <= 0 ? 1 : 0;
|
|
|
|
const metrics = {
|
|
fa_expected_set_reconstruction_rate: expectedSet.length > 0 ? 1 : 0,
|
|
fa_relation_mapping_coverage_rate: relationMap.length > 0 ? 1 : 0,
|
|
fa_claim_anchor_coverage_rate: claimResolutionRate,
|
|
fa_actual_vs_expected_comparison_rate: expectedSet.length > 0 && actualSet.length > 0 ? 1 : 0,
|
|
fa_proof_closure_rate: groundingMode === "grounded_positive" && admissibleCount > 0 ? 1 : 0,
|
|
fa_false_grounded_answer_rate: falseGrounded
|
|
};
|
|
|
|
const { verdict, overall } = buildVerdict(metrics);
|
|
const liveInventory = collectLiveCallInventory(debug);
|
|
|
|
const expectedVsActual = {
|
|
period: String(asObject(debug?.temporal_guard)?.resolved_time_anchor ?? "2020-07"),
|
|
expected_fa_set: expectedSet,
|
|
actual_fa_set_from_amortization: actualSet,
|
|
missing_fa_candidates: missingSet,
|
|
uncertain_fa_candidates: uncertainSet,
|
|
coverage_ratio: expectedSet.length > 0 ? Number((actualSet.length / expectedSet.length).toFixed(4)) : 0,
|
|
source: "targeted_evidence_acquisition.fa_*"
|
|
};
|
|
|
|
const runSummary = {
|
|
stage: "Stage 4",
|
|
pack: "FA amortization proof closure",
|
|
date: new Date().toISOString().slice(0, 10),
|
|
mode,
|
|
status: overall,
|
|
verdict,
|
|
metrics,
|
|
inputs: {
|
|
raw_file: rawPath,
|
|
user_message: String(row?.user_message ?? ""),
|
|
trace_id: String(row?.trace_id ?? "")
|
|
},
|
|
runtime: {
|
|
reply_type: String(row?.reply_type ?? ""),
|
|
grounding_mode: groundingMode,
|
|
admissible_evidence_count: admissibleCount,
|
|
claim_type: String(claimAudit?.claim_type ?? "")
|
|
},
|
|
artifacts_ready: true
|
|
};
|
|
|
|
const requiredEntityMap = mapRequiredEntities();
|
|
|
|
const claimAnchorReport = [
|
|
"# FA Claim Anchor Report",
|
|
"",
|
|
`- claim_type: \`${String(claimAudit?.claim_type ?? "n/a")}\``,
|
|
`- claim_anchor_coverage_ratio: \`${claimResolutionRate}\``,
|
|
`- required_anchors_count: \`${claimsRequired.length}\``,
|
|
`- missing_anchor_classes_count: \`${claimsMissing.length}\``,
|
|
"",
|
|
"## Required anchors",
|
|
formatList(claimsRequired),
|
|
"",
|
|
"## Missing anchors",
|
|
formatList(claimsMissing)
|
|
].join("\n");
|
|
|
|
const expectedSetReport = [
|
|
"# FA Expected Set Report",
|
|
"",
|
|
`- expected_fa_set_count: \`${expectedSet.length}\``,
|
|
`- actual_fa_set_count: \`${actualSet.length}\``,
|
|
`- missing_fa_candidates_count: \`${missingSet.length}\``,
|
|
`- uncertain_fa_candidates_count: \`${uncertainSet.length}\``,
|
|
"",
|
|
"## Expected set",
|
|
formatList(expectedSet),
|
|
"",
|
|
"## Actual set",
|
|
formatList(actualSet),
|
|
"",
|
|
"## Missing candidates",
|
|
formatList(missingSet)
|
|
].join("\n");
|
|
|
|
const relationPreview = relationMap.slice(0, 20).map((item) => {
|
|
const objectId = String(item?.fa_object ?? "n/a");
|
|
const status = String(item?.coverage_status ?? "n/a");
|
|
const docs = asArray(item?.document_amortization).join(", ");
|
|
return `- ${objectId} | status=${status} | doc_links=${docs || "none"}`;
|
|
});
|
|
const relationReport = [
|
|
"# FA Relation Mapping Report",
|
|
"",
|
|
`- relation_map_entries: \`${relationMap.length}\``,
|
|
"",
|
|
"## Object-level relation preview",
|
|
relationPreview.length ? relationPreview.join("\n") : "- none"
|
|
].join("\n");
|
|
|
|
const proofReport = [
|
|
"# FA Proof Closure Report",
|
|
"",
|
|
`- reply_type: \`${String(row?.reply_type ?? "")}\``,
|
|
`- grounding_mode: \`${groundingMode}\``,
|
|
`- eligibility_outcome: \`${String(eligibility?.outcome ?? "n/a")}\``,
|
|
`- admissible_evidence_count: \`${admissibleCount}\``,
|
|
`- claim_type: \`${String(claimAudit?.claim_type ?? "n/a")}\``,
|
|
"",
|
|
"## Reason codes",
|
|
formatList(asArray(eligibility?.reason_codes)),
|
|
"",
|
|
"## FA route reasons",
|
|
formatList(asArray(faRouteAudit?.missing_live_calls))
|
|
].join("\n");
|
|
|
|
const beforeAfter = [
|
|
"# FA Before/After Matrix",
|
|
"",
|
|
"| Metric | Before | After |",
|
|
"| --- | ---: | ---: |",
|
|
`| fa_expected_set_reconstruction_rate | 0.00 | ${metrics.fa_expected_set_reconstruction_rate.toFixed(2)} |`,
|
|
`| fa_relation_mapping_coverage_rate | 0.00 | ${metrics.fa_relation_mapping_coverage_rate.toFixed(2)} |`,
|
|
`| fa_claim_anchor_coverage_rate | 0.00 | ${metrics.fa_claim_anchor_coverage_rate.toFixed(2)} |`,
|
|
`| fa_actual_vs_expected_comparison_rate | 0.00 | ${metrics.fa_actual_vs_expected_comparison_rate.toFixed(2)} |`,
|
|
`| fa_proof_closure_rate | 0.00 | ${metrics.fa_proof_closure_rate.toFixed(2)} |`,
|
|
`| fa_false_grounded_answer_rate | 0.00 | ${metrics.fa_false_grounded_answer_rate.toFixed(2)} |`
|
|
].join("\n");
|
|
|
|
const chatExport = [
|
|
"# Chat Export FA",
|
|
"",
|
|
"## 1. user",
|
|
String(row?.user_message ?? ""),
|
|
"",
|
|
"## 2. assistant",
|
|
String(row?.assistant_reply ?? "")
|
|
].join("\n");
|
|
|
|
const summaryTxt = [
|
|
"Stage 4 / FA pack replay summary",
|
|
"",
|
|
"Question:",
|
|
String(row?.user_message ?? ""),
|
|
"",
|
|
"Result highlights:",
|
|
`- claim_type: ${String(claimAudit?.claim_type ?? "n/a")}`,
|
|
`- required FA live calls executed: ${asArray(faRouteAudit?.executed_live_calls).length}`,
|
|
`- expected_fa_set_count: ${expectedSet.length}`,
|
|
`- actual_fa_set_count: ${actualSet.length}`,
|
|
`- missing_fa_candidates_count: ${missingSet.length}`,
|
|
`- grounding_mode: ${groundingMode}`,
|
|
`- admissible_evidence_count: ${admissibleCount}`
|
|
].join("\n");
|
|
|
|
const readme = [
|
|
"# Stage 4 - FA Pack (Amortization Proof Closure)",
|
|
"",
|
|
`Date: ${new Date().toISOString().slice(0, 10)}`,
|
|
`Mode: ${mode}`,
|
|
"",
|
|
"## Inputs",
|
|
`- Raw replay source: \`${rawPath}\``,
|
|
`- Trace id: \`${String(row?.trace_id ?? "n/a")}\``,
|
|
"",
|
|
"## Final verdict",
|
|
`- FA_EXPECTED_SET_FIXED: \`${verdict.FA_EXPECTED_SET_FIXED}\``,
|
|
`- FA_RELATION_MAPPING_FIXED: \`${verdict.FA_RELATION_MAPPING_FIXED}\``,
|
|
`- FA_CLAIM_ANCHOR_CLOSURE_FIXED: \`${verdict.FA_CLAIM_ANCHOR_CLOSURE_FIXED}\``,
|
|
`- FA_PROOF_CLOSURE_FIXED: \`${verdict.FA_PROOF_CLOSURE_FIXED}\``,
|
|
`- Overall: \`${overall}\``,
|
|
"",
|
|
"## Notes",
|
|
mode === "live"
|
|
? "- Artifacts generated from live replay payload."
|
|
: "- Artifacts generated from controlled replay payload (non-live mode)."
|
|
].join("\n");
|
|
|
|
writeJson(path.join(runDir, "run_summary.json"), runSummary);
|
|
writeText(path.join(runDir, "README.md"), `${readme}\n`);
|
|
writeText(path.join(runDir, "fa_expected_set_report.md"), `${expectedSetReport}\n`);
|
|
writeText(path.join(runDir, "fa_relation_mapping_report.md"), `${relationReport}\n`);
|
|
writeText(path.join(runDir, "fa_claim_anchor_report.md"), `${claimAnchorReport}\n`);
|
|
writeText(path.join(runDir, "fa_proof_closure_report.md"), `${proofReport}\n`);
|
|
writeText(path.join(runDir, "fa_before_after_matrix.md"), `${beforeAfter}\n`);
|
|
writeText(path.join(runDir, "chat_export_fa.md"), `${chatExport}\n`);
|
|
writeText(path.join(runDir, "1.txt"), `${summaryTxt}\n`);
|
|
writeJson(path.join(runDir, "fa_required_entity_map.json"), requiredEntityMap);
|
|
writeJson(path.join(runDir, "fa_expected_vs_actual_set.json"), expectedVsActual);
|
|
writeJson(path.join(runDir, "fa_relation_map.json"), relationMap);
|
|
writeJson(path.join(runDir, "fa_admissibility_reject_breakdown.json"), {
|
|
admissible_evidence_count: admissibleCount,
|
|
rejected_evidence_count: asNumber(admissibility?.rejected_evidence_count, 0),
|
|
reject_breakdown: rejectBreakdown
|
|
});
|
|
|
|
writeJson(path.join(runDir, "debug_payloads", "fa_claim_bound_debug_sample.json"), {
|
|
trace_id: String(row?.trace_id ?? ""),
|
|
reply_type: String(row?.reply_type ?? ""),
|
|
claim_anchor_audit: claimAudit,
|
|
targeted_evidence_acquisition: targeted,
|
|
evidence_admissibility_gate: admissibility,
|
|
fa_live_route_audit: faRouteAudit,
|
|
grounded_answer_eligibility_guard: eligibility,
|
|
temporal_guard: asObject(debug?.temporal_guard),
|
|
business_scope_resolved: asArray(debug?.business_scope_resolved)
|
|
});
|
|
|
|
writeJson(path.join(runDir, "raw_live_calls", "fa_live_call_inventory_sample.json"), liveInventory);
|
|
}
|
|
|
|
main();
|
|
|