Этап 4 / Волна 10: корректировка settlement-кейса — доменная фиксация синтеза, честное покрытие, удержание фокуса / Этап 4 / Волна 11: бизнес-якоря, доменное заземление и устранение утечки дебага

llm_normalizer/backend/src/eval/p0_acceptance_gate.ts

@@ -0,0 +1,265 @@
+import {
+  P0_DEFAULT_ACCEPTANCE_THRESHOLDS,
+  P0_DEFAULT_QUALITY_GAP_THRESHOLDS,
+  type P0AcceptanceThresholds,
+  type P0EvalMetricVector,
+  type P0QualityGapMetricVector,
+  type P0QualityGapThresholds
+} from "./p0_metric_definitions";
+
+export type P0AcceptanceVerdict = "P0_ACCEPTED" | "P0_ACCEPTED_WITH_LIMITATIONS" | "P0_NOT_ACCEPTED";
+export type P0BaselineStabilityVerdict = "P0_BASELINE_STABLE" | "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS";
+
+interface MetricCheck {
+  metric: keyof P0EvalMetricVector;
+  value: number;
+  threshold: number;
+  comparator: ">=" | "<=";
+  passed: boolean;
+  severity: "blocking" | "quality";
+}
+
+interface QualityGapMetricCheck {
+  metric: keyof P0QualityGapMetricVector;
+  value: number;
+  threshold: number;
+  comparator: ">=" | "<=";
+  passed: boolean;
+}
+
+export interface P0AcceptanceResult {
+  verdict: P0AcceptanceVerdict;
+  passed_all: boolean;
+  blocking_failures: MetricCheck[];
+  quality_failures: MetricCheck[];
+  metric_checks: MetricCheck[];
+  rationale: string[];
+}
+
+export interface P0BaselineStabilityResult {
+  verdict: P0BaselineStabilityVerdict;
+  baseline_integrity_passed: boolean;
+  quality_gaps_open: boolean;
+  blocking_regressions: MetricCheck[];
+  legacy_quality_failures: MetricCheck[];
+  quality_gap_failures: QualityGapMetricCheck[];
+  quality_gap_checks: QualityGapMetricCheck[];
+  rationale: string[];
+}
+
+function buildMetricChecks(input: {
+  metrics: P0EvalMetricVector;
+  thresholds: P0AcceptanceThresholds;
+}): MetricCheck[] {
+  const { metrics, thresholds } = input;
+  return [
+    {
+      metric: "problem_first_answer_rate",
+      value: metrics.problem_first_answer_rate,
+      threshold: thresholds.problem_first_answer_rate_min,
+      comparator: ">=",
+      passed: metrics.problem_first_answer_rate >= thresholds.problem_first_answer_rate_min,
+      severity: "blocking"
+    },
+    {
+      metric: "entity_leakage_rate",
+      value: metrics.entity_leakage_rate,
+      threshold: thresholds.entity_leakage_rate_max,
+      comparator: "<=",
+      passed: metrics.entity_leakage_rate <= thresholds.entity_leakage_rate_max,
+      severity: "blocking"
+    },
+    {
+      metric: "route_correctness_rate",
+      value: metrics.route_correctness_rate,
+      threshold: thresholds.route_correctness_rate_min,
+      comparator: ">=",
+      passed: metrics.route_correctness_rate >= thresholds.route_correctness_rate_min,
+      severity: "blocking"
+    },
+    {
+      metric: "domain_purity_rate",
+      value: metrics.domain_purity_rate,
+      threshold: thresholds.domain_purity_rate_min,
+      comparator: ">=",
+      passed: metrics.domain_purity_rate >= thresholds.domain_purity_rate_min,
+      severity: "blocking"
+    },
+    {
+      metric: "mechanism_coherence_score",
+      value: metrics.mechanism_coherence_score,
+      threshold: thresholds.mechanism_coherence_score_min,
+      comparator: ">=",
+      passed: metrics.mechanism_coherence_score >= thresholds.mechanism_coherence_score_min,
+      severity: "quality"
+    },
+    {
+      metric: "accountant_actionability_score",
+      value: metrics.accountant_actionability_score,
+      threshold: thresholds.accountant_actionability_score_min,
+      comparator: ">=",
+      passed: metrics.accountant_actionability_score >= thresholds.accountant_actionability_score_min,
+      severity: "quality"
+    },
+    {
+      metric: "limitation_honesty_rate",
+      value: metrics.limitation_honesty_rate,
+      threshold: thresholds.limitation_honesty_rate_min,
+      comparator: ">=",
+      passed: metrics.limitation_honesty_rate >= thresholds.limitation_honesty_rate_min,
+      severity: "quality"
+    },
+    {
+      metric: "top_problem_unit_match_rate",
+      value: metrics.top_problem_unit_match_rate,
+      threshold: thresholds.top_problem_unit_match_rate_min,
+      comparator: ">=",
+      passed: metrics.top_problem_unit_match_rate >= thresholds.top_problem_unit_match_rate_min,
+      severity: "quality"
+    }
+  ];
+}
+
+function toFailureLine(check: MetricCheck): string {
+  return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
+}
+
+function toQualityGapFailureLine(check: QualityGapMetricCheck): string {
+  return `${check.metric}: ${check.value.toFixed(2)} ${check.comparator} ${check.threshold.toFixed(2)} (failed)`;
+}
+
+function buildQualityGapChecks(input: {
+  metrics: P0QualityGapMetricVector;
+  thresholds: P0QualityGapThresholds;
+}): QualityGapMetricCheck[] {
+  const { metrics, thresholds } = input;
+  return [
+    {
+      metric: "generic_explanation_rate",
+      value: metrics.generic_explanation_rate,
+      threshold: thresholds.generic_explanation_rate_max,
+      comparator: "<=",
+      passed: metrics.generic_explanation_rate <= thresholds.generic_explanation_rate_max
+    },
+    {
+      metric: "false_confidence_rate",
+      value: metrics.false_confidence_rate,
+      threshold: thresholds.false_confidence_rate_max,
+      comparator: "<=",
+      passed: metrics.false_confidence_rate <= thresholds.false_confidence_rate_max
+    },
+    {
+      metric: "mechanism_specificity_score",
+      value: metrics.mechanism_specificity_score,
+      threshold: thresholds.mechanism_specificity_score_min,
+      comparator: ">=",
+      passed: metrics.mechanism_specificity_score >= thresholds.mechanism_specificity_score_min
+    },
+    {
+      metric: "followup_context_retention_score",
+      value: metrics.followup_context_retention_score,
+      threshold: thresholds.followup_context_retention_score_min,
+      comparator: ">=",
+      passed: metrics.followup_context_retention_score >= thresholds.followup_context_retention_score_min
+    }
+  ];
+}
+
+export function evaluateP0AcceptanceGate(input: {
+  metrics: P0EvalMetricVector;
+  thresholds?: Partial<P0AcceptanceThresholds>;
+}): P0AcceptanceResult {
+  const thresholds: P0AcceptanceThresholds = {
+    ...P0_DEFAULT_ACCEPTANCE_THRESHOLDS,
+    ...(input.thresholds ?? {})
+  };
+  const checks = buildMetricChecks({
+    metrics: input.metrics,
+    thresholds
+  });
+  const blockingFailures = checks.filter((item) => item.severity === "blocking" && !item.passed);
+  const qualityFailures = checks.filter((item) => item.severity === "quality" && !item.passed);
+
+  let verdict: P0AcceptanceVerdict = "P0_NOT_ACCEPTED";
+  if (blockingFailures.length === 0 && qualityFailures.length === 0) {
+    verdict = "P0_ACCEPTED";
+  } else if (blockingFailures.length === 0) {
+    verdict = "P0_ACCEPTED_WITH_LIMITATIONS";
+  }
+
+  const rationale: string[] = [];
+  if (blockingFailures.length > 0) {
+    rationale.push("Blocking metrics failed:");
+    rationale.push(...blockingFailures.map((item) => `- ${toFailureLine(item)}`));
+  }
+  if (qualityFailures.length > 0) {
+    rationale.push("Quality metrics failed:");
+    rationale.push(...qualityFailures.map((item) => `- ${toFailureLine(item)}`));
+  }
+  if (rationale.length === 0) {
+    rationale.push("All acceptance thresholds are satisfied.");
+  }
+
+  return {
+    verdict,
+    passed_all: checks.every((item) => item.passed),
+    blocking_failures: blockingFailures,
+    quality_failures: qualityFailures,
+    metric_checks: checks,
+    rationale
+  };
+}
+
+export function evaluateP0BaselineStabilityGate(input: {
+  metrics: P0EvalMetricVector;
+  qualityGapMetrics: P0QualityGapMetricVector;
+  acceptanceThresholds?: Partial<P0AcceptanceThresholds>;
+  qualityGapThresholds?: Partial<P0QualityGapThresholds>;
+}): P0BaselineStabilityResult {
+  const acceptance = evaluateP0AcceptanceGate({
+    metrics: input.metrics,
+    thresholds: input.acceptanceThresholds
+  });
+  const qualityGapThresholds: P0QualityGapThresholds = {
+    ...P0_DEFAULT_QUALITY_GAP_THRESHOLDS,
+    ...(input.qualityGapThresholds ?? {})
+  };
+  const qualityGapChecks = buildQualityGapChecks({
+    metrics: input.qualityGapMetrics,
+    thresholds: qualityGapThresholds
+  });
+  const qualityGapFailures = qualityGapChecks.filter((item) => !item.passed);
+
+  const baselineIntegrityPassed = acceptance.blocking_failures.length === 0;
+  const qualityGapsOpen = acceptance.quality_failures.length > 0 || qualityGapFailures.length > 0;
+  const verdict: P0BaselineStabilityVerdict =
+    baselineIntegrityPassed && !qualityGapsOpen ? "P0_BASELINE_STABLE" : "P0_BASELINE_STABLE_WITH_OPEN_QUALITY_GAPS";
+
+  const rationale: string[] = [];
+  if (!baselineIntegrityPassed) {
+    rationale.push("Blocking baseline regressions detected:");
+    rationale.push(...acceptance.blocking_failures.map((item) => `- ${toFailureLine(item)}`));
+  }
+  if (acceptance.quality_failures.length > 0) {
+    rationale.push("Legacy quality thresholds still below full acceptance:");
+    rationale.push(...acceptance.quality_failures.map((item) => `- ${toFailureLine(item)}`));
+  }
+  if (qualityGapFailures.length > 0) {
+    rationale.push("Wave 9 quality-gap thresholds still open:");
+    rationale.push(...qualityGapFailures.map((item) => `- ${toQualityGapFailureLine(item)}`));
+  }
+  if (rationale.length === 0) {
+    rationale.push("Baseline is stable and Wave 9 quality-gap thresholds are satisfied.");
+  }
+
+  return {
+    verdict,
+    baseline_integrity_passed: baselineIntegrityPassed,
+    quality_gaps_open: qualityGapsOpen,
+    blocking_regressions: acceptance.blocking_failures,
+    legacy_quality_failures: acceptance.quality_failures,
+    quality_gap_failures: qualityGapFailures,
+    quality_gap_checks: qualityGapChecks,
+    rationale
+  };
+}

llm_normalizer/backend/src/eval/p0_metric_definitions.ts

@@ -0,0 +1,425 @@
+import type { ProblemUnitType } from "../types/stage2ProblemUnits";
+
+export type P0DomainId = "settlements_60_62" | "vat_document_register_book" | "month_close_costs_20_44";
+
+export type P0QueryClass =
+  | "symptom_first"
+  | "lifecycle_first"
+  | "causal_query"
+  | "mixed_ambiguity"
+  | "followup_investigation"
+  | "noisy_input"
+  | "translit_noisy"
+  | "multi_intent";
+
+export type P0RouteExpected = "hybrid_store_plus_live" | "store_feature_risk" | "store_canonical" | "batch_refresh_then_store" | "live_mcp_drilldown" | "no_route";
+
+export interface P0ExpectedAnnotations {
+  domain_expected: P0DomainId;
+  route_expected: P0RouteExpected | P0RouteExpected[];
+  problem_unit_type_expected: ProblemUnitType;
+  mechanism_class_expected: string;
+  evidence_expectation: string;
+  must_show_limitation: boolean;
+  must_not_leak_internal: boolean;
+  first_check_action_expected: string;
+}
+
+export interface P0EvalCase {
+  case_id: string;
+  domain: P0DomainId;
+  user_query_raw: string;
+  query_class: P0QueryClass;
+  expected_problem_unit_type: ProblemUnitType;
+  expected_mechanism_summary: string;
+  expected_first_check: string;
+  must_include_limitations: boolean;
+  forbidden_leakage_tokens: string[];
+  followup_seed_query?: string;
+  followup_expected_context_tokens?: string[];
+  notes: string;
+  expected_annotations: P0ExpectedAnnotations;
+}
+
+export interface P0EvalCorpusFile {
+  suite_id: string;
+  suite_version: string;
+  schema_version: string;
+  scenario_count: number;
+  case_ids: string[];
+  cases: P0EvalCase[];
+}
+
+export interface P0EvalMetricVector {
+  problem_first_answer_rate: number;
+  mechanism_coherence_score: number;
+  entity_leakage_rate: number;
+  accountant_actionability_score: number;
+  route_correctness_rate: number;
+  domain_purity_rate: number;
+  limitation_honesty_rate: number;
+  top_problem_unit_match_rate: number;
+}
+
+export type P0MetricName = keyof P0EvalMetricVector;
+
+export interface P0QualityGapMetricVector {
+  generic_explanation_rate: number;
+  false_confidence_rate: number;
+  mechanism_specificity_score: number;
+  followup_context_retention_score: number;
+}
+
+export type P0QualityGapMetricName = keyof P0QualityGapMetricVector;
+
+export interface P0MetricDefinition {
+  description: string;
+  unit: "rate" | "score";
+  direction: "higher_is_better" | "lower_is_better";
+}
+
+export interface P0QualityGapMetricDefinition {
+  description: string;
+  unit: "rate" | "score";
+  direction: "higher_is_better" | "lower_is_better";
+}
+
+export const P0_METRIC_DEFINITIONS: Record<P0MetricName, P0MetricDefinition> = {
+  problem_first_answer_rate: {
+    description: "Share of cases where answer is problem-first and avoids entity-dump framing.",
+    unit: "rate",
+    direction: "higher_is_better"
+  },
+  mechanism_coherence_score: {
+    description: "Average 0..5 score for mechanism clarity and alignment with expected defect class.",
+    unit: "score",
+    direction: "higher_is_better"
+  },
+  entity_leakage_rate: {
+    description: "Share of cases with internal/technical leakage in user-facing answer.",
+    unit: "rate",
+    direction: "lower_is_better"
+  },
+  accountant_actionability_score: {
+    description: "Average 0..5 score for first-check concreteness and practical accounting actionability.",
+    unit: "score",
+    direction: "higher_is_better"
+  },
+  route_correctness_rate: {
+    description: "Share of cases routed to expected path class for query type.",
+    unit: "rate",
+    direction: "higher_is_better"
+  },
+  domain_purity_rate: {
+    description: "Share of cases where top-3 retrieval stays inside expected P0 domain boundary.",
+    unit: "rate",
+    direction: "higher_is_better"
+  },
+  limitation_honesty_rate: {
+    description: "Share of cases where mandatory limitations are explicit when evidence is incomplete.",
+    unit: "rate",
+    direction: "higher_is_better"
+  },
+  top_problem_unit_match_rate: {
+    description: "Share of cases where top problem unit type matches expected annotation.",
+    unit: "rate",
+    direction: "higher_is_better"
+  }
+};
+
+export const P0_QUALITY_GAP_METRIC_DEFINITIONS: Record<P0QualityGapMetricName, P0QualityGapMetricDefinition> = {
+  generic_explanation_rate: {
+    description: "Share of cases where direct answer stays generic and mechanism/actionability specificity is insufficient.",
+    unit: "rate",
+    direction: "lower_is_better"
+  },
+  false_confidence_rate: {
+    description: "Share of cases with overconfident framing not supported by evidence/limitation state.",
+    unit: "rate",
+    direction: "lower_is_better"
+  },
+  mechanism_specificity_score: {
+    description: "Average 0..5 score of mechanism-level specificity in accountant-facing answer.",
+    unit: "score",
+    direction: "higher_is_better"
+  },
+  followup_context_retention_score: {
+    description: "Rate of correct context retention in follow-up investigation cases.",
+    unit: "rate",
+    direction: "higher_is_better"
+  }
+};
+
+export const P0_DEFAULT_FORBIDDEN_LEAKAGE_TOKENS = [
+  "graph_",
+  "domain_scope",
+  "relation_patterns",
+  "account_scope",
+  "semantic_profile",
+  "route",
+  "profile",
+  "store_canonical",
+  "hybrid_store_plus_live",
+  "deterministic_v2",
+  "candidate_evidence",
+  "problem_unit",
+  "raw_entities",
+  "graph_domain_scope"
+];
+
+export interface P0AcceptanceThresholds {
+  problem_first_answer_rate_min: number;
+  mechanism_coherence_score_min: number;
+  entity_leakage_rate_max: number;
+  accountant_actionability_score_min: number;
+  route_correctness_rate_min: number;
+  domain_purity_rate_min: number;
+  limitation_honesty_rate_min: number;
+  top_problem_unit_match_rate_min: number;
+}
+
+export interface P0QualityGapThresholds {
+  generic_explanation_rate_max: number;
+  false_confidence_rate_max: number;
+  mechanism_specificity_score_min: number;
+  followup_context_retention_score_min: number;
+}
+
+export const P0_DEFAULT_ACCEPTANCE_THRESHOLDS: P0AcceptanceThresholds = {
+  problem_first_answer_rate_min: 0.8,
+  mechanism_coherence_score_min: 3.5,
+  entity_leakage_rate_max: 0.08,
+  accountant_actionability_score_min: 3.5,
+  route_correctness_rate_min: 0.85,
+  domain_purity_rate_min: 0.9,
+  limitation_honesty_rate_min: 0.9,
+  top_problem_unit_match_rate_min: 0.65
+};
+
+export const P0_DEFAULT_QUALITY_GAP_THRESHOLDS: P0QualityGapThresholds = {
+  generic_explanation_rate_max: 0.2,
+  false_confidence_rate_max: 0.12,
+  mechanism_specificity_score_min: 3.0,
+  followup_context_retention_score_min: 0.75
+};
+
+function isProblemUnitType(value: unknown): value is ProblemUnitType {
+  return (
+    value === "document_conflict" ||
+    value === "broken_chain_segment" ||
+    value === "lifecycle_anomaly_node" ||
+    value === "unresolved_settlement_cluster" ||
+    value === "period_risk_cluster" ||
+    value === "cross_branch_inconsistency_cluster"
+  );
+}
+
+function isP0DomainId(value: unknown): value is P0DomainId {
+  return value === "settlements_60_62" || value === "vat_document_register_book" || value === "month_close_costs_20_44";
+}
+
+function isP0QueryClass(value: unknown): value is P0QueryClass {
+  return (
+    value === "symptom_first" ||
+    value === "lifecycle_first" ||
+    value === "causal_query" ||
+    value === "mixed_ambiguity" ||
+    value === "followup_investigation" ||
+    value === "noisy_input" ||
+    value === "translit_noisy" ||
+    value === "multi_intent"
+  );
+}
+
+function isRouteExpected(value: unknown): value is P0RouteExpected {
+  return (
+    value === "hybrid_store_plus_live" ||
+    value === "store_feature_risk" ||
+    value === "store_canonical" ||
+    value === "batch_refresh_then_store" ||
+    value === "live_mcp_drilldown" ||
+    value === "no_route"
+  );
+}
+
+function assertString(value: unknown, fieldPath: string): string {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new Error(`Invalid corpus field '${fieldPath}': non-empty string is required.`);
+  }
+  return value.trim();
+}
+
+function assertBoolean(value: unknown, fieldPath: string): boolean {
+  if (typeof value !== "boolean") {
+    throw new Error(`Invalid corpus field '${fieldPath}': boolean is required.`);
+  }
+  return value;
+}
+
+function assertStringArray(value: unknown, fieldPath: string): string[] {
+  if (!Array.isArray(value)) {
+    throw new Error(`Invalid corpus field '${fieldPath}': string[] is required.`);
+  }
+  const output = value.map((item, index) => assertString(item, `${fieldPath}[${index}]`));
+  return output;
+}
+
+function assertRouteExpected(value: unknown, fieldPath: string): P0RouteExpected | P0RouteExpected[] {
+  if (Array.isArray(value)) {
+    const routes = value.map((item, index) => {
+      if (!isRouteExpected(item)) {
+        throw new Error(`Invalid corpus field '${fieldPath}[${index}]': unsupported route '${String(item)}'.`);
+      }
+      return item;
+    });
+    if (routes.length === 0) {
+      throw new Error(`Invalid corpus field '${fieldPath}': route list must not be empty.`);
+    }
+    return routes;
+  }
+  if (!isRouteExpected(value)) {
+    throw new Error(`Invalid corpus field '${fieldPath}': unsupported route '${String(value)}'.`);
+  }
+  return value;
+}
+
+function parseCase(raw: unknown, index: number): P0EvalCase {
+  if (!raw || typeof raw !== "object") {
+    throw new Error(`Invalid corpus case at index ${index}: object is required.`);
+  }
+  const row = raw as Record<string, unknown>;
+  const expected = row.expected_annotations;
+  if (!expected || typeof expected !== "object") {
+    throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations is required.`);
+  }
+  const annotations = expected as Record<string, unknown>;
+
+  const domain = row.domain;
+  if (!isP0DomainId(domain)) {
+    throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported domain '${String(domain)}'.`);
+  }
+  const queryClass = row.query_class;
+  if (!isP0QueryClass(queryClass)) {
+    throw new Error(`Invalid corpus case '${String(row.case_id ?? index)}': unsupported query_class '${String(queryClass)}'.`);
+  }
+  const expectedProblemType = row.expected_problem_unit_type;
+  if (!isProblemUnitType(expectedProblemType)) {
+    throw new Error(
+      `Invalid corpus case '${String(row.case_id ?? index)}': expected_problem_unit_type '${String(expectedProblemType)}' is not supported.`
+    );
+  }
+  const annotationProblemType = annotations.problem_unit_type_expected;
+  if (!isProblemUnitType(annotationProblemType)) {
+    throw new Error(
+      `Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.problem_unit_type_expected '${String(annotationProblemType)}' is not supported.`
+    );
+  }
+  const annotationDomain = annotations.domain_expected;
+  if (!isP0DomainId(annotationDomain)) {
+    throw new Error(
+      `Invalid corpus case '${String(row.case_id ?? index)}': expected_annotations.domain_expected '${String(annotationDomain)}' is not supported.`
+    );
+  }
+
+  const followupSeedQuery =
+    typeof row.followup_seed_query === "string" && row.followup_seed_query.trim() ? row.followup_seed_query.trim() : undefined;
+  const followupExpectedContextTokens = Array.isArray(row.followup_expected_context_tokens)
+    ? assertStringArray(row.followup_expected_context_tokens, `cases[${index}].followup_expected_context_tokens`)
+    : undefined;
+  if (queryClass === "followup_investigation" && !followupSeedQuery) {
+    throw new Error(
+      `Invalid corpus case '${String(row.case_id ?? index)}': followup_investigation requires followup_seed_query.`
+    );
+  }
+
+  return {
+    case_id: assertString(row.case_id, `cases[${index}].case_id`),
+    domain,
+    user_query_raw: assertString(row.user_query_raw, `cases[${index}].user_query_raw`),
+    query_class: queryClass,
+    expected_problem_unit_type: expectedProblemType,
+    expected_mechanism_summary: assertString(row.expected_mechanism_summary, `cases[${index}].expected_mechanism_summary`),
+    expected_first_check: assertString(row.expected_first_check, `cases[${index}].expected_first_check`),
+    must_include_limitations: assertBoolean(row.must_include_limitations, `cases[${index}].must_include_limitations`),
+    forbidden_leakage_tokens: assertStringArray(row.forbidden_leakage_tokens, `cases[${index}].forbidden_leakage_tokens`),
+    followup_seed_query: followupSeedQuery,
+    followup_expected_context_tokens: followupExpectedContextTokens,
+    notes: assertString(row.notes, `cases[${index}].notes`),
+    expected_annotations: {
+      domain_expected: annotationDomain,
+      route_expected: assertRouteExpected(annotations.route_expected, `cases[${index}].expected_annotations.route_expected`),
+      problem_unit_type_expected: annotationProblemType,
+      mechanism_class_expected: assertString(
+        annotations.mechanism_class_expected,
+        `cases[${index}].expected_annotations.mechanism_class_expected`
+      ),
+      evidence_expectation: assertString(annotations.evidence_expectation, `cases[${index}].expected_annotations.evidence_expectation`),
+      must_show_limitation: assertBoolean(annotations.must_show_limitation, `cases[${index}].expected_annotations.must_show_limitation`),
+      must_not_leak_internal: assertBoolean(
+        annotations.must_not_leak_internal,
+        `cases[${index}].expected_annotations.must_not_leak_internal`
+      ),
+      first_check_action_expected: assertString(
+        annotations.first_check_action_expected,
+        `cases[${index}].expected_annotations.first_check_action_expected`
+      )
+    }
+  };
+}
+
+export function validateP0EvalCorpus(input: unknown): P0EvalCorpusFile {
+  if (!input || typeof input !== "object") {
+    throw new Error("Invalid P0 corpus: root object is required.");
+  }
+  const parsed = input as Record<string, unknown>;
+  const casesRaw = parsed.cases;
+  if (!Array.isArray(casesRaw)) {
+    throw new Error("Invalid P0 corpus: cases[] is required.");
+  }
+  const cases = casesRaw.map((item, index) => parseCase(item, index));
+  if (cases.length < 30 || cases.length > 150) {
+    throw new Error(`Invalid P0 corpus: expected 30..150 cases, got ${cases.length}.`);
+  }
+
+  const caseIds = assertStringArray(parsed.case_ids, "case_ids");
+  const declaredIds = [...caseIds].sort();
+  const actualIds = cases.map((item) => item.case_id).sort();
+  const idsMatch = declaredIds.length === actualIds.length && declaredIds.every((value, index) => value === actualIds[index]);
+  if (!idsMatch) {
+    throw new Error("Invalid P0 corpus: case_ids[] does not match cases[].");
+  }
+  const scenarioCount = Number(parsed.scenario_count ?? 0);
+  if (!Number.isFinite(scenarioCount) || scenarioCount !== cases.length) {
+    throw new Error("Invalid P0 corpus: scenario_count must match cases.length.");
+  }
+
+  const perDomain = cases.reduce<Record<P0DomainId, number>>(
+    (acc, item) => {
+      acc[item.domain] += 1;
+      return acc;
+    },
+    {
+      settlements_60_62: 0,
+      vat_document_register_book: 0,
+      month_close_costs_20_44: 0
+    }
+  );
+  for (const [domain, count] of Object.entries(perDomain)) {
+    if (count < 10) {
+      throw new Error(`Invalid P0 corpus: domain '${domain}' must contain at least 10 cases, got ${count}.`);
+    }
+  }
+
+  return {
+    suite_id: assertString(parsed.suite_id, "suite_id"),
+    suite_version: assertString(parsed.suite_version, "suite_version"),
+    schema_version: assertString(parsed.schema_version, "schema_version"),
+    scenario_count: scenarioCount,
+    case_ids: caseIds,
+    cases
+  };
+}
+
+export function normalizeExpectedRoutes(value: P0RouteExpected | P0RouteExpected[]): P0RouteExpected[] {
+  return Array.isArray(value) ? value : [value];
+}