АДРЕСНЫЙ РЕЖИМ -ADDRESS:Шаг 0 (Pre-Prod Rails): Предпродакшн-постановка ADDRESS Query V1 на рельсы референсный домен, nightly-автоматизация и подтверждённый global gate (102/102 + 25/25)
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricBundle:
|
||||
summary_path: str
|
||||
run_id: str
|
||||
questions_total: int | None
|
||||
strict_pass_rate: float
|
||||
route_pass_rate: float
|
||||
execution_error_count: int
|
||||
false_factual_rate: float
|
||||
notes: list[str]
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare ADDRESS run_summary against baseline and fail on gate regressions."
|
||||
)
|
||||
parser.add_argument("--baseline-summary", required=True, help="Path to baseline run_summary.json")
|
||||
parser.add_argument("--candidate-summary", required=True, help="Path to candidate run_summary.json")
|
||||
parser.add_argument(
|
||||
"--report-json",
|
||||
default="",
|
||||
help="Optional path to write comparator report JSON.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--epsilon",
|
||||
type=float,
|
||||
default=1e-9,
|
||||
help="Tolerance for float comparison.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_summary(path: Path) -> dict[str, Any]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{path} must contain JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def extract_metric_float(summary: dict[str, Any], key: str, *, default: float | None = None) -> float:
|
||||
totals = summary.get("totals")
|
||||
if isinstance(totals, dict) and key in totals:
|
||||
return float(totals.get(key) or 0.0)
|
||||
if key in summary:
|
||||
return float(summary.get(key) or 0.0)
|
||||
if default is not None:
|
||||
return default
|
||||
raise ValueError(f"missing required metric: {key}")
|
||||
|
||||
|
||||
def extract_metric_int(summary: dict[str, Any], key: str, *, default: int | None = None) -> int:
|
||||
totals = summary.get("totals")
|
||||
if isinstance(totals, dict) and key in totals:
|
||||
return int(totals.get(key) or 0)
|
||||
if key in summary:
|
||||
return int(summary.get(key) or 0)
|
||||
if default is not None:
|
||||
return default
|
||||
raise ValueError(f"missing required metric: {key}")
|
||||
|
||||
|
||||
def extract_limited_reason_count(summary: dict[str, Any], reason: str) -> int:
|
||||
distributions = summary.get("distributions")
|
||||
if not isinstance(distributions, dict):
|
||||
return 0
|
||||
limited = distributions.get("limited_reason_category")
|
||||
if not isinstance(limited, dict):
|
||||
return 0
|
||||
return int(limited.get(reason) or 0)
|
||||
|
||||
|
||||
def collect_metrics(path: Path) -> MetricBundle:
|
||||
summary = load_summary(path)
|
||||
notes: list[str] = []
|
||||
|
||||
run_id = str(summary.get("run_id", "")).strip() or "<unknown>"
|
||||
|
||||
questions_total: int | None = None
|
||||
try:
|
||||
questions_total = extract_metric_int(summary, "questions_total")
|
||||
except ValueError:
|
||||
notes.append("questions_total missing")
|
||||
|
||||
strict_pass_rate = extract_metric_float(summary, "strict_pass_rate", default=0.0)
|
||||
route_pass_rate = extract_metric_float(summary, "route_pass_rate", default=0.0)
|
||||
|
||||
http_error_count = extract_metric_int(summary, "http_error_count", default=0)
|
||||
explicit_execution_error_count = extract_metric_int(summary, "execution_error_count", default=-1)
|
||||
limited_execution_error_count = extract_limited_reason_count(summary, "execution_error")
|
||||
if explicit_execution_error_count >= 0:
|
||||
execution_error_count = explicit_execution_error_count
|
||||
else:
|
||||
execution_error_count = http_error_count + limited_execution_error_count
|
||||
notes.append("execution_error_count derived as http_error_count + limited_reason_category.execution_error")
|
||||
|
||||
explicit_false_factual_rate = extract_metric_float(summary, "false_factual_rate", default=-1.0)
|
||||
if explicit_false_factual_rate >= 0:
|
||||
false_factual_rate = explicit_false_factual_rate
|
||||
else:
|
||||
false_factual_count = extract_metric_int(summary, "false_factual_count", default=0)
|
||||
if questions_total and questions_total > 0:
|
||||
false_factual_rate = false_factual_count / questions_total
|
||||
notes.append("false_factual_rate derived from false_factual_count/questions_total")
|
||||
else:
|
||||
false_factual_rate = 0.0
|
||||
notes.append("false_factual_rate defaulted to 0.0 (no questions_total)")
|
||||
|
||||
return MetricBundle(
|
||||
summary_path=str(path),
|
||||
run_id=run_id,
|
||||
questions_total=questions_total,
|
||||
strict_pass_rate=strict_pass_rate,
|
||||
route_pass_rate=route_pass_rate,
|
||||
execution_error_count=execution_error_count,
|
||||
false_factual_rate=false_factual_rate,
|
||||
notes=notes,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
baseline_path = Path(args.baseline_summary).resolve()
|
||||
candidate_path = Path(args.candidate_summary).resolve()
|
||||
|
||||
baseline = collect_metrics(baseline_path)
|
||||
candidate = collect_metrics(candidate_path)
|
||||
epsilon = float(args.epsilon)
|
||||
|
||||
checks: list[dict[str, Any]] = []
|
||||
|
||||
def add_check(name: str, passed: bool, baseline_value: Any, candidate_value: Any, rule: str) -> None:
|
||||
checks.append(
|
||||
{
|
||||
"metric": name,
|
||||
"passed": passed,
|
||||
"baseline": baseline_value,
|
||||
"candidate": candidate_value,
|
||||
"rule": rule,
|
||||
}
|
||||
)
|
||||
|
||||
add_check(
|
||||
"strict_pass_rate",
|
||||
candidate.strict_pass_rate + epsilon >= baseline.strict_pass_rate,
|
||||
baseline.strict_pass_rate,
|
||||
candidate.strict_pass_rate,
|
||||
"candidate >= baseline",
|
||||
)
|
||||
add_check(
|
||||
"route_pass_rate",
|
||||
candidate.route_pass_rate + epsilon >= baseline.route_pass_rate,
|
||||
baseline.route_pass_rate,
|
||||
candidate.route_pass_rate,
|
||||
"candidate >= baseline",
|
||||
)
|
||||
add_check(
|
||||
"execution_error_count",
|
||||
candidate.execution_error_count <= baseline.execution_error_count,
|
||||
baseline.execution_error_count,
|
||||
candidate.execution_error_count,
|
||||
"candidate <= baseline",
|
||||
)
|
||||
add_check(
|
||||
"false_factual_rate",
|
||||
candidate.false_factual_rate <= baseline.false_factual_rate + epsilon,
|
||||
baseline.false_factual_rate,
|
||||
candidate.false_factual_rate,
|
||||
"candidate <= baseline",
|
||||
)
|
||||
|
||||
if baseline.questions_total is not None and candidate.questions_total is not None:
|
||||
add_check(
|
||||
"questions_total_match",
|
||||
candidate.questions_total == baseline.questions_total,
|
||||
baseline.questions_total,
|
||||
candidate.questions_total,
|
||||
"candidate == baseline",
|
||||
)
|
||||
|
||||
overall_pass = all(bool(item["passed"]) for item in checks)
|
||||
|
||||
report = {
|
||||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"overall_pass": overall_pass,
|
||||
"baseline": baseline.__dict__,
|
||||
"candidate": candidate.__dict__,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
if args.report_json:
|
||||
output_path = Path(args.report_json).resolve()
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"Baseline: {baseline.summary_path} ({baseline.run_id})")
|
||||
print(f"Candidate: {candidate.summary_path} ({candidate.run_id})")
|
||||
for item in checks:
|
||||
status = "PASS" if item["passed"] else "FAIL"
|
||||
print(
|
||||
f"[{status}] {item['metric']}: baseline={item['baseline']} candidate={item['candidate']} rule={item['rule']}"
|
||||
)
|
||||
|
||||
if baseline.notes:
|
||||
print("\nBaseline notes:")
|
||||
for note in baseline.notes:
|
||||
print(f"- {note}")
|
||||
if candidate.notes:
|
||||
print("\nCandidate notes:")
|
||||
for note in candidate.notes:
|
||||
print(f"- {note}")
|
||||
|
||||
if not overall_pass:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
param(
|
||||
[string]$TaskName = "NDC_ADDRESS_Nightly_Regression",
|
||||
[string]$StartTime = "03:30",
|
||||
[string]$PythonExe = "python",
|
||||
[string]$BackendUrl = "http://127.0.0.1:8787/api/assistant/message",
|
||||
[string]$LlmProvider = "local",
|
||||
[string]$LlmModel = "qwen2.5-14b-instruct-1m",
|
||||
[string]$LlmBaseUrl = "http://127.0.0.1:1234",
|
||||
[ValidateSet("semantic", "route", "factual")]
|
||||
[string]$StrictPolicy = "route",
|
||||
[string]$OwnerUser = "",
|
||||
[switch]$Unregister
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$RepoRoot = Split-Path -Parent $ScriptDir
|
||||
$NightlyWrapper = Join-Path $ScriptDir "run_address_nightly_regression.ps1"
|
||||
|
||||
if (-not (Test-Path $NightlyWrapper)) {
|
||||
throw "Nightly wrapper script not found: $NightlyWrapper"
|
||||
}
|
||||
|
||||
if ($Unregister) {
|
||||
if (Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue) {
|
||||
Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false
|
||||
Write-Host "Removed scheduled task: $TaskName"
|
||||
} else {
|
||||
Write-Host "Scheduled task not found: $TaskName"
|
||||
}
|
||||
exit 0
|
||||
}
|
||||
|
||||
if (-not $OwnerUser) {
|
||||
if ($env:USERDOMAIN -and $env:USERNAME) {
|
||||
$OwnerUser = "$($env:USERDOMAIN)\$($env:USERNAME)"
|
||||
} else {
|
||||
$OwnerUser = $env:USERNAME
|
||||
}
|
||||
}
|
||||
|
||||
$startDateTime = [DateTime]::ParseExact($StartTime, "HH:mm", $null)
|
||||
$taskTrigger = New-ScheduledTaskTrigger -Daily -At $startDateTime
|
||||
|
||||
$taskArgs = @(
|
||||
"-NoProfile",
|
||||
"-ExecutionPolicy", "Bypass",
|
||||
"-File", "`"$NightlyWrapper`"",
|
||||
"-PythonExe", "`"$PythonExe`"",
|
||||
"-BackendUrl", "`"$BackendUrl`"",
|
||||
"-LlmProvider", "`"$LlmProvider`"",
|
||||
"-LlmModel", "`"$LlmModel`"",
|
||||
"-LlmBaseUrl", "`"$LlmBaseUrl`"",
|
||||
"-StrictPolicy", "`"$StrictPolicy`""
|
||||
) -join " "
|
||||
|
||||
$taskAction = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $taskArgs -WorkingDirectory $RepoRoot
|
||||
$taskPrincipal = New-ScheduledTaskPrincipal -UserId $OwnerUser -LogonType Interactive -RunLevel Limited
|
||||
$taskSettings = New-ScheduledTaskSettingsSet -StartWhenAvailable -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries
|
||||
$taskObject = New-ScheduledTask -Action $taskAction -Trigger $taskTrigger -Principal $taskPrincipal -Settings $taskSettings
|
||||
|
||||
Register-ScheduledTask -TaskName $TaskName -InputObject $taskObject -Force | Out-Null
|
||||
|
||||
Write-Host "Scheduled task registered:"
|
||||
Write-Host "- Name: $TaskName"
|
||||
Write-Host "- Owner: $OwnerUser"
|
||||
Write-Host "- StartTime: $StartTime (daily)"
|
||||
Write-Host "- Action: powershell.exe $taskArgs"
|
||||
@@ -0,0 +1,50 @@
|
||||
param(
|
||||
[string]$PythonExe = "",
|
||||
[string]$BackendUrl = "http://127.0.0.1:8787/api/assistant/message",
|
||||
[string]$LlmProvider = "local",
|
||||
[string]$LlmModel = "qwen2.5-14b-instruct-1m",
|
||||
[string]$LlmBaseUrl = "http://127.0.0.1:1234",
|
||||
[ValidateSet("semantic", "route", "factual")]
|
||||
[string]$StrictPolicy = "route",
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$RepoRoot = Split-Path -Parent $ScriptDir
|
||||
|
||||
if (-not $PythonExe) {
|
||||
$PythonExe = "python"
|
||||
}
|
||||
|
||||
$NightlyScript = Join-Path $ScriptDir "run_address_nightly_regression.py"
|
||||
if (-not (Test-Path $NightlyScript)) {
|
||||
throw "Nightly script not found: $NightlyScript"
|
||||
}
|
||||
|
||||
$argsList = @(
|
||||
$NightlyScript,
|
||||
"--backend-url", $BackendUrl,
|
||||
"--llm-provider", $LlmProvider,
|
||||
"--llm-model", $LlmModel,
|
||||
"--llm-base-url", $LlmBaseUrl,
|
||||
"--strict-policy", $StrictPolicy
|
||||
)
|
||||
|
||||
if ($DryRun) {
|
||||
$argsList += "--dry-run"
|
||||
}
|
||||
|
||||
Write-Host "Running ADDRESS nightly regression from $RepoRoot"
|
||||
Write-Host "$PythonExe $($argsList -join ' ')"
|
||||
|
||||
Push-Location $RepoRoot
|
||||
try {
|
||||
& $PythonExe @argsList
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Nightly regression failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_OUTPUT_ROOT = PROJECT_ROOT / "docs" / "ADDRESS" / "runs"
|
||||
RUNNER_SCRIPT = PROJECT_ROOT / "scripts" / "run_address_live_slang_stress.py"
|
||||
VALIDATOR_SCRIPT = PROJECT_ROOT / "scripts" / "validate_address_run_pack.py"
|
||||
COMPARATOR_SCRIPT = PROJECT_ROOT / "scripts" / "compare_address_run_summary.py"
|
||||
|
||||
|
||||
@dataclass
|
||||
class NightlyPackConfig:
|
||||
name: str
|
||||
questions_file: Path
|
||||
baseline_summary: Path
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run ADDRESS nightly regression packs (102 + 25), validate run packs and compare vs baseline."
|
||||
)
|
||||
parser.add_argument("--backend-url", default="http://127.0.0.1:8787/api/assistant/message")
|
||||
parser.add_argument("--prompt-version", default="address_query_runtime_v1")
|
||||
parser.add_argument("--llm-provider", default="local")
|
||||
parser.add_argument("--llm-model", default="qwen2.5-14b-instruct-1m")
|
||||
parser.add_argument("--llm-base-url", default="http://127.0.0.1:1234")
|
||||
parser.add_argument("--temperature", type=float, default=0.0)
|
||||
parser.add_argument("--max-output-tokens", type=int, default=900)
|
||||
parser.add_argument("--timeout-sec", type=int, default=120)
|
||||
parser.add_argument("--strict-policy", default="route", choices=["semantic", "route", "factual"])
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
default=str(DEFAULT_OUTPUT_ROOT),
|
||||
help="Root where nightly bundle folder will be created.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--nightly-run-id",
|
||||
default="",
|
||||
help="Optional nightly bundle id. Default: <date>_Address_Nightly_Regression_<time>.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--python-bin",
|
||||
default=sys.executable,
|
||||
help="Python executable to run child scripts.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print planned commands and baseline checks without executing live requests.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-comparator",
|
||||
action="store_true",
|
||||
help="Run packs + validator only, skip baseline comparison.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def now_stamp() -> str:
|
||||
return datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
|
||||
|
||||
def run_command(cmd: list[str], *, cwd: Path, dry_run: bool) -> tuple[int, str, str]:
|
||||
rendered = " ".join(f'"{token}"' if " " in token else token for token in cmd)
|
||||
print(f"$ {rendered}")
|
||||
if dry_run:
|
||||
return 0, "", ""
|
||||
completed = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if completed.stdout:
|
||||
print(completed.stdout.strip())
|
||||
if completed.stderr:
|
||||
print(completed.stderr.strip())
|
||||
return completed.returncode, completed.stdout, completed.stderr
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
output_root = Path(args.output_root).resolve()
|
||||
nightly_run_id = args.nightly_run_id.strip() or f"{datetime.now().date().isoformat()}_Address_Nightly_Regression_{now_stamp()}"
|
||||
nightly_dir = output_root / nightly_run_id
|
||||
nightly_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
packs = [
|
||||
NightlyPackConfig(
|
||||
name="stress_102",
|
||||
questions_file=(PROJECT_ROOT / "docs" / "ADDRESS" / "question_sets" / "address_slang_stress_full_2026-04-02.json").resolve(),
|
||||
baseline_summary=(
|
||||
PROJECT_ROOT
|
||||
/ "docs"
|
||||
/ "ADDRESS"
|
||||
/ "runs"
|
||||
/ "2026-04-02_Address_Slang_Live_Stress_2026-04-02_12-57-27"
|
||||
/ "run_summary.json"
|
||||
).resolve(),
|
||||
),
|
||||
NightlyPackConfig(
|
||||
name="followup_25",
|
||||
questions_file=(PROJECT_ROOT / "docs" / "ADDRESS" / "question_sets" / "address_followup_context_chains_2026-04-02.json").resolve(),
|
||||
baseline_summary=(
|
||||
PROJECT_ROOT
|
||||
/ "docs"
|
||||
/ "ADDRESS"
|
||||
/ "runs"
|
||||
/ "2026-04-02_Address_Followup_Context_Chains_2026-04-02_19-15-Run5"
|
||||
/ "run_summary.json"
|
||||
).resolve(),
|
||||
),
|
||||
]
|
||||
|
||||
failures: list[str] = []
|
||||
packs_report: list[dict[str, Any]] = []
|
||||
|
||||
print(f"Nightly bundle: {nightly_dir}")
|
||||
for pack in packs:
|
||||
if not pack.questions_file.exists():
|
||||
failures.append(f"[{pack.name}] missing questions file: {pack.questions_file}")
|
||||
if not pack.baseline_summary.exists():
|
||||
failures.append(f"[{pack.name}] missing baseline summary: {pack.baseline_summary}")
|
||||
if failures:
|
||||
for item in failures:
|
||||
print(f"ERROR: {item}")
|
||||
raise SystemExit(1)
|
||||
|
||||
for pack in packs:
|
||||
stamp = datetime.now().strftime("%H-%M-%S")
|
||||
run_id = f"{datetime.now().date().isoformat()}_Address_Nightly_{pack.name}_{stamp}"
|
||||
run_dir = nightly_dir / run_id
|
||||
|
||||
row: dict[str, Any] = {
|
||||
"pack": pack.name,
|
||||
"run_id": run_id,
|
||||
"questions_file": str(pack.questions_file),
|
||||
"baseline_summary": str(pack.baseline_summary),
|
||||
"run_dir": str(run_dir),
|
||||
"runner_ok": False,
|
||||
"validator_ok": False,
|
||||
"comparator_ok": None if args.skip_comparator else False,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
runner_cmd = [
|
||||
str(Path(args.python_bin).resolve()),
|
||||
str(RUNNER_SCRIPT),
|
||||
"--questions-file",
|
||||
str(pack.questions_file),
|
||||
"--backend-url",
|
||||
args.backend_url,
|
||||
"--prompt-version",
|
||||
args.prompt_version,
|
||||
"--llm-provider",
|
||||
args.llm_provider,
|
||||
"--llm-model",
|
||||
args.llm_model,
|
||||
"--llm-base-url",
|
||||
args.llm_base_url,
|
||||
"--temperature",
|
||||
str(args.temperature),
|
||||
"--max-output-tokens",
|
||||
str(args.max_output_tokens),
|
||||
"--timeout-sec",
|
||||
str(args.timeout_sec),
|
||||
"--strict-policy",
|
||||
args.strict_policy,
|
||||
"--run-id",
|
||||
run_id,
|
||||
"--output-root",
|
||||
str(nightly_dir),
|
||||
]
|
||||
code, _, _ = run_command(runner_cmd, cwd=PROJECT_ROOT, dry_run=args.dry_run)
|
||||
if code != 0:
|
||||
row["errors"].append(f"runner failed with exit code {code}")
|
||||
packs_report.append(row)
|
||||
continue
|
||||
row["runner_ok"] = True
|
||||
|
||||
validator_report = nightly_dir / f"{run_id}_validator_report.json"
|
||||
validator_cmd = [
|
||||
str(Path(args.python_bin).resolve()),
|
||||
str(VALIDATOR_SCRIPT),
|
||||
str(run_dir),
|
||||
"--report-json",
|
||||
str(validator_report),
|
||||
]
|
||||
code, _, _ = run_command(validator_cmd, cwd=PROJECT_ROOT, dry_run=args.dry_run)
|
||||
if code != 0:
|
||||
row["errors"].append(f"validator failed with exit code {code}")
|
||||
packs_report.append(row)
|
||||
continue
|
||||
row["validator_ok"] = True
|
||||
row["validator_report"] = str(validator_report)
|
||||
|
||||
if not args.skip_comparator:
|
||||
comparator_report = nightly_dir / f"{run_id}_comparator_report.json"
|
||||
comparator_cmd = [
|
||||
str(Path(args.python_bin).resolve()),
|
||||
str(COMPARATOR_SCRIPT),
|
||||
"--baseline-summary",
|
||||
str(pack.baseline_summary),
|
||||
"--candidate-summary",
|
||||
str(run_dir / "run_summary.json"),
|
||||
"--report-json",
|
||||
str(comparator_report),
|
||||
]
|
||||
code, _, _ = run_command(comparator_cmd, cwd=PROJECT_ROOT, dry_run=args.dry_run)
|
||||
if code != 0:
|
||||
row["errors"].append(f"comparator failed with exit code {code}")
|
||||
packs_report.append(row)
|
||||
continue
|
||||
row["comparator_ok"] = True
|
||||
row["comparator_report"] = str(comparator_report)
|
||||
|
||||
packs_report.append(row)
|
||||
|
||||
overall_ok = all(item.get("runner_ok") and item.get("validator_ok") and (args.skip_comparator or item.get("comparator_ok")) for item in packs_report)
|
||||
if len(packs_report) != len(packs):
|
||||
overall_ok = False
|
||||
|
||||
report = {
|
||||
"nightly_run_id": nightly_run_id,
|
||||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"dry_run": bool(args.dry_run),
|
||||
"overall_ok": overall_ok,
|
||||
"strict_policy": args.strict_policy,
|
||||
"packs": packs_report,
|
||||
}
|
||||
|
||||
report_path = nightly_dir / "nightly_summary.json"
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
md_lines = [
|
||||
f"# {nightly_run_id}",
|
||||
"",
|
||||
f"Generated at: {report['generated_at']}",
|
||||
f"Dry run: {report['dry_run']}",
|
||||
f"Strict policy: {report['strict_policy']}",
|
||||
f"Overall: {'PASS' if overall_ok else 'FAIL'}",
|
||||
"",
|
||||
"## Packs",
|
||||
]
|
||||
for item in packs_report:
|
||||
md_lines.extend(
|
||||
[
|
||||
f"### {item['pack']}",
|
||||
f"- run_id: {item['run_id']}",
|
||||
f"- runner_ok: {item['runner_ok']}",
|
||||
f"- validator_ok: {item['validator_ok']}",
|
||||
f"- comparator_ok: {item['comparator_ok']}",
|
||||
f"- run_dir: {item['run_dir']}",
|
||||
]
|
||||
)
|
||||
if item.get("errors"):
|
||||
md_lines.append("- errors:")
|
||||
for err in item["errors"]:
|
||||
md_lines.append(f" - {err}")
|
||||
md_lines.append("")
|
||||
|
||||
(nightly_dir / "README.md").write_text("\n".join(md_lines) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"\nNightly summary: {report_path}")
|
||||
print(f"Overall: {'PASS' if overall_ok else 'FAIL'}")
|
||||
if not overall_ok:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REQUIRED_FILES = ("run_summary.json", "full_live_results.json", "failures_only.json", "README.md")
|
||||
REQUIRED_TOTAL_KEYS = (
|
||||
"questions_total",
|
||||
"semantic_pass_count",
|
||||
"route_pass_count",
|
||||
"strict_pass_count",
|
||||
"factual_count",
|
||||
"partial_coverage_count",
|
||||
"http_error_count",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunValidationResult:
|
||||
run_dir: str
|
||||
valid: bool
|
||||
errors: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
metrics: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate ADDRESS run-pack structure and summary consistency."
|
||||
)
|
||||
parser.add_argument(
|
||||
"run_dirs",
|
||||
nargs="+",
|
||||
help="One or more run directories (for example docs/ADDRESS/runs/<run_id>).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-legacy-summary",
|
||||
action="store_true",
|
||||
help="Allow minimal/legacy run_summary format (without totals).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--report-json",
|
||||
default="",
|
||||
help="Optional path to write full validation report JSON.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_json(path: Path) -> tuple[dict[str, Any] | list[Any] | None, str | None]:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except FileNotFoundError:
|
||||
return None, f"missing file: {path.name}"
|
||||
except json.JSONDecodeError as exc:
|
||||
return None, f"invalid json in {path.name}: {exc}"
|
||||
return payload, None
|
||||
|
||||
|
||||
def validate_totals(totals: dict[str, Any], errors: list[str]) -> dict[str, Any]:
|
||||
metrics: dict[str, Any] = {}
|
||||
missing = [key for key in REQUIRED_TOTAL_KEYS if key not in totals]
|
||||
if missing:
|
||||
errors.append(f"run_summary.totals missing keys: {', '.join(missing)}")
|
||||
return metrics
|
||||
|
||||
questions_total = int(totals.get("questions_total", 0) or 0)
|
||||
metrics["questions_total"] = questions_total
|
||||
metrics["route_pass_rate"] = float(totals.get("route_pass_rate", 0.0) or 0.0)
|
||||
metrics["strict_pass_rate"] = float(totals.get("strict_pass_rate", 0.0) or 0.0)
|
||||
metrics["http_error_count"] = int(totals.get("http_error_count", 0) or 0)
|
||||
|
||||
if questions_total <= 0:
|
||||
errors.append("run_summary.totals.questions_total must be > 0")
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
def validate_single_run(run_dir: Path, allow_legacy_summary: bool) -> RunValidationResult:
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
metrics: dict[str, Any] = {}
|
||||
|
||||
if not run_dir.exists() or not run_dir.is_dir():
|
||||
return RunValidationResult(run_dir=str(run_dir), valid=False, errors=["run directory does not exist"])
|
||||
|
||||
missing_files = [name for name in REQUIRED_FILES if not (run_dir / name).exists()]
|
||||
if missing_files:
|
||||
errors.append(f"missing required files: {', '.join(missing_files)}")
|
||||
|
||||
summary_obj, summary_err = load_json(run_dir / "run_summary.json")
|
||||
full_obj, full_err = load_json(run_dir / "full_live_results.json")
|
||||
failures_obj, failures_err = load_json(run_dir / "failures_only.json")
|
||||
|
||||
if summary_err:
|
||||
errors.append(summary_err)
|
||||
if full_err:
|
||||
errors.append(full_err)
|
||||
if failures_err:
|
||||
errors.append(failures_err)
|
||||
|
||||
readme_path = run_dir / "README.md"
|
||||
if readme_path.exists():
|
||||
content = readme_path.read_text(encoding="utf-8-sig").strip()
|
||||
if not content:
|
||||
errors.append("README.md is empty")
|
||||
|
||||
summary = summary_obj if isinstance(summary_obj, dict) else None
|
||||
full = full_obj if isinstance(full_obj, dict) else None
|
||||
failures = failures_obj if isinstance(failures_obj, list) else None
|
||||
|
||||
if summary is None and summary_obj is not None:
|
||||
errors.append("run_summary.json must contain object")
|
||||
if full is None and full_obj is not None:
|
||||
errors.append("full_live_results.json must contain object")
|
||||
if failures is None and failures_obj is not None:
|
||||
errors.append("failures_only.json must contain array")
|
||||
|
||||
if summary:
|
||||
run_id = str(summary.get("run_id", "")).strip()
|
||||
if not run_id:
|
||||
errors.append("run_summary.run_id is required")
|
||||
else:
|
||||
metrics["run_id"] = run_id
|
||||
if run_id != run_dir.name:
|
||||
warnings.append(f"run_id ({run_id}) differs from directory name ({run_dir.name})")
|
||||
|
||||
if "generated_at" not in summary and "date" not in summary:
|
||||
errors.append("run_summary must contain generated_at or date")
|
||||
|
||||
totals = summary.get("totals")
|
||||
if isinstance(totals, dict):
|
||||
metrics.update(validate_totals(totals, errors))
|
||||
elif not allow_legacy_summary:
|
||||
errors.append("run_summary.totals is required")
|
||||
else:
|
||||
warnings.append("legacy run_summary format (without totals) accepted")
|
||||
|
||||
rows: list[Any] = []
|
||||
if full:
|
||||
full_run_id = str(full.get("run_id", "")).strip()
|
||||
if full_run_id and summary and str(summary.get("run_id", "")).strip() and full_run_id != str(summary.get("run_id")).strip():
|
||||
errors.append("run_id mismatch between run_summary.json and full_live_results.json")
|
||||
rows_obj = full.get("rows")
|
||||
if not isinstance(rows_obj, list):
|
||||
errors.append("full_live_results.rows must be array")
|
||||
else:
|
||||
rows = rows_obj
|
||||
metrics["rows_count"] = len(rows)
|
||||
|
||||
if failures is not None:
|
||||
metrics["failures_count"] = len(failures)
|
||||
|
||||
questions_total = metrics.get("questions_total")
|
||||
if isinstance(questions_total, int) and rows:
|
||||
if questions_total != len(rows):
|
||||
errors.append(
|
||||
f"questions_total mismatch: run_summary.totals.questions_total={questions_total}, full_live_results.rows={len(rows)}"
|
||||
)
|
||||
|
||||
if isinstance(questions_total, int) and isinstance(metrics.get("failures_count"), int):
|
||||
if int(metrics["failures_count"]) > questions_total:
|
||||
errors.append("failures_only count exceeds questions_total")
|
||||
|
||||
return RunValidationResult(run_dir=str(run_dir), valid=not errors, errors=errors, warnings=warnings, metrics=metrics)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
run_dirs = [Path(p).resolve() for p in args.run_dirs]
|
||||
results = [validate_single_run(path, allow_legacy_summary=bool(args.allow_legacy_summary)) for path in run_dirs]
|
||||
|
||||
total = len(results)
|
||||
passed = sum(1 for item in results if item.valid)
|
||||
failed = total - passed
|
||||
|
||||
report = {
|
||||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"total": total,
|
||||
"passed": passed,
|
||||
"failed": failed,
|
||||
"results": [
|
||||
{
|
||||
"run_dir": item.run_dir,
|
||||
"valid": item.valid,
|
||||
"errors": item.errors,
|
||||
"warnings": item.warnings,
|
||||
"metrics": item.metrics,
|
||||
}
|
||||
for item in results
|
||||
],
|
||||
}
|
||||
|
||||
if args.report_json:
|
||||
report_path = Path(args.report_json).resolve()
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
for item in results:
|
||||
status = "PASS" if item.valid else "FAIL"
|
||||
print(f"[{status}] {item.run_dir}")
|
||||
for warning in item.warnings:
|
||||
print(f" warning: {warning}")
|
||||
for error in item.errors:
|
||||
print(f" error: {error}")
|
||||
|
||||
print(f"\nValidated run packs: {total}, passed: {passed}, failed: {failed}")
|
||||
if failed:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user