feat(perception): qualify M4.8T semantic identity
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the frozen RF-DETR risk contour on independent COCO 2017 val truth."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from k1link.perception.m48t_risk_quality import (
|
||||
CocoRiskImage,
|
||||
RiskPrediction,
|
||||
RiskTruth,
|
||||
load_coco_risk_truth,
|
||||
load_m48t_risk_quality_profile,
|
||||
score_risk_quality,
|
||||
)
|
||||
from k1link.perception.rf_detr_object_detector import (
|
||||
TritonRfDetrHttpInferenceBackend,
|
||||
postprocess_rf_detr,
|
||||
preprocess_raw_kb4_rf_detr,
|
||||
)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--annotations", type=Path, required=True)
|
||||
parser.add_argument("--images-root", type=Path, required=True)
|
||||
parser.add_argument("--triton-origin", default="http://127.0.0.1:8000")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--predictions", type=Path, required=True)
|
||||
parser.add_argument("--failures", type=Path, required=True)
|
||||
parser.add_argument("--progress", type=Path, required=True)
|
||||
parser.add_argument("--review-root", type=Path, required=True)
|
||||
parser.add_argument("--runtime-artifact-sha256", required=True)
|
||||
parser.add_argument("--runner-sha256", required=True)
|
||||
parser.add_argument("--images-archive-sha256", required=True)
|
||||
parser.add_argument("--annotations-document-sha256", required=True)
|
||||
parser.add_argument("--maximum-images", type=int, default=0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
_require_sha256(arguments.runtime_artifact_sha256, "runtime artifact")
|
||||
_require_sha256(arguments.runner_sha256, "runner")
|
||||
_require_sha256(arguments.images_archive_sha256, "images archive")
|
||||
_require_sha256(arguments.annotations_document_sha256, "annotations document")
|
||||
if arguments.maximum_images < 0:
|
||||
raise RuntimeError("maximum images cannot be negative")
|
||||
for target in (
|
||||
arguments.output,
|
||||
arguments.predictions,
|
||||
arguments.failures,
|
||||
arguments.progress,
|
||||
):
|
||||
if target.exists():
|
||||
raise RuntimeError(f"output already exists: {target}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if arguments.review_root.exists():
|
||||
raise RuntimeError("review output already exists")
|
||||
arguments.review_root.mkdir(parents=True)
|
||||
|
||||
profile = load_m48t_risk_quality_profile(arguments.profile)
|
||||
images, truth = load_coco_risk_truth(arguments.annotations, profile)
|
||||
if arguments.maximum_images:
|
||||
images = images[: arguments.maximum_images]
|
||||
image_ids = {item.image_id for item in images}
|
||||
truth = tuple(item for item in truth if item.image_id in image_ids)
|
||||
if not images or not truth:
|
||||
raise RuntimeError("selected COCO risk set is empty")
|
||||
|
||||
mask = np.ones((600, 800), dtype=np.bool_)
|
||||
class_to_family = profile.class_to_family
|
||||
predictions: list[RiskPrediction] = []
|
||||
image_timings_ms: list[float] = []
|
||||
inference_timings_ms: list[float] = []
|
||||
rejected: Counter[str] = Counter()
|
||||
started_at = time.time_ns()
|
||||
backend = TritonRfDetrHttpInferenceBackend(arguments.triton_origin)
|
||||
with arguments.progress.open("x", encoding="utf-8") as progress:
|
||||
try:
|
||||
for image_index, image in enumerate(images, start=1):
|
||||
loop_started = time.perf_counter_ns()
|
||||
image_path = (arguments.images_root / image.file_name).resolve(strict=True)
|
||||
with Image.open(image_path) as opened:
|
||||
rgb = np.asarray(
|
||||
opened.convert("RGB").resize((800, 600), Image.Resampling.BILINEAR),
|
||||
dtype=np.uint8,
|
||||
)
|
||||
image_bgr = np.ascontiguousarray(rgb[:, :, ::-1])
|
||||
tensor = preprocess_raw_kb4_rf_detr(image_bgr, mask)
|
||||
inference_started = time.perf_counter_ns()
|
||||
output = backend.infer(tensor)
|
||||
inference_ms = (time.perf_counter_ns() - inference_started) / 1_000_000
|
||||
inference_timings_ms.append(inference_ms)
|
||||
processed = postprocess_rf_detr(output, mask)
|
||||
rejected.update(dict(processed.rejected))
|
||||
for detection_index, detection in enumerate(processed.detections, start=1):
|
||||
family = class_to_family.get(detection.label)
|
||||
if family is None:
|
||||
raise RuntimeError(
|
||||
"RF-DETR emitted a class outside the frozen risk profile"
|
||||
)
|
||||
predictions.append(
|
||||
RiskPrediction(
|
||||
image_id=image.image_id,
|
||||
prediction_id=f"{image.image_id:012d}:{detection_index:03d}",
|
||||
class_name=detection.label,
|
||||
family=family,
|
||||
score=detection.score,
|
||||
bbox_xyxy=detection.bbox_xyxy,
|
||||
)
|
||||
)
|
||||
image_ms = (time.perf_counter_ns() - loop_started) / 1_000_000
|
||||
image_timings_ms.append(image_ms)
|
||||
if image_index == 1 or image_index % 100 == 0 or image_index == len(images):
|
||||
progress.write(
|
||||
_canonical_json(
|
||||
{
|
||||
"schema_version": "missioncore.m48t-risk-quality-progress/v1",
|
||||
"completed_images": image_index,
|
||||
"total_images": len(images),
|
||||
"prediction_count": len(predictions),
|
||||
"last_image_ms": image_ms,
|
||||
"last_inference_ms": inference_ms,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
progress.flush()
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
result = score_risk_quality(
|
||||
images=images,
|
||||
truth=truth,
|
||||
predictions=tuple(predictions),
|
||||
profile=profile,
|
||||
)
|
||||
completed_at = time.time_ns()
|
||||
prediction_rows = tuple(_prediction_row(item) for item in predictions)
|
||||
_write_jsonl(arguments.predictions, prediction_rows)
|
||||
_write_jsonl(arguments.failures, result.failures)
|
||||
review_files = _render_review_cases(
|
||||
images_root=arguments.images_root,
|
||||
review_root=arguments.review_root,
|
||||
images=images,
|
||||
truth=truth,
|
||||
predictions=tuple(predictions),
|
||||
failures=result.failures,
|
||||
)
|
||||
report = dict(result.report)
|
||||
report["execution"] = {
|
||||
"worker": "DESKTOP-OPJ8J04",
|
||||
"started_at_unix_ns": started_at,
|
||||
"completed_at_unix_ns": completed_at,
|
||||
"duration_seconds": (completed_at - started_at) / 1_000_000_000,
|
||||
"image_timing_ms": _distribution(image_timings_ms),
|
||||
"triton_inference_ms": _distribution(inference_timings_ms),
|
||||
"effective_images_per_second": len(images)
|
||||
/ ((completed_at - started_at) / 1_000_000_000),
|
||||
"timing_is_admission_evidence": False,
|
||||
}
|
||||
report["provenance"] = {
|
||||
"profile_sha256": profile.profile_sha256,
|
||||
"annotations_document_sha256": _sha256(arguments.annotations),
|
||||
"images_archive_sha256": arguments.images_archive_sha256,
|
||||
"annotations_document_expected_sha256": arguments.annotations_document_sha256,
|
||||
"runtime_artifact_sha256": arguments.runtime_artifact_sha256,
|
||||
"runner_sha256": arguments.runner_sha256,
|
||||
"predictions_sha256": _sha256(arguments.predictions),
|
||||
"failures_sha256": _sha256(arguments.failures),
|
||||
}
|
||||
report["artifacts"] = {
|
||||
"predictions": arguments.predictions.name,
|
||||
"failures": arguments.failures.name,
|
||||
"progress": arguments.progress.name,
|
||||
"review_files": review_files,
|
||||
}
|
||||
report["detector_rejections"] = dict(sorted(rejected.items()))
|
||||
report["report_identity_sha256"] = hashlib.sha256(
|
||||
_canonical_json(
|
||||
{
|
||||
"profile_sha256": profile.profile_sha256,
|
||||
"dataset": report["dataset"],
|
||||
"counts": report["counts"],
|
||||
"metrics": report["metrics"],
|
||||
"failure_buckets": report["failure_buckets"],
|
||||
"quality_gates": report["quality_gates"],
|
||||
"provenance": report["provenance"],
|
||||
}
|
||||
).encode()
|
||||
).hexdigest()
|
||||
arguments.output.write_text(_canonical_json(report) + "\n", "utf-8")
|
||||
quality_gates = report.get("quality_gates")
|
||||
if not isinstance(quality_gates, dict) or not isinstance(
|
||||
quality_gates.get("passed"), bool
|
||||
):
|
||||
raise RuntimeError("quality gate result is invalid")
|
||||
print(
|
||||
_canonical_json(
|
||||
{
|
||||
"result": str(arguments.output),
|
||||
"risk_images": len(images),
|
||||
"truth_instances": len(truth),
|
||||
"predictions": len(predictions),
|
||||
"quality_gate_passed": quality_gates["passed"],
|
||||
"report_identity_sha256": report["report_identity_sha256"],
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def _prediction_row(item: RiskPrediction) -> dict[str, object]:
|
||||
return {
|
||||
"image_id": item.image_id,
|
||||
"prediction_id": item.prediction_id,
|
||||
"class_name": item.class_name,
|
||||
"family": item.family,
|
||||
"score": item.score,
|
||||
"bbox_xyxy": list(item.bbox_xyxy),
|
||||
}
|
||||
|
||||
|
||||
def _render_review_cases(
|
||||
*,
|
||||
images_root: Path,
|
||||
review_root: Path,
|
||||
images: tuple[CocoRiskImage, ...],
|
||||
truth: tuple[RiskTruth, ...],
|
||||
predictions: tuple[RiskPrediction, ...],
|
||||
failures: tuple[dict[str, object], ...],
|
||||
) -> list[str]:
|
||||
image_by_id = {item.image_id: item for item in images}
|
||||
selected_ids: list[int] = []
|
||||
for failure in failures:
|
||||
if failure.get("kind") != "false-negative":
|
||||
continue
|
||||
raw_image_id = failure.get("image_id")
|
||||
if not isinstance(raw_image_id, int) or isinstance(raw_image_id, bool):
|
||||
raise RuntimeError("failure image id is invalid")
|
||||
image_id = raw_image_id
|
||||
if image_id not in selected_ids:
|
||||
selected_ids.append(image_id)
|
||||
if len(selected_ids) == 16:
|
||||
break
|
||||
result: list[str] = []
|
||||
for image_id in selected_ids:
|
||||
metadata = image_by_id[image_id]
|
||||
with Image.open((images_root / metadata.file_name).resolve(strict=True)) as opened:
|
||||
canvas = opened.convert("RGB").resize((800, 600), Image.Resampling.BILINEAR)
|
||||
draw = ImageDraw.Draw(canvas)
|
||||
for truth_item in truth:
|
||||
if truth_item.image_id != image_id:
|
||||
continue
|
||||
draw.rectangle(truth_item.bbox_xyxy, outline=(80, 230, 120), width=3)
|
||||
draw.text(
|
||||
(truth_item.bbox_xyxy[0] + 2, truth_item.bbox_xyxy[1] + 2),
|
||||
f"GT {truth_item.class_name}",
|
||||
fill=(80, 230, 120),
|
||||
)
|
||||
for prediction_item in predictions:
|
||||
if prediction_item.image_id != image_id:
|
||||
continue
|
||||
draw.rectangle(prediction_item.bbox_xyxy, outline=(255, 200, 50), width=2)
|
||||
draw.text(
|
||||
(prediction_item.bbox_xyxy[0] + 2, prediction_item.bbox_xyxy[3] - 13),
|
||||
f"P {prediction_item.class_name} {prediction_item.score:.2f}",
|
||||
fill=(255, 200, 50),
|
||||
)
|
||||
name = f"review-{image_id:012d}.jpg"
|
||||
canvas.save(review_root / name, format="JPEG", quality=90, optimize=True)
|
||||
result.append(f"review/{name}")
|
||||
return result
|
||||
|
||||
|
||||
def _distribution(values: list[float]) -> dict[str, float]:
|
||||
if not values:
|
||||
raise RuntimeError("timing distribution is empty")
|
||||
ordered = sorted(values)
|
||||
return {
|
||||
"count": float(len(ordered)),
|
||||
"mean": statistics.fmean(ordered),
|
||||
"p50": _percentile(ordered, 0.50),
|
||||
"p95": _percentile(ordered, 0.95),
|
||||
"p99": _percentile(ordered, 0.99),
|
||||
"maximum": ordered[-1],
|
||||
}
|
||||
|
||||
|
||||
def _percentile(values: list[float], quantile: float) -> float:
|
||||
position = (len(values) - 1) * quantile
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return values[lower]
|
||||
return values[lower] + (values[upper] - values[lower]) * (position - lower)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: tuple[dict[str, object], ...]) -> None:
|
||||
with path.open("x", encoding="utf-8") as handle:
|
||||
for row in rows:
|
||||
handle.write(_canonical_json(row) + "\n")
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _require_sha256(value: str, label: str) -> None:
|
||||
if len(value) != 64 or any(character not in "0123456789abcdef" for character in value):
|
||||
raise RuntimeError(f"{label} SHA-256 is invalid")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay M4.8T semantic stabilization over geometry-owned component ids."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.perception.m48t_risk_quality import (
|
||||
BoundedTemporalSemanticIdentity,
|
||||
TemporalSemanticObservation,
|
||||
load_m48t_risk_quality_profile,
|
||||
)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
repository = Path(__file__).resolve().parents[2]
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
type=Path,
|
||||
default=repository / "config/perception/m48t-risk-quality-temporal-v1.json",
|
||||
)
|
||||
parser.add_argument("--frames", type=Path, required=True)
|
||||
parser.add_argument("--expected-frames-sha256", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
if arguments.output.exists():
|
||||
raise RuntimeError("temporal semantic output already exists")
|
||||
if not _is_sha256(arguments.expected_frames_sha256):
|
||||
raise RuntimeError("expected frame ledger SHA-256 is invalid")
|
||||
profile = load_m48t_risk_quality_profile(arguments.profile)
|
||||
stabilizer = BoundedTemporalSemanticIdentity(profile)
|
||||
digest = hashlib.sha256()
|
||||
frames = 0
|
||||
publications = 0
|
||||
semantic_current = 0
|
||||
selected_publications = 0
|
||||
raw_switches = 0
|
||||
stable_switches = 0
|
||||
raw_family_switches = 0
|
||||
stable_family_switches = 0
|
||||
rolling_retained_skipped = 0
|
||||
resolutions: Counter[str] = Counter()
|
||||
stable_families: Counter[str] = Counter()
|
||||
last_raw: dict[str, str] = {}
|
||||
last_stable: dict[str, str] = {}
|
||||
class_to_family = profile.class_to_family
|
||||
|
||||
with arguments.frames.expanduser().resolve(strict=True).open("rb") as handle:
|
||||
for line_number, raw_line in enumerate(handle, start=1):
|
||||
digest.update(raw_line)
|
||||
try:
|
||||
document = json.loads(raw_line)
|
||||
frame_time_ns = document["source_envelope"]["timestamps"]["source_ns"]
|
||||
occupied = document["delivery"]["obstacle_map"]["occupied"]
|
||||
except (KeyError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"frame ledger row {line_number} is invalid") from exc
|
||||
if not isinstance(frame_time_ns, int) or not isinstance(occupied, list):
|
||||
raise RuntimeError(f"frame ledger row {line_number} contract changed")
|
||||
frames += 1
|
||||
for value in occupied:
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError("temporal obstacle is invalid")
|
||||
component_id = value.get("component_id")
|
||||
state = value.get("state")
|
||||
if state == "retained":
|
||||
rolling_retained_skipped += 1
|
||||
continue
|
||||
raw_class = value.get("semantic_hint") if state == "current" else None
|
||||
if not isinstance(component_id, str) or state not in {
|
||||
"current",
|
||||
"held",
|
||||
"expired",
|
||||
}:
|
||||
raise RuntimeError("temporal obstacle identity or state changed")
|
||||
if raw_class is not None and not isinstance(raw_class, str):
|
||||
raise RuntimeError("temporal semantic hint is invalid")
|
||||
if raw_class is not None:
|
||||
semantic_current += 1
|
||||
previous_raw = last_raw.get(component_id)
|
||||
if previous_raw is not None and previous_raw != raw_class:
|
||||
raw_switches += 1
|
||||
if class_to_family[previous_raw] != class_to_family[raw_class]:
|
||||
raw_family_switches += 1
|
||||
last_raw[component_id] = raw_class
|
||||
result = stabilizer.update(
|
||||
TemporalSemanticObservation(
|
||||
component_id=component_id,
|
||||
evidence_time_ns=frame_time_ns,
|
||||
raw_class_name=raw_class,
|
||||
currentness=state,
|
||||
)
|
||||
)
|
||||
publications += 1
|
||||
resolutions[result.resolution] += 1
|
||||
if result.selected_class_name is not None:
|
||||
selected_publications += 1
|
||||
if result.selected_family is None:
|
||||
raise RuntimeError("selected semantic family is missing")
|
||||
stable_families[result.selected_family] += 1
|
||||
previous_stable = last_stable.get(component_id)
|
||||
if (
|
||||
previous_stable is not None
|
||||
and previous_stable != result.selected_class_name
|
||||
):
|
||||
stable_switches += 1
|
||||
if (
|
||||
class_to_family[previous_stable]
|
||||
!= class_to_family[result.selected_class_name]
|
||||
):
|
||||
stable_family_switches += 1
|
||||
last_stable[component_id] = result.selected_class_name
|
||||
|
||||
frames_sha256 = digest.hexdigest()
|
||||
if frames_sha256 != arguments.expected_frames_sha256:
|
||||
raise RuntimeError("temporal frame ledger SHA-256 changed")
|
||||
snapshot = stabilizer.snapshot()
|
||||
gate_checks = {
|
||||
"all-publications-accounted": snapshot.input_observations == publications,
|
||||
"bounded-active-components": (
|
||||
snapshot.peak_active_components <= profile.temporal.maximum_active_components
|
||||
),
|
||||
"stable-switches-not-greater-than-raw-switches": stable_switches <= raw_switches,
|
||||
"stable-family-switches-not-greater-than-raw-family-switches": (
|
||||
stable_family_switches <= raw_family_switches
|
||||
),
|
||||
"association-remains-class-independent": True,
|
||||
"occupancy-remains-class-independent": True,
|
||||
}
|
||||
report = {
|
||||
"schema_version": "missioncore.m48t-temporal-semantic-shadow/v1",
|
||||
"profile_id": profile.profile_id,
|
||||
"profile_sha256": profile.profile_sha256,
|
||||
"source": {
|
||||
"source_id": "RAVNOVES00",
|
||||
"frame_ledger": str(arguments.frames),
|
||||
"frame_ledger_sha256": frames_sha256,
|
||||
"frames": frames,
|
||||
"publications": publications,
|
||||
"independent_semantic_truth_available": False,
|
||||
},
|
||||
"metrics": {
|
||||
"semantic_current_observations": semantic_current,
|
||||
"rolling_retained_publications_skipped": rolling_retained_skipped,
|
||||
"selected_publications": selected_publications,
|
||||
"raw_class_switches": raw_switches,
|
||||
"stable_class_switches": stable_switches,
|
||||
"suppressed_or_deferred_class_switches": raw_switches - stable_switches,
|
||||
"raw_family_switches": raw_family_switches,
|
||||
"stable_family_switches": stable_family_switches,
|
||||
"suppressed_or_deferred_family_switches": (
|
||||
raw_family_switches - stable_family_switches
|
||||
),
|
||||
"resolutions": dict(sorted(resolutions.items())),
|
||||
"stable_family_publications": dict(sorted(stable_families.items())),
|
||||
"snapshot": {
|
||||
field: getattr(snapshot, field)
|
||||
for field in snapshot.__dataclass_fields__
|
||||
},
|
||||
},
|
||||
"temporal_invariant_gate": {
|
||||
"checks": gate_checks,
|
||||
"passed": all(gate_checks.values()),
|
||||
"semantic_quality_accepted": False,
|
||||
},
|
||||
"policy": {
|
||||
"initial_confirmation_observations": (
|
||||
profile.temporal.initial_confirmation_observations
|
||||
),
|
||||
"switch_confirmation_observations": (
|
||||
profile.temporal.switch_confirmation_observations
|
||||
),
|
||||
"semantic_hold_seconds": profile.temporal.semantic_hold_seconds,
|
||||
"state_expiry_seconds": profile.temporal.state_expiry_seconds,
|
||||
"history_size": profile.temporal.history_size,
|
||||
"maximum_active_components": profile.temporal.maximum_active_components,
|
||||
"cross_family_conflict_fallback": "unknown",
|
||||
"association_uses_semantic_class": False,
|
||||
"occupancy_uses_semantic_class": False,
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth_for_ravnoves00": False,
|
||||
"candidate_accepted": False,
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
arguments.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
"utf-8",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(arguments.output),
|
||||
"frames": frames,
|
||||
"publications": publications,
|
||||
"raw_class_switches": raw_switches,
|
||||
"stable_class_switches": stable_switches,
|
||||
"invariant_gate_passed": all(gate_checks.values()),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _is_sha256(value: object) -> bool:
|
||||
return (
|
||||
isinstance(value, str)
|
||||
and len(value) == 64
|
||||
and all(character in "0123456789abcdef" for character in value)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,290 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ReleaseRoot,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||
[string]$ExpectedWheelSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
||||
[string]$RunId,
|
||||
[ValidateRange(0, 5000)]
|
||||
[int]$MaximumImages = 0,
|
||||
[string]$DatasetRoot = "D:\NDC_MISSIONCORE\datasets\coco-2017-val",
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48t-risk-quality"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
||||
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
||||
$null = New-Item -ItemType Directory -Path $Path
|
||||
}
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if (
|
||||
-not $item.PSIsContainer -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
||||
) {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Assert-RegularFile([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "$Label must be a regular file"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) {
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Ensure-PinnedMirrorFile(
|
||||
[string]$Path,
|
||||
[string]$Url,
|
||||
[long]$ExpectedLength,
|
||||
[string]$ExpectedSha256,
|
||||
[string]$Label
|
||||
) {
|
||||
if (Test-Path -LiteralPath $Path -PathType Leaf) {
|
||||
if (
|
||||
(Get-Item -LiteralPath $Path).Length -ne $ExpectedLength -or
|
||||
(Get-Sha256 $Path) -cne $ExpectedSha256
|
||||
) {
|
||||
Remove-Item -LiteralPath $Path -Force
|
||||
}
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
$partial = $Path + ".partial"
|
||||
if (Test-Path -LiteralPath $partial) { Remove-Item -LiteralPath $partial -Force }
|
||||
& curl.exe --fail --location --retry 5 `
|
||||
--speed-limit 1048576 --speed-time 30 --output $partial $Url
|
||||
Assert-LastExitCode "$Label download"
|
||||
if (
|
||||
(Get-Item -LiteralPath $partial).Length -ne $ExpectedLength -or
|
||||
(Get-Sha256 $partial) -cne $ExpectedSha256
|
||||
) {
|
||||
throw "$Label length or SHA-256 changed"
|
||||
}
|
||||
Move-Item -LiteralPath $partial -Destination $Path
|
||||
}
|
||||
if (
|
||||
(Get-Item -LiteralPath $Path).Length -ne $ExpectedLength -or
|
||||
(Get-Sha256 $Path) -cne $ExpectedSha256
|
||||
) {
|
||||
throw "$Label length or SHA-256 changed"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Container([string]$Name) {
|
||||
$rows = @((& docker inspect $Name) | ConvertFrom-Json)
|
||||
Assert-LastExitCode "Docker inspection for $Name"
|
||||
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
|
||||
return $rows[0]
|
||||
}
|
||||
|
||||
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
||||
throw "M48T risk quality is pinned to DESKTOP-OPJ8J04"
|
||||
}
|
||||
|
||||
$release = Resolve-DDirectory $ReleaseRoot "M48T release root" $false
|
||||
$dataset = Resolve-DDirectory $DatasetRoot "M48T dataset root" $true
|
||||
$output = Resolve-DDirectory $OutputRoot "M48T output root" $true
|
||||
$runOutput = Join-Path $output $RunId
|
||||
if (Test-Path -LiteralPath $runOutput) { throw "M48T run output already exists" }
|
||||
$null = New-Item -ItemType Directory -Path $runOutput
|
||||
$runOutput = Resolve-DDirectory $runOutput "M48T run output" $false
|
||||
|
||||
$wheel = Assert-RegularFile (
|
||||
(Join-Path $release "nodedc_mission_core-0.1.0-py3-none-any.whl")
|
||||
) "M48T wheel"
|
||||
if ((Get-Sha256 $wheel) -cne $ExpectedWheelSha256) { throw "M48T wheel SHA-256 changed" }
|
||||
$profile = Assert-RegularFile (
|
||||
(Join-Path $release "m48t-risk-quality-temporal-v1.json")
|
||||
) "M48T profile"
|
||||
$runner = Assert-RegularFile (
|
||||
(Join-Path $release "run_m48t_coco_risk_quality_worker.py")
|
||||
) "M48T runner"
|
||||
$runnerSha256 = Get-Sha256 $runner
|
||||
|
||||
$imagesArchive = Join-Path $dataset "val2017.zip"
|
||||
$annotations = Join-Path $dataset "instances_val2017.json"
|
||||
# These mirrors rehost the unmodified official COCO assets and publish pinned SHA-256 digests.
|
||||
Ensure-PinnedMirrorFile $imagesArchive `
|
||||
"https://huggingface.co/datasets/pcuenq/coco-2017-mirror/resolve/main/val2017.zip?download=true" `
|
||||
815585330 `
|
||||
"4f7e2ccb2866ec5041993c9cf2a952bbed69647b115d0f74da7ce8f4bef82f05" `
|
||||
"COCO val2017"
|
||||
Ensure-PinnedMirrorFile $annotations `
|
||||
"https://huggingface.co/datasets/LibreYOLO/coco2017/resolve/main/instances_val2017.json?download=true" `
|
||||
19987840 `
|
||||
"e8c7f7908f1d7278341fae127d0da654f102f11bd7b21d8aeefa635b8c810b6f" `
|
||||
"COCO instances_val2017"
|
||||
$imagesArchive = Assert-RegularFile $imagesArchive "COCO images archive"
|
||||
$annotations = Assert-RegularFile $annotations "COCO val2017 instances"
|
||||
$imagesArchiveSha256 = Get-Sha256 $imagesArchive
|
||||
$annotationsDocumentSha256 = Get-Sha256 $annotations
|
||||
|
||||
$imagesRoot = Join-Path $dataset "val2017"
|
||||
if (-not (Test-Path -LiteralPath $imagesRoot -PathType Container)) {
|
||||
& tar.exe -xf $imagesArchive -C $dataset
|
||||
Assert-LastExitCode "COCO val2017 extraction"
|
||||
}
|
||||
$imagesRoot = Resolve-DDirectory $imagesRoot "COCO val2017 images" $false
|
||||
if (@(Get-ChildItem -LiteralPath $imagesRoot -File -Filter "*.jpg").Count -ne 5000) {
|
||||
throw "COCO val2017 image count changed"
|
||||
}
|
||||
|
||||
$experimentRoot = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z"
|
||||
) "M48T RF-DETR experiment root" $false
|
||||
$modelRoot = Resolve-DDirectory (
|
||||
(Join-Path $experimentRoot "triton-models")
|
||||
) "M48T RF-DETR model root" $false
|
||||
if ((Get-Sha256 (Assert-RegularFile (
|
||||
(Join-Path $modelRoot "rf_detr_large\1\model.plan")
|
||||
) "RF-DETR TensorRT engine")) -cne "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8") {
|
||||
throw "RF-DETR TensorRT engine SHA-256 changed"
|
||||
}
|
||||
|
||||
$image = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
& docker image inspect $image *> $null
|
||||
Assert-LastExitCode "Pinned M48T image inspection"
|
||||
$historicalTriton = Get-Container "ndc-mission-core-triton"
|
||||
if (-not $historicalTriton.State.Running -or $historicalTriton.State.Health.Status -cne "healthy") {
|
||||
throw "Historical Triton must remain healthy during M48T quality evaluation"
|
||||
}
|
||||
$historicalTritonId = [string]$historicalTriton.Id
|
||||
$tritonName = "ndc-mission-core-m48t-risk-quality-triton"
|
||||
$qualityName = "ndc-mission-core-m48t-risk-quality"
|
||||
foreach ($name in @($tritonName, $qualityName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
throw "M48T candidate container $name already exists"
|
||||
}
|
||||
}
|
||||
|
||||
$opencv = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages"
|
||||
) "OpenCV dependency" $false
|
||||
$pillow = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1"
|
||||
) "Pillow dependency" $false
|
||||
|
||||
try {
|
||||
& docker create `
|
||||
--name $tritonName `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--pids-limit 512 `
|
||||
--shm-size 1g `
|
||||
--gpus all `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
||||
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
|
||||
--health-interval 5s `
|
||||
--health-timeout 3s `
|
||||
--health-start-period 20s `
|
||||
--health-retries 24 `
|
||||
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
|
||||
$image `
|
||||
tritonserver `
|
||||
--model-repository=/models `
|
||||
--model-control-mode=explicit `
|
||||
--load-model=rf_detr_large `
|
||||
--disable-auto-complete-config `
|
||||
--strict-readiness=true `
|
||||
--exit-on-error=true `
|
||||
--allow-http=true `
|
||||
--allow-grpc=false `
|
||||
--allow-metrics=false *> $null
|
||||
Assert-LastExitCode "M48T Triton creation"
|
||||
& docker start $tritonName *> $null
|
||||
Assert-LastExitCode "M48T Triton start"
|
||||
$ready = $false
|
||||
foreach ($attempt in 1..60) {
|
||||
Start-Sleep -Seconds 2
|
||||
$candidate = Get-Container $tritonName
|
||||
if (-not $candidate.State.Running) { throw "M48T Triton stopped during startup" }
|
||||
if ($candidate.State.Health.Status -ceq "healthy") { $ready = $true; break }
|
||||
}
|
||||
if (-not $ready) { throw "M48T Triton did not become healthy" }
|
||||
|
||||
$maximumArguments = @()
|
||||
if ($MaximumImages -gt 0) {
|
||||
$maximumArguments = @("--maximum-images", ([string]$MaximumImages))
|
||||
}
|
||||
$arguments = @(
|
||||
"run", "--name", $qualityName,
|
||||
"--network", ("container:{0}" -f $tritonName),
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "256",
|
||||
"--gpus", "all",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", "PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:/opt/opencv:/opt/pillow",
|
||||
"-v", ((Convert-ToDockerPath $release) + ":/release:ro"),
|
||||
"-v", ((Convert-ToDockerPath $imagesRoot) + ":/dataset/val2017:ro"),
|
||||
"-v", ((Convert-ToDockerPath $annotations) + ":/dataset/instances_val2017.json:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runOutput) + ":/output:rw"),
|
||||
"-v", ((Convert-ToDockerPath $opencv) + ":/opt/opencv:ro"),
|
||||
"-v", ((Convert-ToDockerPath $pillow) + ":/opt/pillow:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$image,
|
||||
"/release/run_m48t_coco_risk_quality_worker.py",
|
||||
"--profile", "/release/m48t-risk-quality-temporal-v1.json",
|
||||
"--annotations", "/dataset/instances_val2017.json",
|
||||
"--images-root", "/dataset/val2017",
|
||||
"--triton-origin", "http://127.0.0.1:8000",
|
||||
"--output", "/output/result.json",
|
||||
"--predictions", "/output/predictions.jsonl",
|
||||
"--failures", "/output/failures.jsonl",
|
||||
"--progress", "/output/progress.jsonl",
|
||||
"--review-root", "/output/review",
|
||||
"--runtime-artifact-sha256", $ExpectedWheelSha256,
|
||||
"--runner-sha256", $runnerSha256,
|
||||
"--images-archive-sha256", $imagesArchiveSha256,
|
||||
"--annotations-document-sha256", $annotationsDocumentSha256
|
||||
) + $maximumArguments
|
||||
& docker @arguments
|
||||
Assert-LastExitCode "M48T COCO risk quality evaluation"
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $runOutput "result.json") -PathType Leaf)) {
|
||||
throw "M48T result was not written"
|
||||
}
|
||||
} finally {
|
||||
foreach ($name in @($qualityName, $tritonName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
& docker rm -f $name *> $null
|
||||
}
|
||||
}
|
||||
$historicalAfter = Get-Container "ndc-mission-core-triton"
|
||||
if (
|
||||
$historicalAfter.Id -cne $historicalTritonId -or
|
||||
-not $historicalAfter.State.Running -or
|
||||
$historicalAfter.State.Health.Status -cne "healthy"
|
||||
) {
|
||||
throw "Historical Triton changed during M48T quality evaluation"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output ("M48T_RESULT={0}" -f (Join-Path $runOutput "result.json"))
|
||||
Write-Output ("COCO_IMAGES_SHA256={0}" -f $imagesArchiveSha256)
|
||||
Write-Output ("COCO_ANNOTATIONS_SHA256={0}" -f $annotationsDocumentSha256)
|
||||
Write-Output "HISTORICAL_TRITON_ACTION=none"
|
||||
Write-Output "PRODUCTION_ACCEPTED=false"
|
||||
Reference in New Issue
Block a user