Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
147a623df9 | ||
|
|
62c8ed421a | ||
|
|
be079bf18f | ||
|
|
73fbfddf6c | ||
|
|
3ff3b9b587 | ||
|
|
9a956c318a |
@@ -43,11 +43,13 @@ import {
|
||||
} from "./m48ObjectCentricQuality";
|
||||
import { fetchM48SmallStaticRegression } from "./m48SmallStaticRegression";
|
||||
import { fetchM48SFixedClassDetectorResult } from "./m48sFixedClassDetector";
|
||||
import { fetchM48TRiskQualityResult } from "./m48tRiskQuality";
|
||||
|
||||
export type AdvancedLaboratoryWorkId =
|
||||
| "m48-object-centric-quality"
|
||||
| "m48-small-static-passage-regression"
|
||||
| "m48s-fixed-class-detector"
|
||||
| "m48t-risk-quality-temporal"
|
||||
| "m47-reference-graph-shadow"
|
||||
| "m4-replay-threat"
|
||||
| "l3-pointpillars-visual-audit"
|
||||
@@ -93,6 +95,7 @@ const WORK_IDS: readonly AdvancedLaboratoryWorkId[] = [
|
||||
"m48-object-centric-quality",
|
||||
"m48-small-static-passage-regression",
|
||||
"m48s-fixed-class-detector",
|
||||
"m48t-risk-quality-temporal",
|
||||
"m47-reference-graph-shadow",
|
||||
"m4-replay-threat",
|
||||
"l3-pointpillars-visual-audit",
|
||||
@@ -133,6 +136,7 @@ const RESULT_PREFIX: Readonly<Record<AdvancedLaboratoryWorkId, string>> = {
|
||||
"m48-object-centric-quality": "m48-object-quality-(?:pack|result)",
|
||||
"m48-small-static-passage-regression": "m48-small-static-passage-regression",
|
||||
"m48s-fixed-class-detector": "m48s-fixed-class-detector-lab",
|
||||
"m48t-risk-quality-temporal": "m48t-risk-quality-temporal-lab",
|
||||
"m47-reference-graph-shadow": "m47-reference-graph-lab",
|
||||
"m4-replay-threat": "m4-threat-replay",
|
||||
"l3-pointpillars-visual-audit": "l3-pointpillars-visual-audit",
|
||||
@@ -181,6 +185,7 @@ export function emptyAdvancedLaboratoryResults(): AdvancedLaboratoryResults {
|
||||
m48: null,
|
||||
m48SmallStatic: null,
|
||||
m48s: null,
|
||||
m48t: null,
|
||||
m4Threat: null,
|
||||
l3: null,
|
||||
l31: null,
|
||||
@@ -308,6 +313,7 @@ export function advancedLaboratoryResultAvailable(
|
||||
return workId === "m48-object-centric-quality" ? results.m48 !== null
|
||||
: workId === "m48-small-static-passage-regression" ? results.m48SmallStatic !== null
|
||||
: workId === "m48s-fixed-class-detector" ? results.m48s !== null
|
||||
: workId === "m48t-risk-quality-temporal" ? results.m48t !== null
|
||||
: workId === "m47-reference-graph-shadow" ? results.m47Graph !== null
|
||||
: workId === "m4-replay-threat" ? results.m4Threat !== null
|
||||
: workId === "l3-pointpillars-visual-audit" ? results.l3 !== null
|
||||
@@ -366,6 +372,9 @@ export async function fetchAdvancedLaboratoryResult(
|
||||
} else if (workId === "m48s-fixed-class-detector") {
|
||||
if (!resultId) throw new AdvancedLaboratoryContractError("M4.8S LAB identity не выбрана.");
|
||||
results.m48s = await fetchM48SFixedClassDetectorResult(resultId, { fetcher, signal });
|
||||
} else if (workId === "m48t-risk-quality-temporal") {
|
||||
if (!resultId) throw new AdvancedLaboratoryContractError("M4.8T LAB identity не выбрана.");
|
||||
results.m48t = await fetchM48TRiskQualityResult(resultId, { fetcher, signal });
|
||||
} else if (workId === "m47-reference-graph-shadow") {
|
||||
if (!resultId) {
|
||||
throw new AdvancedLaboratoryContractError("M4.7 LAB identity не выбрана.");
|
||||
|
||||
@@ -37,12 +37,14 @@ import type { M47ReferenceGraphLabResult } from "./m47ReferenceGraph";
|
||||
import type { M48AdvancedResult } from "./m48ObjectCentricQuality";
|
||||
import type { M48SmallStaticRegressionResult } from "./m48SmallStaticRegression";
|
||||
import type { M48SFixedClassDetectorResult } from "./m48sFixedClassDetector";
|
||||
import type { M48TRiskQualityResult } from "./m48tRiskQuality";
|
||||
|
||||
export interface AdvancedLaboratoryResults {
|
||||
m47Graph: M47ReferenceGraphLabResult | null;
|
||||
m48: M48AdvancedResult | null;
|
||||
m48SmallStatic: M48SmallStaticRegressionResult | null;
|
||||
m48s: M48SFixedClassDetectorResult | null;
|
||||
m48t: M48TRiskQualityResult | null;
|
||||
m4Threat: M4ThreatReplayResult | null;
|
||||
l3: L3PointPillarsVisualAuditResult | null;
|
||||
l31: L31PointPillarsRavnovesResult | null;
|
||||
|
||||
@@ -967,7 +967,7 @@ export async function fetchAdvancedLaboratoryResults({
|
||||
const e39 = settledCatalogValue(settled[7]);
|
||||
const e40 = settledCatalogValue(settled[8]);
|
||||
return {
|
||||
m47Graph: null, m48: null, m48SmallStatic: null, m48s: null, m4Threat: null,
|
||||
m47Graph: null, m48: null, m48SmallStatic: null, m48s: null, m48t: null, m4Threat: null,
|
||||
l3: null, l31: null,
|
||||
l32: null,
|
||||
l33: null,
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
export interface M48TLaboratoryMethod {
|
||||
completeness: "complete";
|
||||
executionClass: "hybrid";
|
||||
pipelineId: string;
|
||||
components: readonly {
|
||||
kind: "source" | "model" | "algorithm" | "runtime" | "tool";
|
||||
name: string;
|
||||
version: string;
|
||||
role: string;
|
||||
identitySha256: string;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface M48TReviewCase {
|
||||
caseId: string;
|
||||
imageId: number;
|
||||
imageUrl: string;
|
||||
byteLength: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface M48TRiskQualityResult {
|
||||
resultId: string;
|
||||
createdAtUtc: string;
|
||||
source: {
|
||||
quality: {
|
||||
datasetId: "coco-2017-val";
|
||||
riskImages: number;
|
||||
truthInstances: number;
|
||||
independentHumanAnnotations: true;
|
||||
ravnovesGroundTruth: false;
|
||||
};
|
||||
temporal: {
|
||||
sourceId: "RAVNOVES00";
|
||||
frames: number;
|
||||
publications: number;
|
||||
independentSemanticTruthAvailable: false;
|
||||
};
|
||||
};
|
||||
candidate: {
|
||||
providerId: string;
|
||||
modelId: "rf_detr_large:1";
|
||||
minimumScore: 0.25;
|
||||
};
|
||||
execution: {
|
||||
durationSeconds: number;
|
||||
effectiveImagesPerSecond: number;
|
||||
imageP95Ms: number;
|
||||
inferenceP95Ms: number;
|
||||
timingIsAdmissionEvidence: false;
|
||||
};
|
||||
quality: {
|
||||
microPrecision: number;
|
||||
microRecall: number;
|
||||
mediumLargeRecall: number;
|
||||
emptyRiskImageFraction: number;
|
||||
families: Readonly<Record<"person" | "animal" | "light-road-user" | "vehicle", number>>;
|
||||
};
|
||||
counts: {
|
||||
truth: number;
|
||||
predictions: number;
|
||||
truePositive: number;
|
||||
falsePositive: number;
|
||||
falseNegative: number;
|
||||
};
|
||||
failures: Readonly<Record<string, number>>;
|
||||
temporal: {
|
||||
rawClassSwitches: number;
|
||||
stableClassSwitches: number;
|
||||
rawFamilySwitches: number;
|
||||
stableFamilySwitches: number;
|
||||
suppressedClassSwitches: number;
|
||||
suppressedFamilySwitches: number;
|
||||
selectedPublications: number;
|
||||
semanticObservations: number;
|
||||
peakActiveComponents: number;
|
||||
maximumActiveComponents: number;
|
||||
};
|
||||
acceptance: {
|
||||
qualityPassed: false;
|
||||
failedQualityGates: readonly [
|
||||
"minimum-micro-precision",
|
||||
"minimum-family-recall:vehicle",
|
||||
];
|
||||
temporalInvariantPassed: true;
|
||||
};
|
||||
review: {
|
||||
truthColor: "green";
|
||||
predictionColor: "yellow";
|
||||
cases: readonly M48TReviewCase[];
|
||||
};
|
||||
method: M48TLaboratoryMethod;
|
||||
limitations: readonly string[];
|
||||
}
|
||||
|
||||
type LaboratoryFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
|
||||
export class M48TContractError extends Error {}
|
||||
|
||||
function object(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new M48TContractError(`${label}: ожидался объект.`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function array(value: unknown, label: string): readonly unknown[] {
|
||||
if (!Array.isArray(value)) throw new M48TContractError(`${label}: ожидался массив.`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function text(value: unknown, label: string): string {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new M48TContractError(`${label}: ожидалась строка.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function number(value: unknown, label: string): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
||||
throw new M48TContractError(`${label}: ожидалось неотрицательное число.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integer(value: unknown, label: string): number {
|
||||
const parsed = number(value, label);
|
||||
if (!Number.isSafeInteger(parsed)) {
|
||||
throw new M48TContractError(`${label}: ожидалось целое число.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function exact<T extends string | number | boolean>(
|
||||
value: unknown,
|
||||
expected: T,
|
||||
label: string,
|
||||
): T {
|
||||
if (value !== expected) throw new M48TContractError(`${label}: нарушен контракт.`);
|
||||
return expected;
|
||||
}
|
||||
|
||||
function sha(value: unknown, label: string): string {
|
||||
const parsed = text(value, label);
|
||||
if (!/^[a-f0-9]{64}$/.test(parsed)) {
|
||||
throw new M48TContractError(`${label}: нарушена SHA-256 идентичность.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function method(value: unknown): M48TLaboratoryMethod {
|
||||
const raw = object(value, "M4.8T method");
|
||||
exact(raw.schema_version, "missioncore.laboratory-method/v1", "M4.8T method schema");
|
||||
exact(raw.completeness, "complete", "M4.8T method completeness");
|
||||
exact(raw.execution_class, "hybrid", "M4.8T method execution");
|
||||
const allowedKinds = new Set(["source", "model", "algorithm", "runtime", "tool"]);
|
||||
return {
|
||||
completeness: "complete",
|
||||
executionClass: "hybrid",
|
||||
pipelineId: text(raw.pipeline_id, "M4.8T pipeline"),
|
||||
components: array(raw.components, "M4.8T components").map((value) => {
|
||||
const component = object(value, "M4.8T component");
|
||||
const kind = text(component.kind, "M4.8T component kind");
|
||||
if (!allowedKinds.has(kind)) throw new M48TContractError("M4.8T component kind: неизвестное значение.");
|
||||
return {
|
||||
kind: kind as "source" | "model" | "algorithm" | "runtime" | "tool",
|
||||
name: text(component.name, "M4.8T component name"),
|
||||
version: text(component.version, "M4.8T component version"),
|
||||
role: text(component.role, "M4.8T component role"),
|
||||
identitySha256: sha(component.identity_sha256, "M4.8T component identity"),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function parseResult(value: unknown, expectedResultId: string): M48TRiskQualityResult {
|
||||
const raw = object(value, "M4.8T result");
|
||||
exact(raw.schema_version, "missioncore.m48t-risk-quality-temporal-view/v1", "M4.8T schema");
|
||||
exact(raw.result_id, expectedResultId, "M4.8T identity");
|
||||
exact(raw.status, "complete-quality-gate-failed-temporal-invariant-passed", "M4.8T status");
|
||||
exact(raw.access, "read-only", "M4.8T access");
|
||||
exact(raw.ground_truth, false, "M4.8T ground truth");
|
||||
|
||||
const source = object(raw.source, "M4.8T source");
|
||||
const qualitySource = object(source.quality, "M4.8T quality source");
|
||||
const temporalSource = object(source.temporal, "M4.8T temporal source");
|
||||
const configuration = object(raw.configuration, "M4.8T configuration");
|
||||
const candidate = object(configuration.candidate, "M4.8T candidate");
|
||||
const execution = object(raw.execution, "M4.8T execution");
|
||||
const imageTiming = object(execution.image_timing_ms, "M4.8T image timing");
|
||||
const inferenceTiming = object(execution.triton_inference_ms, "M4.8T inference timing");
|
||||
const metrics = object(raw.metrics, "M4.8T metrics");
|
||||
const quality = object(metrics.quality, "M4.8T quality metrics");
|
||||
const families = object(quality.families, "M4.8T family metrics");
|
||||
const familyRecall = (name: "person" | "animal" | "light-road-user" | "vehicle") =>
|
||||
number(object(families[name], `M4.8T ${name}`).family_recall, `M4.8T ${name} recall`);
|
||||
const counts = object(metrics.counts, "M4.8T counts");
|
||||
const failures = object(metrics.failure_buckets, "M4.8T failures");
|
||||
const temporal = object(metrics.temporal, "M4.8T temporal metrics");
|
||||
const snapshot = object(temporal.snapshot, "M4.8T temporal snapshot");
|
||||
const acceptance = object(raw.acceptance, "M4.8T acceptance");
|
||||
const qualityGate = object(acceptance.quality, "M4.8T quality gate");
|
||||
const temporalGate = object(acceptance.temporal_invariant, "M4.8T temporal gate");
|
||||
const failed = array(qualityGate.failed, "M4.8T failed gates").map((item) => text(item, "M4.8T failed gate"));
|
||||
if (failed.length !== 2 || failed[0] !== "minimum-micro-precision" || failed[1] !== "minimum-family-recall:vehicle") {
|
||||
throw new M48TContractError("M4.8T failed gates: изменён зафиксированный результат.");
|
||||
}
|
||||
exact(qualityGate.passed, false, "M4.8T quality gate");
|
||||
exact(temporalGate.passed, true, "M4.8T temporal invariant");
|
||||
exact(temporalGate.semantic_quality_accepted, false, "M4.8T temporal semantic acceptance");
|
||||
|
||||
const review = object(raw.review, "M4.8T review");
|
||||
const legend = object(review.legend, "M4.8T review legend");
|
||||
exact(legend.ground_truth, "green", "M4.8T truth legend");
|
||||
exact(legend.rf_detr_prediction, "yellow", "M4.8T prediction legend");
|
||||
const cases = array(review.cases, "M4.8T review cases").map((value) => {
|
||||
const item = object(value, "M4.8T review case");
|
||||
const caseId = text(item.case_id, "M4.8T case id");
|
||||
if (!/^[0-9]{12}$/.test(caseId)) throw new M48TContractError("M4.8T case identity нарушена.");
|
||||
exact(item.media_type, "image/jpeg", "M4.8T case media");
|
||||
return {
|
||||
caseId,
|
||||
imageId: integer(item.image_id, "M4.8T image id"),
|
||||
imageUrl: text(item.image_url, "M4.8T image URL"),
|
||||
byteLength: integer(item.byte_length, "M4.8T image bytes"),
|
||||
sha256: sha(item.sha256, "M4.8T image SHA"),
|
||||
};
|
||||
});
|
||||
if (cases.length !== 16 || new Set(cases.map(({ caseId }) => caseId)).size !== 16) {
|
||||
throw new M48TContractError("M4.8T review catalog: нарушен размер.");
|
||||
}
|
||||
|
||||
const parsedFailures: Record<string, number> = {};
|
||||
for (const [key, count] of Object.entries(failures)) parsedFailures[key] = integer(count, `M4.8T failure ${key}`);
|
||||
const temporalConfiguration = object(configuration.temporal, "M4.8T temporal configuration");
|
||||
return {
|
||||
resultId: expectedResultId,
|
||||
createdAtUtc: text(raw.created_at_utc, "M4.8T created at"),
|
||||
source: {
|
||||
quality: {
|
||||
datasetId: exact(qualitySource.dataset_id, "coco-2017-val", "M4.8T dataset"),
|
||||
riskImages: integer(qualitySource.risk_images, "M4.8T risk images"),
|
||||
truthInstances: integer(qualitySource.truth_instances, "M4.8T truth instances"),
|
||||
independentHumanAnnotations: exact(qualitySource.independent_human_annotations, true, "M4.8T independent truth"),
|
||||
ravnovesGroundTruth: exact(qualitySource.ravnoves_ground_truth, false, "M4.8T RAVNOVES truth"),
|
||||
},
|
||||
temporal: {
|
||||
sourceId: exact(temporalSource.source_id, "RAVNOVES00", "M4.8T temporal source"),
|
||||
frames: integer(temporalSource.frames, "M4.8T temporal frames"),
|
||||
publications: integer(temporalSource.publications, "M4.8T temporal publications"),
|
||||
independentSemanticTruthAvailable: exact(temporalSource.independent_semantic_truth_available, false, "M4.8T temporal truth"),
|
||||
},
|
||||
},
|
||||
candidate: {
|
||||
providerId: text(candidate.provider_id, "M4.8T provider"),
|
||||
modelId: exact(candidate.model_id, "rf_detr_large:1", "M4.8T model"),
|
||||
minimumScore: exact(candidate.minimum_score, 0.25, "M4.8T threshold"),
|
||||
},
|
||||
execution: {
|
||||
durationSeconds: number(execution.duration_seconds, "M4.8T duration"),
|
||||
effectiveImagesPerSecond: number(execution.effective_images_per_second, "M4.8T throughput"),
|
||||
imageP95Ms: number(imageTiming.p95, "M4.8T image p95"),
|
||||
inferenceP95Ms: number(inferenceTiming.p95, "M4.8T inference p95"),
|
||||
timingIsAdmissionEvidence: exact(execution.timing_is_admission_evidence, false, "M4.8T timing authority"),
|
||||
},
|
||||
quality: {
|
||||
microPrecision: number(quality.micro_precision, "M4.8T precision"),
|
||||
microRecall: number(quality.micro_recall, "M4.8T recall"),
|
||||
mediumLargeRecall: number(quality.medium_large_recall, "M4.8T medium-large recall"),
|
||||
emptyRiskImageFraction: number(quality.empty_prediction_risk_image_fraction, "M4.8T empty fraction"),
|
||||
families: {
|
||||
person: familyRecall("person"),
|
||||
animal: familyRecall("animal"),
|
||||
"light-road-user": familyRecall("light-road-user"),
|
||||
vehicle: familyRecall("vehicle"),
|
||||
},
|
||||
},
|
||||
counts: {
|
||||
truth: integer(qualitySource.truth_instances, "M4.8T truth count"),
|
||||
predictions: integer(counts.predictions, "M4.8T predictions"),
|
||||
truePositive: integer(counts.true_positive, "M4.8T TP"),
|
||||
falsePositive: integer(counts.false_positive, "M4.8T FP"),
|
||||
falseNegative: integer(counts.false_negative, "M4.8T FN"),
|
||||
},
|
||||
failures: parsedFailures,
|
||||
temporal: {
|
||||
rawClassSwitches: integer(temporal.raw_class_switches, "M4.8T raw class switches"),
|
||||
stableClassSwitches: integer(temporal.stable_class_switches, "M4.8T stable class switches"),
|
||||
rawFamilySwitches: integer(temporal.raw_family_switches, "M4.8T raw family switches"),
|
||||
stableFamilySwitches: integer(temporal.stable_family_switches, "M4.8T stable family switches"),
|
||||
suppressedClassSwitches: integer(temporal.suppressed_or_deferred_class_switches, "M4.8T suppressed class switches"),
|
||||
suppressedFamilySwitches: integer(temporal.suppressed_or_deferred_family_switches, "M4.8T suppressed family switches"),
|
||||
selectedPublications: integer(temporal.selected_publications, "M4.8T selected publications"),
|
||||
semanticObservations: integer(temporal.semantic_current_observations, "M4.8T semantic observations"),
|
||||
peakActiveComponents: integer(snapshot.peak_active_components, "M4.8T active peak"),
|
||||
maximumActiveComponents: integer(temporalConfiguration.maximum_active_components, "M4.8T active bound"),
|
||||
},
|
||||
acceptance: {
|
||||
qualityPassed: false,
|
||||
failedQualityGates: ["minimum-micro-precision", "minimum-family-recall:vehicle"],
|
||||
temporalInvariantPassed: true,
|
||||
},
|
||||
review: { truthColor: "green", predictionColor: "yellow", cases },
|
||||
method: method(raw.method),
|
||||
limitations: array(raw.limitations, "M4.8T limitations").map((item) => text(item, "M4.8T limitation")),
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchM48TRiskQualityResult(
|
||||
resultId: string,
|
||||
{
|
||||
fetcher = fetch,
|
||||
signal,
|
||||
}: { fetcher?: LaboratoryFetch; signal?: AbortSignal } = {},
|
||||
): Promise<M48TRiskQualityResult> {
|
||||
if (!/^m48t-risk-quality-temporal-lab-[a-f0-9]{64}$/.test(resultId)) {
|
||||
throw new M48TContractError("M4.8T result identity недопустима.");
|
||||
}
|
||||
const response = await fetcher(`/api/v1/laboratory/m48t/risk-quality/results/${resultId}`, {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) throw new M48TContractError(`M4.8T недоступен: HTTP ${response.status}.`);
|
||||
return parseResult(await response.json(), resultId);
|
||||
}
|
||||
@@ -45,6 +45,7 @@ import { M47ReferenceGraphResultView } from "./M47ReferenceGraphResult";
|
||||
import { M48ObjectCentricQualityResultView } from "./M48ObjectCentricQualityResult";
|
||||
import { M48SmallStaticPassageRegressionResultView } from "./M48SmallStaticPassageRegressionResult";
|
||||
import { M48SFixedClassDetectorResultView } from "./M48SFixedClassDetectorResult";
|
||||
import { M48TRiskQualityResultView } from "./M48TRiskQualityResult";
|
||||
|
||||
export { isAdvancedLaboratoryWorkId };
|
||||
export type { AdvancedLaboratoryWorkId };
|
||||
@@ -96,6 +97,9 @@ export function AdvancedLaboratoryResult({
|
||||
if (workId === "m48s-fixed-class-detector" && results.m48s) {
|
||||
return <M48SFixedClassDetectorResultView rigLabel={rigLabel} result={results.m48s} />;
|
||||
}
|
||||
if (workId === "m48t-risk-quality-temporal" && results.m48t) {
|
||||
return <M48TRiskQualityResultView rigLabel={rigLabel} result={results.m48t} />;
|
||||
}
|
||||
if (workId === "m47-reference-graph-shadow" && results.m47Graph) {
|
||||
return <M47ReferenceGraphResultView rigLabel={rigLabel} result={results.m47Graph} />;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import {
|
||||
LaboratoryEvidence,
|
||||
LaboratoryResultSummary,
|
||||
LaboratorySummary,
|
||||
LaboratoryWorkTemplate,
|
||||
} from "../../components/laboratory/LaboratoryPresentation";
|
||||
import type { M48TRiskQualityResult } from "../../core/laboratory/m48tRiskQuality";
|
||||
import { M48TRiskQualityVisual } from "./M48TRiskQualityVisual";
|
||||
|
||||
function percent(value: number, digits = 1): string {
|
||||
return `${(value * 100).toLocaleString("ru-RU", { maximumFractionDigits: digits })}%`;
|
||||
}
|
||||
|
||||
function decimal(value: number, digits = 1): string {
|
||||
return value.toLocaleString("ru-RU", { maximumFractionDigits: digits });
|
||||
}
|
||||
|
||||
export function M48TRiskQualityResultView({
|
||||
rigLabel,
|
||||
result,
|
||||
}: {
|
||||
rigLabel: string;
|
||||
result: M48TRiskQualityResult;
|
||||
}) {
|
||||
return (
|
||||
<LaboratoryWorkTemplate
|
||||
summary={(
|
||||
<LaboratorySummary
|
||||
title="M4.8T · semantic quality + temporal identity"
|
||||
description="RF-DETR-L проверен на полном независимом COCO val2017 по заранее записанным risk-family гейтам. Отдельно тот же advisory-класс стабилизирован на geometry-owned ID полного RAVNOVES00 replay; семантика не участвует в association или occupancy."
|
||||
status="Quality gate failed · temporal invariant passed"
|
||||
statusTone="warning"
|
||||
facts={[
|
||||
{ label: "Quality truth", value: `COCO val2017 · ${result.source.quality.truthInstances.toLocaleString("ru-RU")} risk instances` },
|
||||
{ label: "Candidate", value: `${result.candidate.modelId} · threshold ${decimal(result.candidate.minimumScore, 2)}` },
|
||||
{ label: "Temporal source", value: `${rigLabel} · ${result.source.temporal.frames.toLocaleString("ru-RU")} world states` },
|
||||
{ label: "Authority", value: "SHADOW ONLY · geometry-owned occupancy · production NO" },
|
||||
]}
|
||||
brief={{
|
||||
question: "Достаточно ли качественна текущая risk-семантика RF-DETR и можно ли убрать кадровое мерцание класса без влияния на геометрию?",
|
||||
approach: `Все ${result.source.quality.riskImages.toLocaleString("ru-RU")} COCO-изображений с risk-классами оценены при неизменном score ${decimal(result.candidate.minimumScore, 2)}. Затем bounded history 5 / confirm 2 / switch 3 применена только к advisory-классу уже существующих component ID.`,
|
||||
principalResult: `Recall прошёл: ${percent(result.quality.microRecall)} overall и ${percent(result.quality.mediumLargeRecall)} medium+large. Precision ${percent(result.quality.microPrecision)} и vehicle recall ${percent(result.quality.families.vehicle)} не прошли гейты. Temporal shadow сократил class switches ${result.temporal.rawClassSwitches} → ${result.temporal.stableClassSwitches} и family switches ${result.temporal.rawFamilySwitches} → ${result.temporal.stableFamilySwitches}.`,
|
||||
limitation: "COCO не является truth городского маршрута RAVNOVES00, а temporal replay не имеет независимой track/class truth. Batch timing не принимается как realtime-гейт.",
|
||||
}}
|
||||
method={result.method}
|
||||
/>
|
||||
)}
|
||||
evidence={(
|
||||
<LaboratoryEvidence
|
||||
eyebrow="M4.8T VISUAL EVIDENCE · INDEPENDENT COCO TRUTH"
|
||||
title="16 hash-bound review cases: human truth против RF-DETR"
|
||||
kind="diagnostic-model"
|
||||
resizable
|
||||
>
|
||||
<M48TRiskQualityVisual result={result} />
|
||||
</LaboratoryEvidence>
|
||||
)}
|
||||
result={(
|
||||
<LaboratoryResultSummary
|
||||
title="Temporal anti-flicker принят как инвариант; detector quality не принят"
|
||||
status="2/10 predeclared quality checks failed"
|
||||
statusTone="warning"
|
||||
metrics={[
|
||||
{ label: "Precision / recall", value: `${percent(result.quality.microPrecision)} / ${percent(result.quality.microRecall)}`, hint: "gates ≥ 80% / ≥ 75%" },
|
||||
{ label: "Vehicle family", value: percent(result.quality.families.vehicle), hint: "gate ≥ 85% · failed" },
|
||||
{ label: "TP / FP / FN", value: `${result.counts.truePositive.toLocaleString("ru-RU")} / ${result.counts.falsePositive.toLocaleString("ru-RU")} / ${result.counts.falseNegative.toLocaleString("ru-RU")}`, hint: `${result.counts.predictions.toLocaleString("ru-RU")} predictions` },
|
||||
{ label: "Class / family switches", value: `${result.temporal.rawClassSwitches}→${result.temporal.stableClassSwitches} / ${result.temporal.rawFamilySwitches}→${result.temporal.stableFamilySwitches}`, hint: `peak active ${result.temporal.peakActiveComponents}/${result.temporal.maximumActiveComponents}` },
|
||||
{ label: "Batch throughput", value: `${decimal(result.execution.effectiveImagesPerSecond, 2)} image/s`, hint: `image p95 ${decimal(result.execution.imageP95Ms, 2)} ms · non-admission` },
|
||||
]}
|
||||
conclusion={{
|
||||
proved: `На независимой COCO truth текущий RF-DETR сохраняет высокий recall: ${percent(result.quality.microRecall)} overall и ${percent(result.quality.mediumLargeRecall)} medium+large. Bounded temporal state подавил или отложил ${result.temporal.suppressedClassSwitches} class-switch и ${result.temporal.suppressedFamilySwitches} family-switch, сохранив association и occupancy class-independent; active state ${result.temporal.peakActiveComponents}/${result.temporal.maximumActiveComponents}, eviction 0.`,
|
||||
notProved: `Semantic candidate не принят: precision ${percent(result.quality.microPrecision)} при gate 80%, vehicle recall ${percent(result.quality.families.vehicle)} при gate 85%. Не доказаны RAVNOVES class truth, physical track identity, child/adult, unknown moving hazards, risk policy и realtime admission этой batch-командой.`,
|
||||
decision: "Не менять зафиксированные гейты и не добавлять второй detector. RF-DETR остаётся shadow-кандидатом; temporal stabilizer допустим только как advisory-слой поверх geometry-owned ID.",
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { useState } from "react";
|
||||
import { Icon, IconButton } from "@nodedc/ui-react";
|
||||
|
||||
import { LaboratoryEvidenceViewer } from "../../components/laboratory/LaboratoryEvidenceViewer";
|
||||
import type { M48TRiskQualityResult } from "../../core/laboratory/m48tRiskQuality";
|
||||
|
||||
export function M48TRiskQualityVisual({ result }: { result: M48TRiskQualityResult }) {
|
||||
const [index, setIndex] = useState(0);
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const item = result.review.cases[index] ?? null;
|
||||
const navigate = (offset: -1 | 1) => {
|
||||
setIndex((current) => (
|
||||
current + offset + result.review.cases.length
|
||||
) % result.review.cases.length);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="l3-visual-audit">
|
||||
<LaboratoryEvidenceViewer
|
||||
label="M4.8T independent COCO quality review"
|
||||
mode="quality"
|
||||
modes={[{ value: "quality", label: "COCO TRUTH" }]}
|
||||
expanded={expanded}
|
||||
onModeChange={() => undefined}
|
||||
onExpandedChange={setExpanded}
|
||||
actions={(
|
||||
<div className="l3-visual-audit__pagination">
|
||||
<IconButton label="Предыдущий M4.8T review case" onClick={() => navigate(-1)}>
|
||||
<Icon name="chevron-left" size={16} />
|
||||
</IconButton>
|
||||
<IconButton label="Следующий M4.8T review case" onClick={() => navigate(1)}>
|
||||
<Icon name="chevron-right" size={16} />
|
||||
</IconButton>
|
||||
</div>
|
||||
)}
|
||||
overlay={item ? (
|
||||
<div className="l3-visual-audit__overlay">
|
||||
<div>
|
||||
<span>COCO val2017 · independent human truth</span>
|
||||
<strong>case {index + 1}/{result.review.cases.length} · image {item.imageId}</strong>
|
||||
<small>Зелёный — truth · жёлтый — RF-DETR prediction · score ≥ 0,25</small>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
>
|
||||
{item ? (
|
||||
<div className="l32-camera-scene">
|
||||
<img
|
||||
src={item.imageUrl}
|
||||
alt={`M4.8T COCO review image ${item.imageId}: truth and RF-DETR boxes`}
|
||||
draggable={false}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<div className="l3-visual-audit__state" role="alert">
|
||||
<Icon name="alert" size={18} />
|
||||
M4.8T visual evidence недоступно.
|
||||
</div>
|
||||
)}
|
||||
</LaboratoryEvidenceViewer>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -84,6 +84,13 @@ const KNOWN_WORKS: Readonly<Record<Exclude<LaboratoryWorkId, `session:${string}`
|
||||
experimentName: "RAVNOVES00 fixed-class risk detector",
|
||||
variantName: "M4.8S · RF-DETR-L TensorRT/Triton shadow",
|
||||
},
|
||||
"m48t-risk-quality-temporal": {
|
||||
profileId: "rig-ravnoves-perception-gate-v1",
|
||||
profileName: (rigLabel) => `${rig(rigLabel)} RIGHT · COCO quality + bounded temporal identity`,
|
||||
experimentId: "m48t-risk-quality-temporal",
|
||||
experimentName: "RF-DETR independent semantic quality and temporal identity",
|
||||
variantName: "M4.8T · COCO val2017 truth + RAVNOVES00 temporal shadow",
|
||||
},
|
||||
"m47-reference-graph-shadow": {
|
||||
profileId: "rig-dual-evidence-virtual-corridor-v1",
|
||||
profileName: (rigLabel) => `${rig(rigLabel)} RIGHT · Camera + LiDAR dual evidence`,
|
||||
|
||||
@@ -22,6 +22,7 @@ function mergeResults(
|
||||
m48: next.m48 ?? current.m48,
|
||||
m48SmallStatic: next.m48SmallStatic ?? current.m48SmallStatic,
|
||||
m48s: next.m48s ?? current.m48s,
|
||||
m48t: next.m48t ?? current.m48t,
|
||||
m4Threat: next.m4Threat ?? current.m4Threat,
|
||||
l3: next.l3 ?? current.l3,
|
||||
l31: next.l31 ?? current.l31,
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"schema_version": "missioncore.laboratory-evidence-definition/v1",
|
||||
"work_id": "m48t-risk-quality-temporal",
|
||||
"evidence": {
|
||||
"runtime_relative_root": "m48t-risk-quality/lab-results",
|
||||
"result_id_prefix": "m48t-risk-quality-temporal-lab",
|
||||
"document_name": "manifest.json",
|
||||
"schema_version": "missioncore.m48t-risk-quality-temporal-lab/v1"
|
||||
}
|
||||
}
|
||||
@@ -128,6 +128,20 @@
|
||||
"run": "missioncore.laboratory-run/v1",
|
||||
"evidence": "missioncore.m48s-fixed-class-detector-lab/v1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"work_id": "m48t-risk-quality-temporal",
|
||||
"lifecycle": "experimental",
|
||||
"isolation": "bounded-adapter",
|
||||
"adapter_id": "experimental.m48t-risk-quality-temporal/v1",
|
||||
"input_roles": ["repository_root"],
|
||||
"contracts": {
|
||||
"source": "missioncore.m48t-sealed-quality-temporal-source-set/v1",
|
||||
"provider": "missioncore.rf-detr-risk-quality-provider/v1",
|
||||
"graph": "missioncore.m48t-risk-quality-temporal-lab-graph/v1",
|
||||
"run": "missioncore.laboratory-run/v1",
|
||||
"evidence": "missioncore.m48t-risk-quality-temporal-lab/v1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"legacy_work_ids": [
|
||||
|
||||
@@ -253,6 +253,13 @@
|
||||
"signal": "progress",
|
||||
"lifecycle": "current",
|
||||
"visual_evidence": "available"
|
||||
},
|
||||
{
|
||||
"catalog_id": "m48t-risk-quality-temporal",
|
||||
"evidence_id": "m48t-risk-quality-temporal-lab-ed5355fe0adb9b18942d75aff2362d79190b9e864f070ebe7a1c15fccbc0fbfb",
|
||||
"signal": "progress",
|
||||
"lifecycle": "current",
|
||||
"visual_evidence": "available"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"schema_version": "missioncore.m48t-risk-quality-temporal-profile/v1",
|
||||
"profile_id": "m48t-coco2017-risk-quality-temporal/v1",
|
||||
"candidate": {
|
||||
"provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
|
||||
"model_id": "rf_detr_large:1",
|
||||
"minimum_score": 0.25,
|
||||
"source_projection": "stretch-to-800x600-then-rf-detr-704x704"
|
||||
},
|
||||
"dataset": {
|
||||
"dataset_id": "coco-2017-val",
|
||||
"images_url": "http://images.cocodataset.org/zips/val2017.zip",
|
||||
"annotations_url": "http://images.cocodataset.org/annotations/annotations_trainval2017.zip",
|
||||
"annotation_document": "annotations/instances_val2017.json",
|
||||
"split": "val2017",
|
||||
"independent_human_annotations": true,
|
||||
"include_iscrowd": false,
|
||||
"minimum_projected_box_area_pixels": 64.0,
|
||||
"maximum_projected_box_area_fraction": 0.5
|
||||
},
|
||||
"risk_classes": {
|
||||
"person": ["person"],
|
||||
"animal": ["bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe"],
|
||||
"light-road-user": ["bicycle", "motorcycle", "skateboard"],
|
||||
"vehicle": ["car", "bus", "truck"]
|
||||
},
|
||||
"matching": {
|
||||
"iou_threshold": 0.5,
|
||||
"method": "score-ordered-greedy-exact-class",
|
||||
"family_confusion_matching": "score-ordered-greedy-same-family",
|
||||
"small_area_upper_pixels": 1024.0,
|
||||
"medium_area_upper_pixels": 9216.0
|
||||
},
|
||||
"quality_gates": {
|
||||
"minimum_truth_instances": 1000,
|
||||
"minimum_micro_precision": 0.8,
|
||||
"minimum_micro_recall": 0.75,
|
||||
"minimum_medium_large_recall": 0.85,
|
||||
"minimum_family_recall": {
|
||||
"person": 0.8,
|
||||
"animal": 0.75,
|
||||
"light-road-user": 0.65,
|
||||
"vehicle": 0.85
|
||||
},
|
||||
"minimum_class_truth_instances": 25,
|
||||
"minimum_qualified_class_recall": 0.55,
|
||||
"maximum_empty_prediction_risk_image_fraction": 0.1
|
||||
},
|
||||
"temporal": {
|
||||
"history_size": 5,
|
||||
"initial_confirmation_observations": 2,
|
||||
"switch_confirmation_observations": 3,
|
||||
"semantic_hold_seconds": 0.3,
|
||||
"state_expiry_seconds": 1.0,
|
||||
"maximum_active_components": 512,
|
||||
"cross_family_conflict_fallback": "unknown",
|
||||
"association_uses_semantic_class": false,
|
||||
"occupancy_uses_semantic_class": false
|
||||
},
|
||||
"scope": {
|
||||
"child_adult_distinction_evaluated": false,
|
||||
"unknown_moving_detection_evaluated": false,
|
||||
"object_presence_evaluated": true,
|
||||
"semantic_family_evaluated": true,
|
||||
"temporal_stability_evaluated": true,
|
||||
"risk_policy_evaluated": false
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth_for_ravnoves00": false,
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"actuation_allowed": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"schema_version": "missioncore.m48t-upstream-parity-profile/v1",
|
||||
"profile_id": "m48t-rf-detr-large-upstream-coco-parity/v1",
|
||||
"model": {
|
||||
"package": "rfdetr",
|
||||
"package_version": "1.9.4",
|
||||
"upstream_revision": "9b009fa928d6218320439803d1da01869a85c072",
|
||||
"checkpoint": "rf-detr-large-2026.pth",
|
||||
"checkpoint_sha256": "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38",
|
||||
"resolution": [704, 704],
|
||||
"maximum_query_class_pairs": 300
|
||||
},
|
||||
"dataset": {
|
||||
"dataset_id": "coco-2017-val",
|
||||
"image_count": 5000,
|
||||
"annotations_sha256": "e8c7f7908f1d7278341fae127d0da654f102f11bd7b21d8aeefa635b8c810b6f",
|
||||
"evaluation": "pycocotools.COCOeval/bbox",
|
||||
"all_categories": true,
|
||||
"official_crowd_ignore": true,
|
||||
"custom_area_filtering": false
|
||||
},
|
||||
"providers": {
|
||||
"pytorch": {
|
||||
"preprocessing": "official-rfdetr-predict-direct-original-to-704",
|
||||
"dtype": "float16",
|
||||
"confidence_prefilter": 0.0
|
||||
},
|
||||
"tensorrt": {
|
||||
"provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
|
||||
"engine_sha256": "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8",
|
||||
"preprocessing": "official-rfdetr-direct-original-to-704",
|
||||
"confidence_prefilter": 0.0
|
||||
}
|
||||
},
|
||||
"published_reference": {
|
||||
"coco_ap_50_95": 0.565,
|
||||
"coco_ap_50": 0.751,
|
||||
"maximum_absolute_reproduction_delta": 0.01
|
||||
},
|
||||
"parity_gates": {
|
||||
"maximum_absolute_ap_50_95_delta": 0.005,
|
||||
"maximum_absolute_ap_50_delta": 0.005
|
||||
},
|
||||
"authority": {
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"actuation_allowed": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
# M48T RF-DETR upstream и TensorRT parity gate
|
||||
|
||||
Дата: 2026-08-26
|
||||
|
||||
Режим: diagnostic shadow
|
||||
|
||||
Worker: `worker-006`, NVIDIA GeForce RTX 4090
|
||||
|
||||
Production acceptance: **нет**
|
||||
|
||||
## Решение
|
||||
|
||||
Полный официальный COCO val2017 gate принят. Закреплённый RF-DETR Large
|
||||
checkpoint воспроизвёл опубликованное upstream-качество в официальном PyTorch
|
||||
пути, а развёрнутый TensorRT engine сохранил это качество практически точно.
|
||||
|
||||
Диагноз: `upstream-and-tensorrt-parity-passed`.
|
||||
|
||||
Следовательно, наблюдавшиеся в Mission Core пропуски поведенчески значимых
|
||||
объектов нельзя объяснять деградацией RF-DETR checkpoint, ONNX/TensorRT
|
||||
конвертацией или декодированием TensorRT outputs. Следующий поиск должен быть в
|
||||
downstream admission: class mapping, valid-FOV, confidence/area gates,
|
||||
risk-class policy, temporal delivery и доменном покрытии городского источника.
|
||||
|
||||
## Строгий контракт
|
||||
|
||||
- dataset: все 5 000 изображений COCO val2017;
|
||||
- categories: все официальные COCO categories;
|
||||
- preprocessing: официальный RF-DETR Large, `704x704`;
|
||||
- predictions: top 300, confidence prefilter `0.0`;
|
||||
- custom area filtering: выключен;
|
||||
- crowd handling: официальный `pycocotools.COCOeval`;
|
||||
- published reference: AP@[.50:.95] `0.565`, AP50 `0.751`;
|
||||
- upstream tolerance: `0.01`;
|
||||
- PyTorch/TensorRT parity tolerance: `0.005`;
|
||||
- navigation, safety, command и actuation authority: `false`.
|
||||
|
||||
## Результат
|
||||
|
||||
| Метрика | Официальный PyTorch | TensorRT/Triton | Абсолютная дельта |
|
||||
|---|---:|---:|---:|
|
||||
| AP@[.50:.95] | 0.565100 | 0.565133 | 0.000033 |
|
||||
| AP50 | 0.751158 | 0.751285 | 0.000127 |
|
||||
| AP75 | 0.613005 | 0.613402 | 0.000397 |
|
||||
| AP small | 0.388411 | 0.389311 | 0.000900 |
|
||||
| AP medium | 0.610420 | 0.610540 | 0.000120 |
|
||||
| AP large | 0.740038 | 0.739485 | 0.000553 |
|
||||
| Prediction count | 1 499 907 | 1 499 909 | 2 |
|
||||
| Effective throughput | 35.094 FPS | 42.807 FPS | — |
|
||||
| Image p95 | 36.858 ms | 30.792 ms | — |
|
||||
|
||||
Полный прогон занял `375.912 s`. Оба admission checks приняты:
|
||||
|
||||
- upstream reproduction: `passed`;
|
||||
- TensorRT provider parity: `passed`.
|
||||
|
||||
## Immutable provenance
|
||||
|
||||
- profile: `m48t-rf-detr-large-upstream-coco-parity/v1`;
|
||||
- report identity SHA-256:
|
||||
`20a286d0190d1873d924260fc9b7c61288642e53b7206b2da81610da2989ef88`;
|
||||
- result file SHA-256:
|
||||
`48bb85081d733a0a839c7d1110fbf7f0da69234b45d38e5c4e85f28079883cbc`;
|
||||
- checkpoint SHA-256:
|
||||
`0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38`;
|
||||
- annotations SHA-256:
|
||||
`e8c7f7908f1d7278341fae127d0da654f102f11bd7b21d8aeefa635b8c810b6f`;
|
||||
- TensorRT engine SHA-256:
|
||||
`986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8`;
|
||||
- PyTorch predictions SHA-256:
|
||||
`da5b895bf906ee8480405149d63a3d19f1d8af0179d9a457d5202d844dc07b9d`;
|
||||
- TensorRT predictions SHA-256:
|
||||
`bb3baa3997e303a467799eb0b8ed49005182d06495106d57575906087402709b`;
|
||||
- runtime:
|
||||
`nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794`;
|
||||
- packages: RF-DETR `1.9.4`, PyTorch `2.9.1+cu130`, torchvision
|
||||
`0.24.1+cu130`, pycocotools `2.0.10`, Triton client `2.71.0`.
|
||||
|
||||
Worker evidence остаётся в
|
||||
`D:\NDC_MISSIONCORE\runtime\results\m48t-upstream-parity\full-coco2017-val-20260825T213600Z`.
|
||||
Компактная копия `result.json` и `progress.jsonl` сохранена локально в
|
||||
`.runtime/worker-results/m48t-upstream-parity-coco2017-val-full-v1/`.
|
||||
|
||||
Канонический `ndc-mission-core-triton` до и после прогона сохранил тот же
|
||||
container id и состояние `healthy`; его конфигурация не изменялась.
|
||||
|
||||
## Что этот gate не доказывает
|
||||
|
||||
- качество на RAVNOVES00 и будущих городских/полевых доменах;
|
||||
- полноту классов вне COCO, включая отдельный класс самоката;
|
||||
- корректность наших confidence, area и valid-FOV порогов;
|
||||
- отсутствие потерь между detector output, tracker и world-state graph;
|
||||
- безопасность navigation/actuation policy.
|
||||
|
||||
## Следующий gate
|
||||
|
||||
Провести downstream attribution replay на immutable RAVNOVES00: для каждого
|
||||
кадра сохранить один и тот же TensorRT output и посчитать, сколько объектов и
|
||||
каких risk-классов остаётся после каждой ступени:
|
||||
|
||||
1. raw top-300 output;
|
||||
2. COCO class mapping;
|
||||
3. confidence gate;
|
||||
4. valid-FOV и area gates;
|
||||
5. fixed risk-class qualification;
|
||||
6. tracker/temporal/world-state delivery.
|
||||
|
||||
Это локализует конкретную ступень потери `person`, animal, light-road-user и
|
||||
vehicle detections без добавления второй тяжёлой модели и без изменения FPS
|
||||
critical path. После attribution можно менять только доказанно виновный gate и
|
||||
повторять source-paced reference-graph load acceptance.
|
||||
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the frozen RF-DETR risk contour on independent COCO 2017 val truth."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from k1link.perception.m48t_risk_quality import (
|
||||
CocoRiskImage,
|
||||
RiskPrediction,
|
||||
RiskTruth,
|
||||
load_coco_risk_truth,
|
||||
load_m48t_risk_quality_profile,
|
||||
score_risk_quality,
|
||||
)
|
||||
from k1link.perception.rf_detr_object_detector import (
|
||||
TritonRfDetrHttpInferenceBackend,
|
||||
postprocess_rf_detr,
|
||||
preprocess_raw_kb4_rf_detr,
|
||||
)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--annotations", type=Path, required=True)
|
||||
parser.add_argument("--images-root", type=Path, required=True)
|
||||
parser.add_argument("--triton-origin", default="http://127.0.0.1:8000")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--predictions", type=Path, required=True)
|
||||
parser.add_argument("--failures", type=Path, required=True)
|
||||
parser.add_argument("--progress", type=Path, required=True)
|
||||
parser.add_argument("--review-root", type=Path, required=True)
|
||||
parser.add_argument("--runtime-artifact-sha256", required=True)
|
||||
parser.add_argument("--runner-sha256", required=True)
|
||||
parser.add_argument("--images-archive-sha256", required=True)
|
||||
parser.add_argument("--annotations-document-sha256", required=True)
|
||||
parser.add_argument("--maximum-images", type=int, default=0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
_require_sha256(arguments.runtime_artifact_sha256, "runtime artifact")
|
||||
_require_sha256(arguments.runner_sha256, "runner")
|
||||
_require_sha256(arguments.images_archive_sha256, "images archive")
|
||||
_require_sha256(arguments.annotations_document_sha256, "annotations document")
|
||||
if arguments.maximum_images < 0:
|
||||
raise RuntimeError("maximum images cannot be negative")
|
||||
for target in (
|
||||
arguments.output,
|
||||
arguments.predictions,
|
||||
arguments.failures,
|
||||
arguments.progress,
|
||||
):
|
||||
if target.exists():
|
||||
raise RuntimeError(f"output already exists: {target}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if arguments.review_root.exists():
|
||||
raise RuntimeError("review output already exists")
|
||||
arguments.review_root.mkdir(parents=True)
|
||||
|
||||
profile = load_m48t_risk_quality_profile(arguments.profile)
|
||||
images, truth = load_coco_risk_truth(arguments.annotations, profile)
|
||||
if arguments.maximum_images:
|
||||
images = images[: arguments.maximum_images]
|
||||
image_ids = {item.image_id for item in images}
|
||||
truth = tuple(item for item in truth if item.image_id in image_ids)
|
||||
if not images or not truth:
|
||||
raise RuntimeError("selected COCO risk set is empty")
|
||||
|
||||
mask = np.ones((600, 800), dtype=np.bool_)
|
||||
class_to_family = profile.class_to_family
|
||||
predictions: list[RiskPrediction] = []
|
||||
image_timings_ms: list[float] = []
|
||||
inference_timings_ms: list[float] = []
|
||||
rejected: Counter[str] = Counter()
|
||||
started_at = time.time_ns()
|
||||
backend = TritonRfDetrHttpInferenceBackend(arguments.triton_origin)
|
||||
with arguments.progress.open("x", encoding="utf-8") as progress:
|
||||
try:
|
||||
for image_index, image in enumerate(images, start=1):
|
||||
loop_started = time.perf_counter_ns()
|
||||
image_path = (arguments.images_root / image.file_name).resolve(strict=True)
|
||||
with Image.open(image_path) as opened:
|
||||
rgb = np.asarray(
|
||||
opened.convert("RGB").resize((800, 600), Image.Resampling.BILINEAR),
|
||||
dtype=np.uint8,
|
||||
)
|
||||
image_bgr = np.ascontiguousarray(rgb[:, :, ::-1])
|
||||
tensor = preprocess_raw_kb4_rf_detr(image_bgr, mask)
|
||||
inference_started = time.perf_counter_ns()
|
||||
output = backend.infer(tensor)
|
||||
inference_ms = (time.perf_counter_ns() - inference_started) / 1_000_000
|
||||
inference_timings_ms.append(inference_ms)
|
||||
processed = postprocess_rf_detr(output, mask)
|
||||
rejected.update(dict(processed.rejected))
|
||||
for detection_index, detection in enumerate(processed.detections, start=1):
|
||||
family = class_to_family.get(detection.label)
|
||||
if family is None:
|
||||
raise RuntimeError(
|
||||
"RF-DETR emitted a class outside the frozen risk profile"
|
||||
)
|
||||
predictions.append(
|
||||
RiskPrediction(
|
||||
image_id=image.image_id,
|
||||
prediction_id=f"{image.image_id:012d}:{detection_index:03d}",
|
||||
class_name=detection.label,
|
||||
family=family,
|
||||
score=detection.score,
|
||||
bbox_xyxy=detection.bbox_xyxy,
|
||||
)
|
||||
)
|
||||
image_ms = (time.perf_counter_ns() - loop_started) / 1_000_000
|
||||
image_timings_ms.append(image_ms)
|
||||
if image_index == 1 or image_index % 100 == 0 or image_index == len(images):
|
||||
progress.write(
|
||||
_canonical_json(
|
||||
{
|
||||
"schema_version": "missioncore.m48t-risk-quality-progress/v1",
|
||||
"completed_images": image_index,
|
||||
"total_images": len(images),
|
||||
"prediction_count": len(predictions),
|
||||
"last_image_ms": image_ms,
|
||||
"last_inference_ms": inference_ms,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
progress.flush()
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
result = score_risk_quality(
|
||||
images=images,
|
||||
truth=truth,
|
||||
predictions=tuple(predictions),
|
||||
profile=profile,
|
||||
)
|
||||
completed_at = time.time_ns()
|
||||
prediction_rows = tuple(_prediction_row(item) for item in predictions)
|
||||
_write_jsonl(arguments.predictions, prediction_rows)
|
||||
_write_jsonl(arguments.failures, result.failures)
|
||||
review_files = _render_review_cases(
|
||||
images_root=arguments.images_root,
|
||||
review_root=arguments.review_root,
|
||||
images=images,
|
||||
truth=truth,
|
||||
predictions=tuple(predictions),
|
||||
failures=result.failures,
|
||||
)
|
||||
report = dict(result.report)
|
||||
report["execution"] = {
|
||||
"worker": "DESKTOP-OPJ8J04",
|
||||
"started_at_unix_ns": started_at,
|
||||
"completed_at_unix_ns": completed_at,
|
||||
"duration_seconds": (completed_at - started_at) / 1_000_000_000,
|
||||
"image_timing_ms": _distribution(image_timings_ms),
|
||||
"triton_inference_ms": _distribution(inference_timings_ms),
|
||||
"effective_images_per_second": len(images)
|
||||
/ ((completed_at - started_at) / 1_000_000_000),
|
||||
"timing_is_admission_evidence": False,
|
||||
}
|
||||
report["provenance"] = {
|
||||
"profile_sha256": profile.profile_sha256,
|
||||
"annotations_document_sha256": _sha256(arguments.annotations),
|
||||
"images_archive_sha256": arguments.images_archive_sha256,
|
||||
"annotations_document_expected_sha256": arguments.annotations_document_sha256,
|
||||
"runtime_artifact_sha256": arguments.runtime_artifact_sha256,
|
||||
"runner_sha256": arguments.runner_sha256,
|
||||
"predictions_sha256": _sha256(arguments.predictions),
|
||||
"failures_sha256": _sha256(arguments.failures),
|
||||
}
|
||||
report["artifacts"] = {
|
||||
"predictions": arguments.predictions.name,
|
||||
"failures": arguments.failures.name,
|
||||
"progress": arguments.progress.name,
|
||||
"review_files": review_files,
|
||||
}
|
||||
report["detector_rejections"] = dict(sorted(rejected.items()))
|
||||
report["report_identity_sha256"] = hashlib.sha256(
|
||||
_canonical_json(
|
||||
{
|
||||
"profile_sha256": profile.profile_sha256,
|
||||
"dataset": report["dataset"],
|
||||
"counts": report["counts"],
|
||||
"metrics": report["metrics"],
|
||||
"failure_buckets": report["failure_buckets"],
|
||||
"quality_gates": report["quality_gates"],
|
||||
"provenance": report["provenance"],
|
||||
}
|
||||
).encode()
|
||||
).hexdigest()
|
||||
arguments.output.write_text(_canonical_json(report) + "\n", "utf-8")
|
||||
quality_gates = report.get("quality_gates")
|
||||
if not isinstance(quality_gates, dict) or not isinstance(
|
||||
quality_gates.get("passed"), bool
|
||||
):
|
||||
raise RuntimeError("quality gate result is invalid")
|
||||
print(
|
||||
_canonical_json(
|
||||
{
|
||||
"result": str(arguments.output),
|
||||
"risk_images": len(images),
|
||||
"truth_instances": len(truth),
|
||||
"predictions": len(predictions),
|
||||
"quality_gate_passed": quality_gates["passed"],
|
||||
"report_identity_sha256": report["report_identity_sha256"],
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def _prediction_row(item: RiskPrediction) -> dict[str, object]:
|
||||
return {
|
||||
"image_id": item.image_id,
|
||||
"prediction_id": item.prediction_id,
|
||||
"class_name": item.class_name,
|
||||
"family": item.family,
|
||||
"score": item.score,
|
||||
"bbox_xyxy": list(item.bbox_xyxy),
|
||||
}
|
||||
|
||||
|
||||
def _render_review_cases(
|
||||
*,
|
||||
images_root: Path,
|
||||
review_root: Path,
|
||||
images: tuple[CocoRiskImage, ...],
|
||||
truth: tuple[RiskTruth, ...],
|
||||
predictions: tuple[RiskPrediction, ...],
|
||||
failures: tuple[dict[str, object], ...],
|
||||
) -> list[str]:
|
||||
image_by_id = {item.image_id: item for item in images}
|
||||
selected_ids: list[int] = []
|
||||
for failure in failures:
|
||||
if failure.get("kind") != "false-negative":
|
||||
continue
|
||||
raw_image_id = failure.get("image_id")
|
||||
if not isinstance(raw_image_id, int) or isinstance(raw_image_id, bool):
|
||||
raise RuntimeError("failure image id is invalid")
|
||||
image_id = raw_image_id
|
||||
if image_id not in selected_ids:
|
||||
selected_ids.append(image_id)
|
||||
if len(selected_ids) == 16:
|
||||
break
|
||||
result: list[str] = []
|
||||
for image_id in selected_ids:
|
||||
metadata = image_by_id[image_id]
|
||||
with Image.open((images_root / metadata.file_name).resolve(strict=True)) as opened:
|
||||
canvas = opened.convert("RGB").resize((800, 600), Image.Resampling.BILINEAR)
|
||||
draw = ImageDraw.Draw(canvas)
|
||||
for truth_item in truth:
|
||||
if truth_item.image_id != image_id:
|
||||
continue
|
||||
draw.rectangle(truth_item.bbox_xyxy, outline=(80, 230, 120), width=3)
|
||||
draw.text(
|
||||
(truth_item.bbox_xyxy[0] + 2, truth_item.bbox_xyxy[1] + 2),
|
||||
f"GT {truth_item.class_name}",
|
||||
fill=(80, 230, 120),
|
||||
)
|
||||
for prediction_item in predictions:
|
||||
if prediction_item.image_id != image_id:
|
||||
continue
|
||||
draw.rectangle(prediction_item.bbox_xyxy, outline=(255, 200, 50), width=2)
|
||||
draw.text(
|
||||
(prediction_item.bbox_xyxy[0] + 2, prediction_item.bbox_xyxy[3] - 13),
|
||||
f"P {prediction_item.class_name} {prediction_item.score:.2f}",
|
||||
fill=(255, 200, 50),
|
||||
)
|
||||
name = f"review-{image_id:012d}.jpg"
|
||||
canvas.save(review_root / name, format="JPEG", quality=90, optimize=True)
|
||||
result.append(f"review/{name}")
|
||||
return result
|
||||
|
||||
|
||||
def _distribution(values: list[float]) -> dict[str, float]:
|
||||
if not values:
|
||||
raise RuntimeError("timing distribution is empty")
|
||||
ordered = sorted(values)
|
||||
return {
|
||||
"count": float(len(ordered)),
|
||||
"mean": statistics.fmean(ordered),
|
||||
"p50": _percentile(ordered, 0.50),
|
||||
"p95": _percentile(ordered, 0.95),
|
||||
"p99": _percentile(ordered, 0.99),
|
||||
"maximum": ordered[-1],
|
||||
}
|
||||
|
||||
|
||||
def _percentile(values: list[float], quantile: float) -> float:
|
||||
position = (len(values) - 1) * quantile
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return values[lower]
|
||||
return values[lower] + (values[upper] - values[lower]) * (position - lower)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: tuple[dict[str, object], ...]) -> None:
|
||||
with path.open("x", encoding="utf-8") as handle:
|
||||
for row in rows:
|
||||
handle.write(_canonical_json(row) + "\n")
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _require_sha256(value: str, label: str) -> None:
|
||||
if len(value) != 64 or any(character not in "0123456789abcdef" for character in value):
|
||||
raise RuntimeError(f"{label} SHA-256 is invalid")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay M4.8T semantic stabilization over geometry-owned component ids."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.perception.m48t_risk_quality import (
|
||||
BoundedTemporalSemanticIdentity,
|
||||
TemporalSemanticObservation,
|
||||
load_m48t_risk_quality_profile,
|
||||
)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
repository = Path(__file__).resolve().parents[2]
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
type=Path,
|
||||
default=repository / "config/perception/m48t-risk-quality-temporal-v1.json",
|
||||
)
|
||||
parser.add_argument("--frames", type=Path, required=True)
|
||||
parser.add_argument("--expected-frames-sha256", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
if arguments.output.exists():
|
||||
raise RuntimeError("temporal semantic output already exists")
|
||||
if not _is_sha256(arguments.expected_frames_sha256):
|
||||
raise RuntimeError("expected frame ledger SHA-256 is invalid")
|
||||
profile = load_m48t_risk_quality_profile(arguments.profile)
|
||||
stabilizer = BoundedTemporalSemanticIdentity(profile)
|
||||
digest = hashlib.sha256()
|
||||
frames = 0
|
||||
publications = 0
|
||||
semantic_current = 0
|
||||
selected_publications = 0
|
||||
raw_switches = 0
|
||||
stable_switches = 0
|
||||
raw_family_switches = 0
|
||||
stable_family_switches = 0
|
||||
rolling_retained_skipped = 0
|
||||
resolutions: Counter[str] = Counter()
|
||||
stable_families: Counter[str] = Counter()
|
||||
last_raw: dict[str, str] = {}
|
||||
last_stable: dict[str, str] = {}
|
||||
class_to_family = profile.class_to_family
|
||||
|
||||
with arguments.frames.expanduser().resolve(strict=True).open("rb") as handle:
|
||||
for line_number, raw_line in enumerate(handle, start=1):
|
||||
digest.update(raw_line)
|
||||
try:
|
||||
document = json.loads(raw_line)
|
||||
frame_time_ns = document["source_envelope"]["timestamps"]["source_ns"]
|
||||
occupied = document["delivery"]["obstacle_map"]["occupied"]
|
||||
except (KeyError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"frame ledger row {line_number} is invalid") from exc
|
||||
if not isinstance(frame_time_ns, int) or not isinstance(occupied, list):
|
||||
raise RuntimeError(f"frame ledger row {line_number} contract changed")
|
||||
frames += 1
|
||||
for value in occupied:
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError("temporal obstacle is invalid")
|
||||
component_id = value.get("component_id")
|
||||
state = value.get("state")
|
||||
if state == "retained":
|
||||
rolling_retained_skipped += 1
|
||||
continue
|
||||
raw_class = value.get("semantic_hint") if state == "current" else None
|
||||
if not isinstance(component_id, str) or state not in {
|
||||
"current",
|
||||
"held",
|
||||
"expired",
|
||||
}:
|
||||
raise RuntimeError("temporal obstacle identity or state changed")
|
||||
if raw_class is not None and not isinstance(raw_class, str):
|
||||
raise RuntimeError("temporal semantic hint is invalid")
|
||||
if raw_class is not None:
|
||||
semantic_current += 1
|
||||
previous_raw = last_raw.get(component_id)
|
||||
if previous_raw is not None and previous_raw != raw_class:
|
||||
raw_switches += 1
|
||||
if class_to_family[previous_raw] != class_to_family[raw_class]:
|
||||
raw_family_switches += 1
|
||||
last_raw[component_id] = raw_class
|
||||
result = stabilizer.update(
|
||||
TemporalSemanticObservation(
|
||||
component_id=component_id,
|
||||
evidence_time_ns=frame_time_ns,
|
||||
raw_class_name=raw_class,
|
||||
currentness=state,
|
||||
)
|
||||
)
|
||||
publications += 1
|
||||
resolutions[result.resolution] += 1
|
||||
if result.selected_class_name is not None:
|
||||
selected_publications += 1
|
||||
if result.selected_family is None:
|
||||
raise RuntimeError("selected semantic family is missing")
|
||||
stable_families[result.selected_family] += 1
|
||||
previous_stable = last_stable.get(component_id)
|
||||
if (
|
||||
previous_stable is not None
|
||||
and previous_stable != result.selected_class_name
|
||||
):
|
||||
stable_switches += 1
|
||||
if (
|
||||
class_to_family[previous_stable]
|
||||
!= class_to_family[result.selected_class_name]
|
||||
):
|
||||
stable_family_switches += 1
|
||||
last_stable[component_id] = result.selected_class_name
|
||||
|
||||
frames_sha256 = digest.hexdigest()
|
||||
if frames_sha256 != arguments.expected_frames_sha256:
|
||||
raise RuntimeError("temporal frame ledger SHA-256 changed")
|
||||
snapshot = stabilizer.snapshot()
|
||||
gate_checks = {
|
||||
"all-publications-accounted": snapshot.input_observations == publications,
|
||||
"bounded-active-components": (
|
||||
snapshot.peak_active_components <= profile.temporal.maximum_active_components
|
||||
),
|
||||
"stable-switches-not-greater-than-raw-switches": stable_switches <= raw_switches,
|
||||
"stable-family-switches-not-greater-than-raw-family-switches": (
|
||||
stable_family_switches <= raw_family_switches
|
||||
),
|
||||
"association-remains-class-independent": True,
|
||||
"occupancy-remains-class-independent": True,
|
||||
}
|
||||
report = {
|
||||
"schema_version": "missioncore.m48t-temporal-semantic-shadow/v1",
|
||||
"profile_id": profile.profile_id,
|
||||
"profile_sha256": profile.profile_sha256,
|
||||
"source": {
|
||||
"source_id": "RAVNOVES00",
|
||||
"frame_ledger": str(arguments.frames),
|
||||
"frame_ledger_sha256": frames_sha256,
|
||||
"frames": frames,
|
||||
"publications": publications,
|
||||
"independent_semantic_truth_available": False,
|
||||
},
|
||||
"metrics": {
|
||||
"semantic_current_observations": semantic_current,
|
||||
"rolling_retained_publications_skipped": rolling_retained_skipped,
|
||||
"selected_publications": selected_publications,
|
||||
"raw_class_switches": raw_switches,
|
||||
"stable_class_switches": stable_switches,
|
||||
"suppressed_or_deferred_class_switches": raw_switches - stable_switches,
|
||||
"raw_family_switches": raw_family_switches,
|
||||
"stable_family_switches": stable_family_switches,
|
||||
"suppressed_or_deferred_family_switches": (
|
||||
raw_family_switches - stable_family_switches
|
||||
),
|
||||
"resolutions": dict(sorted(resolutions.items())),
|
||||
"stable_family_publications": dict(sorted(stable_families.items())),
|
||||
"snapshot": {
|
||||
field: getattr(snapshot, field)
|
||||
for field in snapshot.__dataclass_fields__
|
||||
},
|
||||
},
|
||||
"temporal_invariant_gate": {
|
||||
"checks": gate_checks,
|
||||
"passed": all(gate_checks.values()),
|
||||
"semantic_quality_accepted": False,
|
||||
},
|
||||
"policy": {
|
||||
"initial_confirmation_observations": (
|
||||
profile.temporal.initial_confirmation_observations
|
||||
),
|
||||
"switch_confirmation_observations": (
|
||||
profile.temporal.switch_confirmation_observations
|
||||
),
|
||||
"semantic_hold_seconds": profile.temporal.semantic_hold_seconds,
|
||||
"state_expiry_seconds": profile.temporal.state_expiry_seconds,
|
||||
"history_size": profile.temporal.history_size,
|
||||
"maximum_active_components": profile.temporal.maximum_active_components,
|
||||
"cross_family_conflict_fallback": "unknown",
|
||||
"association_uses_semantic_class": False,
|
||||
"occupancy_uses_semantic_class": False,
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth_for_ravnoves00": False,
|
||||
"candidate_accepted": False,
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
arguments.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
arguments.output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
"utf-8",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(arguments.output),
|
||||
"frames": frames,
|
||||
"publications": publications,
|
||||
"raw_class_switches": raw_switches,
|
||||
"stable_class_switches": stable_switches,
|
||||
"invariant_gate_passed": all(gate_checks.values()),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _is_sha256(value: object) -> bool:
|
||||
return (
|
||||
isinstance(value, str)
|
||||
and len(value) == 64
|
||||
and all(character in "0123456789abcdef" for character in value)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,618 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproduce official RF-DETR COCO quality and compare the pinned TensorRT engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import gc
|
||||
import gzip
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import time
|
||||
from collections.abc import Iterable, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from k1link.perception.rf_detr_object_detector import (
|
||||
COCO_SPARSE_IDS,
|
||||
TritonRfDetrHttpInferenceBackend,
|
||||
)
|
||||
|
||||
PROFILE_SCHEMA: Final = "missioncore.m48t-upstream-parity-profile/v1"
|
||||
REPORT_SCHEMA: Final = "missioncore.m48t-upstream-parity-report/v1"
|
||||
PROGRESS_SCHEMA: Final = "missioncore.m48t-upstream-parity-progress/v1"
|
||||
FALSE_AUTHORITY: Final = {
|
||||
"candidate_accepted": False,
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
COCO_METRIC_NAMES: Final = (
|
||||
"ap_50_95",
|
||||
"ap_50",
|
||||
"ap_75",
|
||||
"ap_small",
|
||||
"ap_medium",
|
||||
"ap_large",
|
||||
"ar_1",
|
||||
"ar_10",
|
||||
"ar_100",
|
||||
"ar_small",
|
||||
"ar_medium",
|
||||
"ar_large",
|
||||
)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--annotations", type=Path, required=True)
|
||||
parser.add_argument("--images-root", type=Path, required=True)
|
||||
parser.add_argument("--checkpoint", type=Path, required=True)
|
||||
parser.add_argument("--triton-origin", default="http://127.0.0.1:8000")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--pytorch-predictions", type=Path, required=True)
|
||||
parser.add_argument("--tensorrt-predictions", type=Path, required=True)
|
||||
parser.add_argument("--progress", type=Path, required=True)
|
||||
parser.add_argument("--runtime-image", required=True)
|
||||
parser.add_argument("--maximum-images", type=int, default=0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = parse_arguments()
|
||||
profile, profile_sha256 = load_profile(arguments.profile)
|
||||
if arguments.maximum_images < 0:
|
||||
raise RuntimeError("maximum images cannot be negative")
|
||||
for target in (
|
||||
arguments.output,
|
||||
arguments.pytorch_predictions,
|
||||
arguments.tensorrt_predictions,
|
||||
arguments.progress,
|
||||
):
|
||||
if target.exists():
|
||||
raise RuntimeError(f"output already exists: {target}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_profile = _object(profile.get("model"), "model")
|
||||
dataset_profile = _object(profile.get("dataset"), "dataset")
|
||||
annotations_sha256 = sha256_path(arguments.annotations)
|
||||
if annotations_sha256 != _string(dataset_profile, "annotations_sha256"):
|
||||
raise RuntimeError("COCO annotations SHA-256 changed")
|
||||
if sha256_path(arguments.checkpoint) != _string(model_profile, "checkpoint_sha256"):
|
||||
raise RuntimeError("RF-DETR checkpoint SHA-256 changed")
|
||||
if importlib.metadata.version("rfdetr") != _string(model_profile, "package_version"):
|
||||
raise RuntimeError("RF-DETR package version changed")
|
||||
|
||||
coco_document = _object(json.loads(arguments.annotations.read_text("utf-8")), "COCO")
|
||||
images = load_coco_images(coco_document)
|
||||
expected_count = _integer(dataset_profile, "image_count")
|
||||
if len(images) != expected_count:
|
||||
raise RuntimeError("COCO image count changed")
|
||||
if arguments.maximum_images:
|
||||
images = images[: arguments.maximum_images]
|
||||
if not images:
|
||||
raise RuntimeError("COCO parity selection is empty")
|
||||
category_ids_by_name = load_category_ids_by_name(coco_document)
|
||||
image_ids = [_integer(item, "id") for item in images]
|
||||
full_admission_run = len(images) == expected_count
|
||||
|
||||
started_at = time.time_ns()
|
||||
with arguments.progress.open("x", encoding="utf-8") as progress:
|
||||
pytorch_rows, pytorch_timing = run_pytorch_predictions(
|
||||
images=images,
|
||||
images_root=arguments.images_root,
|
||||
checkpoint=arguments.checkpoint,
|
||||
category_ids_by_name=category_ids_by_name,
|
||||
progress=progress,
|
||||
)
|
||||
write_gzip_jsonl(arguments.pytorch_predictions, pytorch_rows)
|
||||
pytorch_metrics, pytorch_summary = evaluate_coco(
|
||||
annotations=arguments.annotations,
|
||||
predictions=pytorch_rows,
|
||||
image_ids=image_ids,
|
||||
)
|
||||
pytorch_count = len(pytorch_rows)
|
||||
del pytorch_rows
|
||||
gc.collect()
|
||||
|
||||
tensorrt_rows, tensorrt_timing = run_tensorrt_predictions(
|
||||
images=images,
|
||||
images_root=arguments.images_root,
|
||||
triton_origin=arguments.triton_origin,
|
||||
progress=progress,
|
||||
)
|
||||
write_gzip_jsonl(arguments.tensorrt_predictions, tensorrt_rows)
|
||||
tensorrt_metrics, tensorrt_summary = evaluate_coco(
|
||||
annotations=arguments.annotations,
|
||||
predictions=tensorrt_rows,
|
||||
image_ids=image_ids,
|
||||
)
|
||||
tensorrt_count = len(tensorrt_rows)
|
||||
del tensorrt_rows
|
||||
gc.collect()
|
||||
|
||||
decision = build_parity_decision(
|
||||
profile=profile,
|
||||
pytorch_metrics=pytorch_metrics,
|
||||
tensorrt_metrics=tensorrt_metrics,
|
||||
full_admission_run=full_admission_run,
|
||||
)
|
||||
completed_at = time.time_ns()
|
||||
report: dict[str, object] = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"profile_id": _string(profile, "profile_id"),
|
||||
"dataset": {
|
||||
"dataset_id": _string(dataset_profile, "dataset_id"),
|
||||
"image_count": len(images),
|
||||
"full_admission_run": full_admission_run,
|
||||
"all_categories": True,
|
||||
"official_crowd_ignore": True,
|
||||
"custom_area_filtering": False,
|
||||
"confidence_prefilter": 0.0,
|
||||
},
|
||||
"providers": {
|
||||
"pytorch": {
|
||||
"prediction_count": pytorch_count,
|
||||
"metrics": pytorch_metrics,
|
||||
"coco_summary": pytorch_summary,
|
||||
"timing": pytorch_timing,
|
||||
},
|
||||
"tensorrt": {
|
||||
"prediction_count": tensorrt_count,
|
||||
"metrics": tensorrt_metrics,
|
||||
"coco_summary": tensorrt_summary,
|
||||
"timing": tensorrt_timing,
|
||||
},
|
||||
},
|
||||
"decision": decision,
|
||||
"execution": {
|
||||
"worker": "DESKTOP-OPJ8J04",
|
||||
"runtime_image": arguments.runtime_image,
|
||||
"started_at_unix_ns": started_at,
|
||||
"completed_at_unix_ns": completed_at,
|
||||
"duration_seconds": (completed_at - started_at) / 1_000_000_000,
|
||||
"packages": package_versions(
|
||||
("rfdetr", "torch", "torchvision", "numpy", "pycocotools", "tritonclient")
|
||||
),
|
||||
},
|
||||
"provenance": {
|
||||
"profile_sha256": profile_sha256,
|
||||
"annotations_sha256": annotations_sha256,
|
||||
"checkpoint_sha256": sha256_path(arguments.checkpoint),
|
||||
"pytorch_predictions_sha256": sha256_path(arguments.pytorch_predictions),
|
||||
"tensorrt_predictions_sha256": sha256_path(arguments.tensorrt_predictions),
|
||||
},
|
||||
"authority": FALSE_AUTHORITY,
|
||||
}
|
||||
report["report_identity_sha256"] = hashlib.sha256(canonical_json(report).encode()).hexdigest()
|
||||
arguments.output.write_text(canonical_json(report) + "\n", "utf-8")
|
||||
print(
|
||||
canonical_json(
|
||||
{
|
||||
"result": str(arguments.output),
|
||||
"image_count": len(images),
|
||||
"pytorch_ap_50_95": pytorch_metrics["ap_50_95"],
|
||||
"tensorrt_ap_50_95": tensorrt_metrics["ap_50_95"],
|
||||
"diagnosis": decision["diagnosis"],
|
||||
"passed": decision["passed"],
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def load_profile(path: Path) -> tuple[dict[str, object], str]:
|
||||
raw = path.read_bytes()
|
||||
profile = _object(json.loads(raw), "profile")
|
||||
if profile.get("schema_version") != PROFILE_SCHEMA:
|
||||
raise RuntimeError("upstream parity profile schema changed")
|
||||
if _object(profile.get("authority"), "authority") != FALSE_AUTHORITY:
|
||||
raise RuntimeError("upstream parity authority changed")
|
||||
return profile, hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def load_coco_images(document: Mapping[str, object]) -> list[dict[str, object]]:
|
||||
values = document.get("images")
|
||||
if not isinstance(values, list):
|
||||
raise RuntimeError("COCO images are invalid")
|
||||
images = [_object(value, "COCO image") for value in values]
|
||||
for image in images:
|
||||
_integer(image, "id")
|
||||
_positive_integer(image, "width")
|
||||
_positive_integer(image, "height")
|
||||
_string(image, "file_name")
|
||||
return sorted(images, key=lambda item: _integer(item, "id"))
|
||||
|
||||
|
||||
def load_category_ids_by_name(document: Mapping[str, object]) -> dict[str, int]:
|
||||
values = document.get("categories")
|
||||
if not isinstance(values, list):
|
||||
raise RuntimeError("COCO categories are invalid")
|
||||
result: dict[str, int] = {}
|
||||
for value in values:
|
||||
category = _object(value, "COCO category")
|
||||
name = _string(category, "name")
|
||||
category_id = _positive_integer(category, "id")
|
||||
if name in result:
|
||||
raise RuntimeError("COCO category names are duplicated")
|
||||
result[name] = category_id
|
||||
if set(result.values()) != set(COCO_SPARSE_IDS):
|
||||
raise RuntimeError("COCO sparse category ids changed")
|
||||
return result
|
||||
|
||||
|
||||
def run_pytorch_predictions(
|
||||
*,
|
||||
images: list[dict[str, object]],
|
||||
images_root: Path,
|
||||
checkpoint: Path,
|
||||
category_ids_by_name: Mapping[str, int],
|
||||
progress: Any,
|
||||
) -> tuple[list[dict[str, float | int | list[float]]], dict[str, object]]:
|
||||
import torch # type: ignore[import-not-found]
|
||||
from rfdetr import RFDETRLarge # type: ignore[import-not-found]
|
||||
|
||||
started_at = time.perf_counter_ns()
|
||||
model = RFDETRLarge(pretrain_weights=str(checkpoint))
|
||||
model.inference(compile=False, dtype=torch.float16, inplace=True)
|
||||
rows: list[dict[str, float | int | list[float]]] = []
|
||||
timings: list[float] = []
|
||||
try:
|
||||
for index, image in enumerate(images, start=1):
|
||||
image_started = time.perf_counter_ns()
|
||||
image_path = (images_root / _string(image, "file_name")).resolve(strict=True)
|
||||
with Image.open(image_path) as opened:
|
||||
prediction = model.predict(
|
||||
opened.convert("RGB"), threshold=0.0, include_source_image=False
|
||||
)
|
||||
boxes = np.asarray(prediction.xyxy, dtype=np.float32)
|
||||
scores = np.asarray(prediction.confidence, dtype=np.float32)
|
||||
names = np.asarray(prediction.data["class_name"])
|
||||
if not (len(boxes) == len(scores) == len(names)):
|
||||
raise RuntimeError("official RF-DETR prediction columns disagree")
|
||||
for box, score, raw_name in zip(boxes, scores, names, strict=True):
|
||||
name = str(raw_name)
|
||||
category_id = category_ids_by_name.get(name)
|
||||
if category_id is None:
|
||||
continue
|
||||
row = coco_prediction_row(
|
||||
image_id=_integer(image, "id"),
|
||||
category_id=category_id,
|
||||
score=float(score),
|
||||
bbox_xyxy=tuple(float(value) for value in box),
|
||||
image_width=_positive_integer(image, "width"),
|
||||
image_height=_positive_integer(image, "height"),
|
||||
)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
timings.append((time.perf_counter_ns() - image_started) / 1_000_000)
|
||||
write_progress(progress, "pytorch", index, len(images), len(rows), timings[-1])
|
||||
finally:
|
||||
del model
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return rows, timing_report(started_at, timings)
|
||||
|
||||
|
||||
def run_tensorrt_predictions(
|
||||
*,
|
||||
images: list[dict[str, object]],
|
||||
images_root: Path,
|
||||
triton_origin: str,
|
||||
progress: Any,
|
||||
) -> tuple[list[dict[str, float | int | list[float]]], dict[str, object]]:
|
||||
import torch
|
||||
import torchvision.transforms.functional as vision_functional # type: ignore[import-not-found]
|
||||
|
||||
means = [0.485, 0.456, 0.406]
|
||||
stds = [0.229, 0.224, 0.225]
|
||||
backend = TritonRfDetrHttpInferenceBackend(triton_origin)
|
||||
rows: list[dict[str, float | int | list[float]]] = []
|
||||
timings: list[float] = []
|
||||
started_at = time.perf_counter_ns()
|
||||
try:
|
||||
for index, image in enumerate(images, start=1):
|
||||
image_started = time.perf_counter_ns()
|
||||
image_path = (images_root / _string(image, "file_name")).resolve(strict=True)
|
||||
with Image.open(image_path) as opened:
|
||||
rgb = opened.convert("RGB")
|
||||
tensor = vision_functional.to_tensor(rgb)
|
||||
tensor = vision_functional.resize(tensor, [704, 704], antialias=False)
|
||||
tensor = vision_functional.normalize(tensor, means, stds)
|
||||
batch = np.ascontiguousarray(tensor.unsqueeze(0).numpy(), dtype=np.float32)
|
||||
output = backend.infer(batch)
|
||||
rows.extend(
|
||||
decode_tensorrt_coco_rows(
|
||||
image_id=_integer(image, "id"),
|
||||
image_width=_positive_integer(image, "width"),
|
||||
image_height=_positive_integer(image, "height"),
|
||||
boxes=output.boxes,
|
||||
logits=output.logits,
|
||||
maximum_detections=300,
|
||||
)
|
||||
)
|
||||
timings.append((time.perf_counter_ns() - image_started) / 1_000_000)
|
||||
write_progress(progress, "tensorrt", index, len(images), len(rows), timings[-1])
|
||||
finally:
|
||||
backend.close()
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
return rows, timing_report(started_at, timings)
|
||||
|
||||
|
||||
def decode_tensorrt_coco_rows(
|
||||
*,
|
||||
image_id: int,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
boxes: np.ndarray[Any, Any],
|
||||
logits: np.ndarray[Any, Any],
|
||||
maximum_detections: int,
|
||||
) -> list[dict[str, float | int | list[float]]]:
|
||||
if boxes.shape != (1, 300, 4) or logits.shape != (1, 300, 91):
|
||||
raise RuntimeError("RF-DETR TensorRT output shape changed")
|
||||
probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits[0].astype(np.float32), -80, 80)))
|
||||
flattened = probabilities.reshape(-1)
|
||||
topk = np.argsort(-flattened, kind="stable")[:maximum_detections]
|
||||
valid_category_ids = set(COCO_SPARSE_IDS)
|
||||
result: list[dict[str, float | int | list[float]]] = []
|
||||
for flat_index in topk:
|
||||
category_id = int(flat_index % logits.shape[2])
|
||||
if category_id not in valid_category_ids:
|
||||
continue
|
||||
query_index = int(flat_index // logits.shape[2])
|
||||
center_x, center_y, width, height = (
|
||||
float(value) for value in boxes[0, query_index].astype(np.float32)
|
||||
)
|
||||
row = coco_prediction_row(
|
||||
image_id=image_id,
|
||||
category_id=category_id,
|
||||
score=float(flattened[flat_index]),
|
||||
bbox_xyxy=(
|
||||
(center_x - width / 2) * image_width,
|
||||
(center_y - height / 2) * image_height,
|
||||
(center_x + width / 2) * image_width,
|
||||
(center_y + height / 2) * image_height,
|
||||
),
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
)
|
||||
if row is not None:
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
def coco_prediction_row(
|
||||
*,
|
||||
image_id: int,
|
||||
category_id: int,
|
||||
score: float,
|
||||
bbox_xyxy: tuple[float, ...],
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
) -> dict[str, float | int | list[float]] | None:
|
||||
if len(bbox_xyxy) != 4 or not all(math.isfinite(value) for value in bbox_xyxy):
|
||||
raise RuntimeError("prediction box is invalid")
|
||||
if not math.isfinite(score) or not 0 <= score <= 1:
|
||||
raise RuntimeError("prediction score is invalid")
|
||||
x1 = min(max(bbox_xyxy[0], 0.0), float(image_width))
|
||||
y1 = min(max(bbox_xyxy[1], 0.0), float(image_height))
|
||||
x2 = min(max(bbox_xyxy[2], 0.0), float(image_width))
|
||||
y2 = min(max(bbox_xyxy[3], 0.0), float(image_height))
|
||||
width = x2 - x1
|
||||
height = y2 - y1
|
||||
if width <= 0 or height <= 0:
|
||||
return None
|
||||
return {
|
||||
"image_id": image_id,
|
||||
"category_id": category_id,
|
||||
"bbox": [x1, y1, width, height],
|
||||
"score": score,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_coco(
|
||||
*,
|
||||
annotations: Path,
|
||||
predictions: list[dict[str, float | int | list[float]]],
|
||||
image_ids: list[int],
|
||||
) -> tuple[dict[str, float], str]:
|
||||
from pycocotools.coco import COCO # type: ignore[import-untyped]
|
||||
from pycocotools.cocoeval import COCOeval # type: ignore[import-untyped]
|
||||
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stdout(output):
|
||||
truth = COCO(str(annotations))
|
||||
detections = truth.loadRes(predictions)
|
||||
evaluator = COCOeval(truth, detections, "bbox")
|
||||
evaluator.params.imgIds = image_ids
|
||||
evaluator.evaluate()
|
||||
evaluator.accumulate()
|
||||
evaluator.summarize()
|
||||
metrics = {
|
||||
name: round(float(value), 9)
|
||||
for name, value in zip(COCO_METRIC_NAMES, evaluator.stats, strict=True)
|
||||
}
|
||||
return metrics, output.getvalue()
|
||||
|
||||
|
||||
def build_parity_decision(
|
||||
*,
|
||||
profile: Mapping[str, object],
|
||||
pytorch_metrics: Mapping[str, float],
|
||||
tensorrt_metrics: Mapping[str, float],
|
||||
full_admission_run: bool,
|
||||
) -> dict[str, object]:
|
||||
reference = _object(profile.get("published_reference"), "published_reference")
|
||||
gates = _object(profile.get("parity_gates"), "parity_gates")
|
||||
reproduction_tolerance = _number(reference, "maximum_absolute_reproduction_delta")
|
||||
pytorch_reference_deltas = {
|
||||
"ap_50_95": abs(pytorch_metrics["ap_50_95"] - _number(reference, "coco_ap_50_95")),
|
||||
"ap_50": abs(pytorch_metrics["ap_50"] - _number(reference, "coco_ap_50")),
|
||||
}
|
||||
provider_deltas = {
|
||||
"ap_50_95": abs(tensorrt_metrics["ap_50_95"] - pytorch_metrics["ap_50_95"]),
|
||||
"ap_50": abs(tensorrt_metrics["ap_50"] - pytorch_metrics["ap_50"]),
|
||||
}
|
||||
reproduction_passed = all(
|
||||
value <= reproduction_tolerance for value in pytorch_reference_deltas.values()
|
||||
)
|
||||
provider_parity_passed = provider_deltas["ap_50_95"] <= _number(
|
||||
gates, "maximum_absolute_ap_50_95_delta"
|
||||
) and provider_deltas["ap_50"] <= _number(gates, "maximum_absolute_ap_50_delta")
|
||||
if not full_admission_run:
|
||||
diagnosis = "bounded-smoke-only"
|
||||
elif not reproduction_passed:
|
||||
diagnosis = "upstream-reproduction-failed"
|
||||
elif not provider_parity_passed:
|
||||
diagnosis = "tensorrt-deployment-parity-failed"
|
||||
else:
|
||||
diagnosis = "upstream-and-tensorrt-parity-passed"
|
||||
passed = full_admission_run and reproduction_passed and provider_parity_passed
|
||||
return {
|
||||
"passed": passed,
|
||||
"diagnosis": diagnosis,
|
||||
"full_admission_run": full_admission_run,
|
||||
"pytorch_reference_deltas": pytorch_reference_deltas,
|
||||
"provider_deltas": provider_deltas,
|
||||
"upstream_reproduction_passed": reproduction_passed,
|
||||
"provider_parity_passed": provider_parity_passed,
|
||||
"semantic_authority_changed": False,
|
||||
}
|
||||
|
||||
|
||||
def write_progress(
|
||||
progress: Any,
|
||||
provider: str,
|
||||
completed: int,
|
||||
total: int,
|
||||
prediction_count: int,
|
||||
last_image_ms: float,
|
||||
) -> None:
|
||||
if completed != 1 and completed % 100 != 0 and completed != total:
|
||||
return
|
||||
progress.write(
|
||||
canonical_json(
|
||||
{
|
||||
"schema_version": PROGRESS_SCHEMA,
|
||||
"provider": provider,
|
||||
"completed_images": completed,
|
||||
"total_images": total,
|
||||
"prediction_count": prediction_count,
|
||||
"last_image_ms": last_image_ms,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
progress.flush()
|
||||
|
||||
|
||||
def timing_report(started_at: int, timings: list[float]) -> dict[str, object]:
|
||||
if not timings:
|
||||
raise RuntimeError("provider timing is empty")
|
||||
duration_seconds = (time.perf_counter_ns() - started_at) / 1_000_000_000
|
||||
ordered = sorted(timings)
|
||||
return {
|
||||
"duration_seconds": duration_seconds,
|
||||
"effective_images_per_second": len(timings) / duration_seconds,
|
||||
"image_ms": {
|
||||
"mean": statistics.fmean(ordered),
|
||||
"p50": percentile(ordered, 0.5),
|
||||
"p95": percentile(ordered, 0.95),
|
||||
"p99": percentile(ordered, 0.99),
|
||||
"maximum": ordered[-1],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def percentile(values: list[float], quantile: float) -> float:
|
||||
position = (len(values) - 1) * quantile
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return values[lower]
|
||||
return values[lower] + (values[upper] - values[lower]) * (position - lower)
|
||||
|
||||
|
||||
def write_gzip_jsonl(path: Path, rows: Iterable[Mapping[str, float | int | list[float]]]) -> None:
|
||||
with (
|
||||
path.open("xb") as raw_handle,
|
||||
gzip.GzipFile(fileobj=raw_handle, mode="wb", compresslevel=6, mtime=0) as compressed,
|
||||
io.TextIOWrapper(compressed, encoding="utf-8") as handle,
|
||||
):
|
||||
for row in rows:
|
||||
handle.write(canonical_json(row) + "\n")
|
||||
|
||||
|
||||
def package_versions(names: Iterable[str]) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
for name in names:
|
||||
try:
|
||||
result[name] = importlib.metadata.version(name)
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
result[name] = "not-installed"
|
||||
return result
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def canonical_json(value: object) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise RuntimeError(f"{label} is not an object")
|
||||
return value
|
||||
|
||||
|
||||
def _string(value: Mapping[str, object], key: str) -> str:
|
||||
result = value.get(key)
|
||||
if not isinstance(result, str) or not result:
|
||||
raise RuntimeError(f"{key} is not a non-empty string")
|
||||
return result
|
||||
|
||||
|
||||
def _integer(value: Mapping[str, object], key: str) -> int:
|
||||
result = value.get(key)
|
||||
if not isinstance(result, int) or isinstance(result, bool):
|
||||
raise RuntimeError(f"{key} is not an integer")
|
||||
return result
|
||||
|
||||
|
||||
def _positive_integer(value: Mapping[str, object], key: str) -> int:
|
||||
result = _integer(value, key)
|
||||
if result <= 0:
|
||||
raise RuntimeError(f"{key} is not positive")
|
||||
return result
|
||||
|
||||
|
||||
def _number(value: Mapping[str, object], key: str) -> float:
|
||||
result = value.get(key)
|
||||
if not isinstance(result, (int, float)) or isinstance(result, bool):
|
||||
raise RuntimeError(f"{key} is not numeric")
|
||||
converted = float(result)
|
||||
if not math.isfinite(converted):
|
||||
raise RuntimeError(f"{key} is not finite")
|
||||
return converted
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,290 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ReleaseRoot,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||
[string]$ExpectedWheelSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
||||
[string]$RunId,
|
||||
[ValidateRange(0, 5000)]
|
||||
[int]$MaximumImages = 0,
|
||||
[string]$DatasetRoot = "D:\NDC_MISSIONCORE\datasets\coco-2017-val",
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48t-risk-quality"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
||||
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
||||
$null = New-Item -ItemType Directory -Path $Path
|
||||
}
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if (
|
||||
-not $item.PSIsContainer -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
||||
) {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Assert-RegularFile([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "$Label must be a regular file"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) {
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Ensure-PinnedMirrorFile(
|
||||
[string]$Path,
|
||||
[string]$Url,
|
||||
[long]$ExpectedLength,
|
||||
[string]$ExpectedSha256,
|
||||
[string]$Label
|
||||
) {
|
||||
if (Test-Path -LiteralPath $Path -PathType Leaf) {
|
||||
if (
|
||||
(Get-Item -LiteralPath $Path).Length -ne $ExpectedLength -or
|
||||
(Get-Sha256 $Path) -cne $ExpectedSha256
|
||||
) {
|
||||
Remove-Item -LiteralPath $Path -Force
|
||||
}
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
$partial = $Path + ".partial"
|
||||
if (Test-Path -LiteralPath $partial) { Remove-Item -LiteralPath $partial -Force }
|
||||
& curl.exe --fail --location --retry 5 `
|
||||
--speed-limit 1048576 --speed-time 30 --output $partial $Url
|
||||
Assert-LastExitCode "$Label download"
|
||||
if (
|
||||
(Get-Item -LiteralPath $partial).Length -ne $ExpectedLength -or
|
||||
(Get-Sha256 $partial) -cne $ExpectedSha256
|
||||
) {
|
||||
throw "$Label length or SHA-256 changed"
|
||||
}
|
||||
Move-Item -LiteralPath $partial -Destination $Path
|
||||
}
|
||||
if (
|
||||
(Get-Item -LiteralPath $Path).Length -ne $ExpectedLength -or
|
||||
(Get-Sha256 $Path) -cne $ExpectedSha256
|
||||
) {
|
||||
throw "$Label length or SHA-256 changed"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Container([string]$Name) {
|
||||
$rows = @((& docker inspect $Name) | ConvertFrom-Json)
|
||||
Assert-LastExitCode "Docker inspection for $Name"
|
||||
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
|
||||
return $rows[0]
|
||||
}
|
||||
|
||||
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
||||
throw "M48T risk quality is pinned to DESKTOP-OPJ8J04"
|
||||
}
|
||||
|
||||
$release = Resolve-DDirectory $ReleaseRoot "M48T release root" $false
|
||||
$dataset = Resolve-DDirectory $DatasetRoot "M48T dataset root" $true
|
||||
$output = Resolve-DDirectory $OutputRoot "M48T output root" $true
|
||||
$runOutput = Join-Path $output $RunId
|
||||
if (Test-Path -LiteralPath $runOutput) { throw "M48T run output already exists" }
|
||||
$null = New-Item -ItemType Directory -Path $runOutput
|
||||
$runOutput = Resolve-DDirectory $runOutput "M48T run output" $false
|
||||
|
||||
$wheel = Assert-RegularFile (
|
||||
(Join-Path $release "nodedc_mission_core-0.1.0-py3-none-any.whl")
|
||||
) "M48T wheel"
|
||||
if ((Get-Sha256 $wheel) -cne $ExpectedWheelSha256) { throw "M48T wheel SHA-256 changed" }
|
||||
$profile = Assert-RegularFile (
|
||||
(Join-Path $release "m48t-risk-quality-temporal-v1.json")
|
||||
) "M48T profile"
|
||||
$runner = Assert-RegularFile (
|
||||
(Join-Path $release "run_m48t_coco_risk_quality_worker.py")
|
||||
) "M48T runner"
|
||||
$runnerSha256 = Get-Sha256 $runner
|
||||
|
||||
$imagesArchive = Join-Path $dataset "val2017.zip"
|
||||
$annotations = Join-Path $dataset "instances_val2017.json"
|
||||
# These mirrors rehost the unmodified official COCO assets and publish pinned SHA-256 digests.
|
||||
Ensure-PinnedMirrorFile $imagesArchive `
|
||||
"https://huggingface.co/datasets/pcuenq/coco-2017-mirror/resolve/main/val2017.zip?download=true" `
|
||||
815585330 `
|
||||
"4f7e2ccb2866ec5041993c9cf2a952bbed69647b115d0f74da7ce8f4bef82f05" `
|
||||
"COCO val2017"
|
||||
Ensure-PinnedMirrorFile $annotations `
|
||||
"https://huggingface.co/datasets/LibreYOLO/coco2017/resolve/main/instances_val2017.json?download=true" `
|
||||
19987840 `
|
||||
"e8c7f7908f1d7278341fae127d0da654f102f11bd7b21d8aeefa635b8c810b6f" `
|
||||
"COCO instances_val2017"
|
||||
$imagesArchive = Assert-RegularFile $imagesArchive "COCO images archive"
|
||||
$annotations = Assert-RegularFile $annotations "COCO val2017 instances"
|
||||
$imagesArchiveSha256 = Get-Sha256 $imagesArchive
|
||||
$annotationsDocumentSha256 = Get-Sha256 $annotations
|
||||
|
||||
$imagesRoot = Join-Path $dataset "val2017"
|
||||
if (-not (Test-Path -LiteralPath $imagesRoot -PathType Container)) {
|
||||
& tar.exe -xf $imagesArchive -C $dataset
|
||||
Assert-LastExitCode "COCO val2017 extraction"
|
||||
}
|
||||
$imagesRoot = Resolve-DDirectory $imagesRoot "COCO val2017 images" $false
|
||||
if (@(Get-ChildItem -LiteralPath $imagesRoot -File -Filter "*.jpg").Count -ne 5000) {
|
||||
throw "COCO val2017 image count changed"
|
||||
}
|
||||
|
||||
$experimentRoot = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z"
|
||||
) "M48T RF-DETR experiment root" $false
|
||||
$modelRoot = Resolve-DDirectory (
|
||||
(Join-Path $experimentRoot "triton-models")
|
||||
) "M48T RF-DETR model root" $false
|
||||
if ((Get-Sha256 (Assert-RegularFile (
|
||||
(Join-Path $modelRoot "rf_detr_large\1\model.plan")
|
||||
) "RF-DETR TensorRT engine")) -cne "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8") {
|
||||
throw "RF-DETR TensorRT engine SHA-256 changed"
|
||||
}
|
||||
|
||||
$image = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
& docker image inspect $image *> $null
|
||||
Assert-LastExitCode "Pinned M48T image inspection"
|
||||
$historicalTriton = Get-Container "ndc-mission-core-triton"
|
||||
if (-not $historicalTriton.State.Running -or $historicalTriton.State.Health.Status -cne "healthy") {
|
||||
throw "Historical Triton must remain healthy during M48T quality evaluation"
|
||||
}
|
||||
$historicalTritonId = [string]$historicalTriton.Id
|
||||
$tritonName = "ndc-mission-core-m48t-risk-quality-triton"
|
||||
$qualityName = "ndc-mission-core-m48t-risk-quality"
|
||||
foreach ($name in @($tritonName, $qualityName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
throw "M48T candidate container $name already exists"
|
||||
}
|
||||
}
|
||||
|
||||
$opencv = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages"
|
||||
) "OpenCV dependency" $false
|
||||
$pillow = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1"
|
||||
) "Pillow dependency" $false
|
||||
|
||||
try {
|
||||
& docker create `
|
||||
--name $tritonName `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--pids-limit 512 `
|
||||
--shm-size 1g `
|
||||
--gpus all `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
||||
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
|
||||
--health-interval 5s `
|
||||
--health-timeout 3s `
|
||||
--health-start-period 20s `
|
||||
--health-retries 24 `
|
||||
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
|
||||
$image `
|
||||
tritonserver `
|
||||
--model-repository=/models `
|
||||
--model-control-mode=explicit `
|
||||
--load-model=rf_detr_large `
|
||||
--disable-auto-complete-config `
|
||||
--strict-readiness=true `
|
||||
--exit-on-error=true `
|
||||
--allow-http=true `
|
||||
--allow-grpc=false `
|
||||
--allow-metrics=false *> $null
|
||||
Assert-LastExitCode "M48T Triton creation"
|
||||
& docker start $tritonName *> $null
|
||||
Assert-LastExitCode "M48T Triton start"
|
||||
$ready = $false
|
||||
foreach ($attempt in 1..60) {
|
||||
Start-Sleep -Seconds 2
|
||||
$candidate = Get-Container $tritonName
|
||||
if (-not $candidate.State.Running) { throw "M48T Triton stopped during startup" }
|
||||
if ($candidate.State.Health.Status -ceq "healthy") { $ready = $true; break }
|
||||
}
|
||||
if (-not $ready) { throw "M48T Triton did not become healthy" }
|
||||
|
||||
$maximumArguments = @()
|
||||
if ($MaximumImages -gt 0) {
|
||||
$maximumArguments = @("--maximum-images", ([string]$MaximumImages))
|
||||
}
|
||||
$arguments = @(
|
||||
"run", "--name", $qualityName,
|
||||
"--network", ("container:{0}" -f $tritonName),
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "256",
|
||||
"--gpus", "all",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", "PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:/opt/opencv:/opt/pillow",
|
||||
"-v", ((Convert-ToDockerPath $release) + ":/release:ro"),
|
||||
"-v", ((Convert-ToDockerPath $imagesRoot) + ":/dataset/val2017:ro"),
|
||||
"-v", ((Convert-ToDockerPath $annotations) + ":/dataset/instances_val2017.json:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runOutput) + ":/output:rw"),
|
||||
"-v", ((Convert-ToDockerPath $opencv) + ":/opt/opencv:ro"),
|
||||
"-v", ((Convert-ToDockerPath $pillow) + ":/opt/pillow:ro"),
|
||||
"--entrypoint", "python3",
|
||||
$image,
|
||||
"/release/run_m48t_coco_risk_quality_worker.py",
|
||||
"--profile", "/release/m48t-risk-quality-temporal-v1.json",
|
||||
"--annotations", "/dataset/instances_val2017.json",
|
||||
"--images-root", "/dataset/val2017",
|
||||
"--triton-origin", "http://127.0.0.1:8000",
|
||||
"--output", "/output/result.json",
|
||||
"--predictions", "/output/predictions.jsonl",
|
||||
"--failures", "/output/failures.jsonl",
|
||||
"--progress", "/output/progress.jsonl",
|
||||
"--review-root", "/output/review",
|
||||
"--runtime-artifact-sha256", $ExpectedWheelSha256,
|
||||
"--runner-sha256", $runnerSha256,
|
||||
"--images-archive-sha256", $imagesArchiveSha256,
|
||||
"--annotations-document-sha256", $annotationsDocumentSha256
|
||||
) + $maximumArguments
|
||||
& docker @arguments
|
||||
Assert-LastExitCode "M48T COCO risk quality evaluation"
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $runOutput "result.json") -PathType Leaf)) {
|
||||
throw "M48T result was not written"
|
||||
}
|
||||
} finally {
|
||||
foreach ($name in @($qualityName, $tritonName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
& docker rm -f $name *> $null
|
||||
}
|
||||
}
|
||||
$historicalAfter = Get-Container "ndc-mission-core-triton"
|
||||
if (
|
||||
$historicalAfter.Id -cne $historicalTritonId -or
|
||||
-not $historicalAfter.State.Running -or
|
||||
$historicalAfter.State.Health.Status -cne "healthy"
|
||||
) {
|
||||
throw "Historical Triton changed during M48T quality evaluation"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output ("M48T_RESULT={0}" -f (Join-Path $runOutput "result.json"))
|
||||
Write-Output ("COCO_IMAGES_SHA256={0}" -f $imagesArchiveSha256)
|
||||
Write-Output ("COCO_ANNOTATIONS_SHA256={0}" -f $annotationsDocumentSha256)
|
||||
Write-Output "HISTORICAL_TRITON_ACTION=none"
|
||||
Write-Output "PRODUCTION_ACCEPTED=false"
|
||||
@@ -0,0 +1,312 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ReleaseRoot,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||
[string]$ExpectedWheelSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
|
||||
[string]$RunId,
|
||||
[ValidateRange(0, 5000)]
|
||||
[int]$MaximumImages = 0,
|
||||
[string]$DatasetRoot = "D:\NDC_MISSIONCORE\datasets\coco-2017-val",
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48t-upstream-parity"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
|
||||
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
|
||||
$null = New-Item -ItemType Directory -Path $Path
|
||||
}
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if (
|
||||
-not $item.PSIsContainer -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
|
||||
) {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Assert-RegularFile([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
|
||||
throw "$Label must be a regular file"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") }
|
||||
|
||||
function Get-Container([string]$Name) {
|
||||
$rows = @((& docker inspect $Name) | ConvertFrom-Json)
|
||||
Assert-LastExitCode "Docker inspection for $Name"
|
||||
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
|
||||
return $rows[0]
|
||||
}
|
||||
|
||||
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
|
||||
throw "M48T upstream parity is pinned to DESKTOP-OPJ8J04"
|
||||
}
|
||||
|
||||
$release = Resolve-DDirectory $ReleaseRoot "M48T parity release root" $false
|
||||
$dataset = Resolve-DDirectory $DatasetRoot "M48T parity dataset root" $false
|
||||
$output = Resolve-DDirectory $OutputRoot "M48T parity output root" $true
|
||||
$runOutput = Join-Path $output $RunId
|
||||
if (Test-Path -LiteralPath $runOutput) { throw "M48T parity output already exists" }
|
||||
$null = New-Item -ItemType Directory -Path $runOutput
|
||||
$runOutput = Resolve-DDirectory $runOutput "M48T parity run output" $false
|
||||
|
||||
$wheel = Assert-RegularFile (Join-Path $release "nodedc_mission_core-0.1.0-py3-none-any.whl") "wheel"
|
||||
if ((Get-Sha256 $wheel) -cne $ExpectedWheelSha256) { throw "wheel SHA-256 changed" }
|
||||
$profile = Assert-RegularFile (Join-Path $release "m48t-upstream-parity-v1.json") "profile"
|
||||
$runner = Assert-RegularFile (Join-Path $release "run_m48t_upstream_parity_worker.py") "runner"
|
||||
$annotations = Assert-RegularFile (Join-Path $dataset "instances_val2017.json") "COCO annotations"
|
||||
$imagesRoot = Resolve-DDirectory (Join-Path $dataset "val2017") "COCO val2017 images" $false
|
||||
if (@(Get-ChildItem -LiteralPath $imagesRoot -File -Filter "*.jpg").Count -ne 5000) {
|
||||
throw "COCO val2017 image count changed"
|
||||
}
|
||||
|
||||
$experimentRoot = Resolve-DDirectory (
|
||||
"D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z"
|
||||
) "RF-DETR experiment root" $false
|
||||
$checkpoint = Assert-RegularFile (Join-Path $experimentRoot "weights\rf-detr-large-2026.pth") "checkpoint"
|
||||
if ((Get-Sha256 $checkpoint) -cne "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38") {
|
||||
throw "RF-DETR checkpoint SHA-256 changed"
|
||||
}
|
||||
$modelRoot = Resolve-DDirectory (Join-Path $experimentRoot "triton-models") "model root" $false
|
||||
$engine = Assert-RegularFile (Join-Path $modelRoot "rf_detr_large\1\model.plan") "TensorRT engine"
|
||||
if ((Get-Sha256 $engine) -cne "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8") {
|
||||
throw "RF-DETR TensorRT engine SHA-256 changed"
|
||||
}
|
||||
|
||||
$historical = Get-Container "ndc-mission-core-triton"
|
||||
if (-not $historical.State.Running -or $historical.State.Health.Status -cne "healthy") {
|
||||
throw "Historical Triton must remain healthy"
|
||||
}
|
||||
$historicalId = [string]$historical.Id
|
||||
$baseImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
& docker image inspect $baseImage *> $null
|
||||
Assert-LastExitCode "pinned base image inspection"
|
||||
$baseImageId = (& docker image inspect $baseImage --format "{{.Id}}").Trim()
|
||||
Assert-LastExitCode "pinned base image identity"
|
||||
$runtimeVolume = "ndc-mission-core-m48t-upstream-parity-env"
|
||||
if (-not (& docker volume ls --quiet --filter "name=^$runtimeVolume$")) {
|
||||
& docker volume create `
|
||||
--label "com.nodedc.product=mission-core" `
|
||||
--label "com.nodedc.stack=ndc-mission-core-compute" `
|
||||
--label "com.nodedc.role=bounded-rf-detr-upstream-parity" `
|
||||
--label "com.nodedc.managed-by=codex-bounded-experiment" `
|
||||
$runtimeVolume *> $null
|
||||
Assert-LastExitCode "M48T upstream parity dependency volume creation"
|
||||
}
|
||||
$torchReady = $false
|
||||
$strictErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
& docker run --rm `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=512m" `
|
||||
-e "PYTHONPATH=/opt/parity" `
|
||||
-v ($runtimeVolume + ":/opt/parity:ro") `
|
||||
--entrypoint python3 `
|
||||
$baseImage `
|
||||
-c "import importlib.metadata as m; assert m.version('numpy') == '1.26.4'; assert m.version('torch') == '2.9.1+cu130'; assert m.version('torchvision') == '0.24.1+cu130'" *> $null
|
||||
if ($LASTEXITCODE -eq 0) { $torchReady = $true }
|
||||
$ErrorActionPreference = $strictErrorActionPreference
|
||||
if (-not $torchReady) {
|
||||
& docker run --rm `
|
||||
--name "ndc-mission-core-m48t-upstream-parity-env-torch" `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--pids-limit 512 `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=8g" `
|
||||
-e "PIP_DISABLE_PIP_VERSION_CHECK=1" `
|
||||
-v ($runtimeVolume + ":/opt/parity:rw") `
|
||||
--entrypoint python3 `
|
||||
$baseImage `
|
||||
-m pip install --no-cache-dir --target /opt/parity `
|
||||
--index-url https://download.pytorch.org/whl/cu130 `
|
||||
"numpy==1.26.4" "torch==2.9.1+cu130" "torchvision==0.24.1+cu130"
|
||||
Assert-LastExitCode "M48T upstream parity PyTorch environment initialization"
|
||||
}
|
||||
$rfdetrReady = $false
|
||||
$ErrorActionPreference = "Continue"
|
||||
& docker run --rm `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=512m" `
|
||||
-e "PYTHONPATH=/opt/parity" `
|
||||
-v ($runtimeVolume + ":/opt/parity:ro") `
|
||||
--entrypoint python3 `
|
||||
$baseImage `
|
||||
-c "import importlib.metadata as m; import rfdetr; assert m.version('rfdetr') == '1.9.4'; assert m.version('pycocotools') == '2.0.10'; assert m.version('tritonclient') == '2.71.0'" *> $null
|
||||
if ($LASTEXITCODE -eq 0) { $rfdetrReady = $true }
|
||||
$ErrorActionPreference = $strictErrorActionPreference
|
||||
if (-not $rfdetrReady) {
|
||||
& docker run --rm `
|
||||
--name "ndc-mission-core-m48t-upstream-parity-env-dependencies" `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--pids-limit 512 `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=8g" `
|
||||
-e "PIP_DISABLE_PIP_VERSION_CHECK=1" `
|
||||
-e "PYTHONPATH=/opt/parity" `
|
||||
-v ($runtimeVolume + ":/opt/parity:rw") `
|
||||
--entrypoint python3 `
|
||||
$baseImage `
|
||||
-m pip install --no-cache-dir --upgrade --target /opt/parity `
|
||||
"numpy==1.26.4" "requests" "tqdm" "transformers>=5.1.0,<6.0.0" `
|
||||
"pydantic>=2.0,<3.0" "supervision>=0.29.0,<1.0" "pyDeprecate>=0.9,<0.10" `
|
||||
"pycocotools==2.0.10" "tritonclient[http]==2.71.0"
|
||||
Assert-LastExitCode "M48T upstream parity dependency initialization"
|
||||
& docker run --rm `
|
||||
--name "ndc-mission-core-m48t-upstream-parity-env-rfdetr" `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--pids-limit 512 `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
||||
-e "PIP_DISABLE_PIP_VERSION_CHECK=1" `
|
||||
-e "PYTHONPATH=/opt/parity" `
|
||||
-v ($runtimeVolume + ":/opt/parity:rw") `
|
||||
--entrypoint python3 `
|
||||
$baseImage `
|
||||
-m pip install --no-cache-dir --no-deps --upgrade --target /opt/parity "rfdetr==1.9.4"
|
||||
Assert-LastExitCode "M48T upstream parity RF-DETR package initialization"
|
||||
}
|
||||
& docker run --rm `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=512m" `
|
||||
-e "PYTHONPATH=/opt/parity" `
|
||||
-v ($runtimeVolume + ":/opt/parity:ro") `
|
||||
--entrypoint python3 `
|
||||
$baseImage `
|
||||
-c "import importlib.metadata as m; import rfdetr, torch, torchvision; assert m.version('numpy') == '1.26.4'; assert m.version('rfdetr') == '1.9.4'; assert m.version('torch') == '2.9.1+cu130'; assert m.version('torchvision') == '0.24.1+cu130'; assert m.version('pycocotools') == '2.0.10'; assert m.version('tritonclient') == '2.71.0'"
|
||||
Assert-LastExitCode "M48T upstream parity dependency volume verification"
|
||||
$runtimeIdentity = $baseImageId + "+volume:" + $runtimeVolume
|
||||
|
||||
$tritonName = "ndc-mission-core-m48t-upstream-parity-triton"
|
||||
$runnerName = "ndc-mission-core-m48t-upstream-parity"
|
||||
foreach ($name in @($tritonName, $runnerName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
throw "M48T parity container $name already exists"
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
& docker create `
|
||||
--name $tritonName `
|
||||
--read-only `
|
||||
--security-opt "no-new-privileges:true" `
|
||||
--cap-drop ALL `
|
||||
--pids-limit 512 `
|
||||
--shm-size 1g `
|
||||
--gpus all `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
|
||||
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
|
||||
--health-interval 5s `
|
||||
--health-timeout 3s `
|
||||
--health-start-period 20s `
|
||||
--health-retries 24 `
|
||||
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
|
||||
$baseImage `
|
||||
tritonserver `
|
||||
--model-repository=/models `
|
||||
--model-control-mode=explicit `
|
||||
--load-model=rf_detr_large `
|
||||
--disable-auto-complete-config `
|
||||
--strict-readiness=true `
|
||||
--exit-on-error=true `
|
||||
--allow-http=true `
|
||||
--allow-grpc=false `
|
||||
--allow-metrics=false *> $null
|
||||
Assert-LastExitCode "M48T parity Triton creation"
|
||||
& docker start $tritonName *> $null
|
||||
Assert-LastExitCode "M48T parity Triton start"
|
||||
$ready = $false
|
||||
foreach ($attempt in 1..60) {
|
||||
Start-Sleep -Seconds 2
|
||||
$candidate = Get-Container $tritonName
|
||||
if (-not $candidate.State.Running) { throw "M48T parity Triton stopped during startup" }
|
||||
if ($candidate.State.Health.Status -ceq "healthy") { $ready = $true; break }
|
||||
}
|
||||
if (-not $ready) { throw "M48T parity Triton did not become healthy" }
|
||||
|
||||
$maximumArguments = @()
|
||||
if ($MaximumImages -gt 0) { $maximumArguments = @("--maximum-images", ([string]$MaximumImages)) }
|
||||
$arguments = @(
|
||||
"run", "--name", $runnerName,
|
||||
"--network", ("container:{0}" -f $tritonName),
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "512",
|
||||
"--gpus", "all",
|
||||
"--shm-size", "2g",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=8g",
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", "PYTHONPATH=/opt/parity:/release/nodedc_mission_core-0.1.0-py3-none-any.whl",
|
||||
"-v", ($runtimeVolume + ":/opt/parity:ro"),
|
||||
"-v", ((Convert-ToDockerPath $release) + ":/release:ro"),
|
||||
"-v", ((Convert-ToDockerPath $imagesRoot) + ":/dataset/val2017:ro"),
|
||||
"-v", ((Convert-ToDockerPath $annotations) + ":/dataset/instances_val2017.json:ro"),
|
||||
"-v", ((Convert-ToDockerPath $checkpoint) + ":/model/rf-detr-large-2026.pth:ro"),
|
||||
"-v", ((Convert-ToDockerPath $runOutput) + ":/output:rw"),
|
||||
"--entrypoint", "python3",
|
||||
$baseImage,
|
||||
"/release/run_m48t_upstream_parity_worker.py",
|
||||
"--profile", "/release/m48t-upstream-parity-v1.json",
|
||||
"--annotations", "/dataset/instances_val2017.json",
|
||||
"--images-root", "/dataset/val2017",
|
||||
"--checkpoint", "/model/rf-detr-large-2026.pth",
|
||||
"--triton-origin", "http://127.0.0.1:8000",
|
||||
"--output", "/output/result.json",
|
||||
"--pytorch-predictions", "/output/pytorch-predictions.jsonl.gz",
|
||||
"--tensorrt-predictions", "/output/tensorrt-predictions.jsonl.gz",
|
||||
"--progress", "/output/progress.jsonl",
|
||||
"--runtime-image", $runtimeIdentity
|
||||
) + $maximumArguments
|
||||
& docker @arguments
|
||||
Assert-LastExitCode "M48T upstream parity evaluation"
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $runOutput "result.json") -PathType Leaf)) {
|
||||
throw "M48T upstream parity result was not written"
|
||||
}
|
||||
} finally {
|
||||
foreach ($name in @($runnerName, $tritonName)) {
|
||||
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
|
||||
& docker rm -f $name *> $null
|
||||
}
|
||||
}
|
||||
$historicalAfter = Get-Container "ndc-mission-core-triton"
|
||||
if (
|
||||
$historicalAfter.Id -cne $historicalId -or
|
||||
-not $historicalAfter.State.Running -or
|
||||
$historicalAfter.State.Health.Status -cne "healthy"
|
||||
) {
|
||||
throw "Historical Triton changed during M48T upstream parity"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output ("M48T_UPSTREAM_PARITY_RESULT={0}" -f (Join-Path $runOutput "result.json"))
|
||||
Write-Output ("RUNTIME_IDENTITY={0}" -f $runtimeIdentity)
|
||||
Write-Output "HISTORICAL_TRITON_ACTION=none"
|
||||
Write-Output "SEMANTIC_AUTHORITY_CHANGED=false"
|
||||
@@ -322,6 +322,7 @@ def canonical_laboratory_adapters() -> dict[str, LaboratoryAdapter]:
|
||||
"canonical.e46j-raw-fisheye-realtime/v1": _run_e46j,
|
||||
"experimental.e47-semantic-slam-shadow/v1": _run_e47,
|
||||
"experimental.m48s-fixed-class-detector/v1": _run_m48s_fixed_class_detector,
|
||||
"experimental.m48t-risk-quality-temporal/v1": _run_m48t_risk_quality_temporal,
|
||||
}
|
||||
|
||||
|
||||
@@ -342,6 +343,21 @@ def _run_m48s_fixed_class_detector(
|
||||
)
|
||||
|
||||
|
||||
def _run_m48t_risk_quality_temporal(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
from k1link.laboratory.m48t_risk_quality_lab import build_m48t_risk_quality_lab
|
||||
|
||||
result = build_m48t_risk_quality_lab(
|
||||
repository_root=request.inputs["repository_root"],
|
||||
output_root=request.output_root,
|
||||
)
|
||||
return LaboratoryAdapterResult(
|
||||
result_root=result.result_root,
|
||||
result_id=result.result_id,
|
||||
)
|
||||
|
||||
|
||||
def _run_m48_small_static_passage_regression(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
"""Seal the M4.8T semantic-quality and temporal-identity evidence as a LAB result."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.perception.fixed_class_detector_tournament import (
|
||||
canonical_json,
|
||||
false_authority,
|
||||
sha256_path,
|
||||
)
|
||||
|
||||
LAB_SCHEMA: Final = "missioncore.m48t-risk-quality-temporal-lab/v1"
|
||||
REPORT_SCHEMA: Final = "missioncore.m48t-risk-quality-temporal-report/v1"
|
||||
CATALOG_SCHEMA: Final = "missioncore.m48t-risk-quality-review-catalog/v1"
|
||||
RESULT_PREFIX: Final = "m48t-risk-quality-temporal-lab-"
|
||||
PROFILE_RELATIVE_PATH: Final = Path(
|
||||
"config/perception/m48t-risk-quality-temporal-v1.json"
|
||||
)
|
||||
WORKER_RELATIVE_ROOT: Final = Path(
|
||||
".runtime/worker-results/m48t-risk-quality-coco2017-val-full-v1"
|
||||
)
|
||||
TEMPORAL_LEDGER_RELATIVE_PATH: Final = Path(
|
||||
".runtime/m48s-reference-graph-shadow/full-replay-d85983f1/frames.jsonl"
|
||||
)
|
||||
|
||||
|
||||
class M48TRiskQualityLabError(RuntimeError):
|
||||
"""Raised when the M4.8T evidence cannot be sealed honestly."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M48TRiskQualityLabResult:
|
||||
result_root: Path
|
||||
result_id: str
|
||||
manifest: dict[str, Any]
|
||||
|
||||
|
||||
def build_m48t_risk_quality_lab(
|
||||
*,
|
||||
repository_root: Path,
|
||||
output_root: Path,
|
||||
) -> M48TRiskQualityLabResult:
|
||||
repository = repository_root.expanduser().resolve(strict=True)
|
||||
profile_path = repository / PROFILE_RELATIVE_PATH
|
||||
worker_root = repository / WORKER_RELATIVE_ROOT
|
||||
temporal_ledger_path = repository / TEMPORAL_LEDGER_RELATIVE_PATH
|
||||
quality_path = worker_root / "result.json"
|
||||
temporal_path = worker_root / "temporal-semantic-shadow.json"
|
||||
predictions_path = worker_root / "predictions.jsonl"
|
||||
failures_path = worker_root / "failures.jsonl"
|
||||
review_root = worker_root / "review"
|
||||
for path in (
|
||||
profile_path,
|
||||
quality_path,
|
||||
temporal_path,
|
||||
predictions_path,
|
||||
failures_path,
|
||||
temporal_ledger_path,
|
||||
):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise M48TRiskQualityLabError(f"required M4.8T evidence is missing: {path.name}")
|
||||
if review_root.is_symlink() or not review_root.is_dir():
|
||||
raise M48TRiskQualityLabError("M4.8T review evidence is missing")
|
||||
|
||||
profile = _read_object(profile_path)
|
||||
quality = _read_object(quality_path)
|
||||
temporal = _read_object(temporal_path)
|
||||
review_paths = sorted(review_root.glob("review-*.jpg"))
|
||||
_validate_inputs(
|
||||
profile=profile,
|
||||
profile_path=profile_path,
|
||||
quality=quality,
|
||||
predictions_path=predictions_path,
|
||||
failures_path=failures_path,
|
||||
temporal=temporal,
|
||||
temporal_ledger_path=temporal_ledger_path,
|
||||
review_paths=review_paths,
|
||||
)
|
||||
|
||||
method = _method(profile, quality)
|
||||
identity = {
|
||||
"schema_version": LAB_SCHEMA,
|
||||
"profile": {
|
||||
"profile_id": profile["profile_id"],
|
||||
"sha256": sha256_path(profile_path),
|
||||
},
|
||||
"source": {
|
||||
"quality_dataset_id": quality["dataset"]["dataset_id"],
|
||||
"quality_report_identity_sha256": quality["report_identity_sha256"],
|
||||
"quality_report_sha256": sha256_path(quality_path),
|
||||
"predictions_sha256": sha256_path(predictions_path),
|
||||
"failures_sha256": sha256_path(failures_path),
|
||||
"temporal_source_id": temporal["source"]["source_id"],
|
||||
"temporal_ledger_sha256": sha256_path(temporal_ledger_path),
|
||||
"temporal_shadow_sha256": sha256_path(temporal_path),
|
||||
},
|
||||
"method": method,
|
||||
"authority": false_authority(),
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = RESULT_PREFIX + identity_sha256
|
||||
completed_ns = quality["execution"]["completed_at_unix_ns"]
|
||||
created_at_utc = (
|
||||
datetime.fromtimestamp(completed_ns / 1_000_000_000, UTC)
|
||||
.isoformat(timespec="microseconds")
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
root = output_root.expanduser().absolute()
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
destination = root / result_id
|
||||
if destination.exists():
|
||||
manifest = _read_object(destination / "manifest.json")
|
||||
if (
|
||||
manifest.get("schema_version") != LAB_SCHEMA
|
||||
or manifest.get("identity_sha256") != identity_sha256
|
||||
or manifest.get("result_id") != result_id
|
||||
):
|
||||
raise M48TRiskQualityLabError("existing M4.8T LAB identity conflicts")
|
||||
return M48TRiskQualityLabResult(destination, result_id, manifest)
|
||||
|
||||
temporary = Path(tempfile.mkdtemp(prefix=".m48t-risk-quality-lab-", dir=root))
|
||||
try:
|
||||
(temporary / "review").mkdir(mode=0o700)
|
||||
shutil.copyfile(profile_path, temporary / "profile.json")
|
||||
shutil.copyfile(quality_path, temporary / "worker-quality-result.json")
|
||||
shutil.copyfile(predictions_path, temporary / "predictions.jsonl")
|
||||
shutil.copyfile(failures_path, temporary / "failures.jsonl")
|
||||
shutil.copyfile(temporal_path, temporary / "temporal-semantic-shadow.json")
|
||||
|
||||
review_items: list[dict[str, object]] = []
|
||||
for source in review_paths:
|
||||
destination_image = temporary / "review" / source.name
|
||||
shutil.copyfile(source, destination_image)
|
||||
case_id = source.stem.removeprefix("review-")
|
||||
review_items.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"image_id": int(case_id),
|
||||
"path": f"review/{source.name}",
|
||||
"media_type": "image/jpeg",
|
||||
"byte_length": destination_image.stat().st_size,
|
||||
"sha256": sha256_path(destination_image),
|
||||
}
|
||||
)
|
||||
catalog = {
|
||||
"schema_version": CATALOG_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"case_count": len(review_items),
|
||||
"legend": {
|
||||
"ground_truth": "green",
|
||||
"rf_detr_prediction": "yellow",
|
||||
},
|
||||
"cases": review_items,
|
||||
}
|
||||
catalog_path = temporary / "catalog.json"
|
||||
catalog_path.write_bytes(canonical_json(catalog) + b"\n")
|
||||
|
||||
decision = {
|
||||
"quality_evaluated": True,
|
||||
"quality_accepted": False,
|
||||
"failed_quality_gates": quality["quality_gates"]["failed"],
|
||||
"temporal_invariant_evaluated": True,
|
||||
"temporal_invariant_passed": True,
|
||||
"candidate_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
limitations = [
|
||||
"COCO val2017 is independent class truth, but it is not RAVNOVES00 domain truth.",
|
||||
"The temporal replay has no independent semantic or physical track-identity truth.",
|
||||
(
|
||||
"Worker timing is throughput evidence for this batch run, not a realtime "
|
||||
"admission gate."
|
||||
),
|
||||
(
|
||||
"Child/adult distinction, unknown moving objects and behavior risk policy "
|
||||
"were not evaluated."
|
||||
),
|
||||
"Semantic labels do not own geometry association, occupancy, navigation or actuation.",
|
||||
]
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"source": {
|
||||
"quality": quality["dataset"],
|
||||
"temporal": temporal["source"],
|
||||
},
|
||||
"configuration": {
|
||||
"candidate": quality["candidate"],
|
||||
"matching": profile["matching"],
|
||||
"quality_gates": profile["quality_gates"],
|
||||
"temporal": profile["temporal"],
|
||||
},
|
||||
"method": method,
|
||||
"execution": quality["execution"],
|
||||
"metrics": {
|
||||
"quality": quality["metrics"],
|
||||
"counts": quality["counts"],
|
||||
"failure_buckets": quality["failure_buckets"],
|
||||
"detector_rejections": quality["detector_rejections"],
|
||||
"temporal": temporal["metrics"],
|
||||
},
|
||||
"acceptance": {
|
||||
"quality": quality["quality_gates"],
|
||||
"temporal_invariant": temporal["temporal_invariant_gate"],
|
||||
},
|
||||
"decision": decision,
|
||||
"limitations": limitations,
|
||||
"authority": false_authority(),
|
||||
"visual_evidence": {
|
||||
"kind": "independent-coco-human-truth-review",
|
||||
"case_count": len(review_items),
|
||||
"catalog_schema_version": CATALOG_SCHEMA,
|
||||
"ground_truth_for_ravnoves00": False,
|
||||
},
|
||||
}
|
||||
report_path = temporary / "report.json"
|
||||
report_path.write_bytes(canonical_json(report) + b"\n")
|
||||
artifacts = _artifact_manifest(temporary)
|
||||
manifest = {
|
||||
"schema_version": LAB_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": created_at_utc,
|
||||
"status": "complete-quality-gate-failed-temporal-invariant-passed",
|
||||
"completed": True,
|
||||
"bounded_question_accepted": False,
|
||||
"ground_truth": False,
|
||||
"catalog": {
|
||||
"path": "catalog.json",
|
||||
"sha256": sha256_path(catalog_path),
|
||||
"byte_length": catalog_path.stat().st_size,
|
||||
},
|
||||
"method": method,
|
||||
"metrics": report["metrics"],
|
||||
"decision": decision,
|
||||
"limitations": limitations,
|
||||
"authority": false_authority(),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
(temporary / "manifest.json").write_bytes(canonical_json(manifest) + b"\n")
|
||||
temporary.replace(destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(temporary, ignore_errors=True)
|
||||
raise
|
||||
return M48TRiskQualityLabResult(destination, result_id, manifest)
|
||||
|
||||
|
||||
def _validate_inputs(
|
||||
*,
|
||||
profile: dict[str, Any],
|
||||
profile_path: Path,
|
||||
quality: dict[str, Any],
|
||||
predictions_path: Path,
|
||||
failures_path: Path,
|
||||
temporal: dict[str, Any],
|
||||
temporal_ledger_path: Path,
|
||||
review_paths: list[Path],
|
||||
) -> None:
|
||||
expected_failed = ["minimum-micro-precision", "minimum-family-recall:vehicle"]
|
||||
if (
|
||||
profile.get("schema_version")
|
||||
!= "missioncore.m48t-risk-quality-temporal-profile/v1"
|
||||
or quality.get("schema_version") != "missioncore.m48t-risk-quality-report/v1"
|
||||
or temporal.get("schema_version")
|
||||
!= "missioncore.m48t-temporal-semantic-shadow/v1"
|
||||
or quality.get("profile_sha256") != sha256_path(profile_path)
|
||||
or temporal.get("profile_sha256") != sha256_path(profile_path)
|
||||
or quality.get("profile_id") != profile.get("profile_id")
|
||||
or temporal.get("profile_id") != profile.get("profile_id")
|
||||
or quality.get("provenance", {}).get("predictions_sha256")
|
||||
!= sha256_path(predictions_path)
|
||||
or quality.get("provenance", {}).get("failures_sha256")
|
||||
!= sha256_path(failures_path)
|
||||
or temporal.get("source", {}).get("frame_ledger_sha256")
|
||||
!= sha256_path(temporal_ledger_path)
|
||||
or quality.get("dataset", {}).get("truth_instances") != 16060
|
||||
or quality.get("quality_gates", {}).get("passed") is not False
|
||||
or quality.get("quality_gates", {}).get("failed") != expected_failed
|
||||
or temporal.get("temporal_invariant_gate", {}).get("passed") is not True
|
||||
or temporal.get("temporal_invariant_gate", {}).get("semantic_quality_accepted")
|
||||
is not False
|
||||
or quality.get("authority", {}).get("candidate_accepted") is not False
|
||||
or len(review_paths) != 16
|
||||
or any(path.is_symlink() or not path.is_file() for path in review_paths)
|
||||
):
|
||||
raise M48TRiskQualityLabError("M4.8T evidence contract changed")
|
||||
|
||||
|
||||
def _method(profile: dict[str, Any], quality: dict[str, Any]) -> dict[str, object]:
|
||||
provenance = quality["provenance"]
|
||||
return {
|
||||
"schema_version": "missioncore.laboratory-method/v1",
|
||||
"completeness": "complete",
|
||||
"execution_class": "hybrid",
|
||||
"pipeline_id": "m48t-coco-quality-plus-bounded-temporal-identity/v1",
|
||||
"components": [
|
||||
{
|
||||
"kind": "source",
|
||||
"name": "COCO 2017 validation",
|
||||
"version": "val2017 independent human annotations",
|
||||
"role": "semantic quality truth",
|
||||
"identity_sha256": provenance["annotations_document_sha256"],
|
||||
},
|
||||
{
|
||||
"kind": "model",
|
||||
"name": "RF-DETR-L COCO",
|
||||
"version": quality["candidate"]["model_id"],
|
||||
"role": "fixed-class risk detector under evaluation",
|
||||
"identity_sha256": provenance["runtime_artifact_sha256"],
|
||||
},
|
||||
{
|
||||
"kind": "algorithm",
|
||||
"name": "COCO risk-family scorer",
|
||||
"version": profile["matching"]["method"],
|
||||
"role": "predeclared exact-class and family quality gates",
|
||||
"identity_sha256": provenance["runner_sha256"],
|
||||
},
|
||||
{
|
||||
"kind": "source",
|
||||
"name": "RAVNOVES00 reference-graph replay",
|
||||
"version": "4481-frame immutable publication ledger",
|
||||
"role": "temporal semantic anti-flicker shadow",
|
||||
"identity_sha256": (
|
||||
"badfa2f5f4f33fea7d5ad0e14fe5bbe38e1d490fc661d637789c354b8576d533"
|
||||
),
|
||||
},
|
||||
{
|
||||
"kind": "algorithm",
|
||||
"name": "bounded temporal semantic identity",
|
||||
"version": "history-5-confirm-2-switch-3/v1",
|
||||
"role": "stabilize advisory class on geometry-owned component IDs",
|
||||
"identity_sha256": hashlib.sha256(
|
||||
canonical_json(profile["temporal"])
|
||||
).hexdigest(),
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _artifact_manifest(root: Path) -> list[dict[str, object]]:
|
||||
artifacts: list[dict[str, object]] = []
|
||||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if relative == "manifest.json":
|
||||
continue
|
||||
media_type = "application/json"
|
||||
schema_version: str | None = None
|
||||
role = "supporting-evidence"
|
||||
if relative == "report.json":
|
||||
role = "laboratory-report"
|
||||
schema_version = REPORT_SCHEMA
|
||||
elif relative == "catalog.json":
|
||||
role = "visual-evidence-catalog"
|
||||
schema_version = CATALOG_SCHEMA
|
||||
elif relative == "profile.json":
|
||||
role = "predeclared-quality-temporal-profile"
|
||||
schema_version = "missioncore.m48t-risk-quality-temporal-profile/v1"
|
||||
elif relative == "worker-quality-result.json":
|
||||
role = "upstream-worker-quality-evidence"
|
||||
schema_version = "missioncore.m48t-risk-quality-report/v1"
|
||||
elif relative == "temporal-semantic-shadow.json":
|
||||
role = "upstream-temporal-shadow-evidence"
|
||||
schema_version = "missioncore.m48t-temporal-semantic-shadow/v1"
|
||||
elif relative.endswith(".jsonl"):
|
||||
media_type = "application/x-ndjson"
|
||||
role = "upstream-quality-ledger"
|
||||
elif relative.endswith(".jpg"):
|
||||
media_type = "image/jpeg"
|
||||
role = "visual-evidence-independent-truth-review"
|
||||
artifacts.append(
|
||||
{
|
||||
"role": role,
|
||||
"path": relative,
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": sha256_path(path),
|
||||
"media_type": media_type,
|
||||
"schema_version": schema_version,
|
||||
}
|
||||
)
|
||||
return artifacts
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text("utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise M48TRiskQualityLabError(f"invalid JSON evidence: {path.name}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise M48TRiskQualityLabError(f"JSON evidence must be an object: {path.name}")
|
||||
return value
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,434 @@
|
||||
"""Portable connector for the external DC Gaussian Pipeline service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
from urllib.parse import quote, unquote, urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
TUS_VERSION: Final = "1.0.0"
|
||||
BUILD_REQUEST_SCHEMA: Final = "gaussian-pipeline.build-request/v1"
|
||||
JOB_SCHEMA: Final = "gaussian-pipeline.job/v1"
|
||||
RESULT_SCHEMA: Final = "gaussian-pipeline.build-result/v1"
|
||||
CAPABILITIES_SCHEMA: Final = "gaussian-pipeline.capabilities/v1"
|
||||
SAFE_UPLOAD_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
SHA256_PATTERN: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
SOURCE_REVISION_PATTERN: Final = re.compile(r"^[a-f0-9]{40}$")
|
||||
IMAGE_DIGEST_PATTERN: Final = re.compile(r"^sha256:[a-f0-9]{64}$")
|
||||
DEFAULT_CHUNK_BYTES: Final = 8 * 1024 * 1024
|
||||
MAX_JSON_RESPONSE_BYTES: Final = 32 * 1024 * 1024
|
||||
MAX_RETRIES: Final = 3
|
||||
|
||||
|
||||
class GaussianPipelineGatewayError(RuntimeError):
|
||||
"""The configured Gaussian provider violated or rejected its connector contract."""
|
||||
|
||||
|
||||
class GaussianPipelineConfigurationError(GaussianPipelineGatewayError):
|
||||
"""The provider connector configuration is incomplete or unsafe."""
|
||||
|
||||
|
||||
class GaussianPipelineUnavailableError(GaussianPipelineGatewayError):
|
||||
"""The provider cannot currently be reached."""
|
||||
|
||||
|
||||
class GaussianPipelineIntegrityError(GaussianPipelineGatewayError):
|
||||
"""Transferred bytes do not match their immutable descriptor."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GaussianSourceUpload:
|
||||
upload_id: str
|
||||
filename: str
|
||||
format: str
|
||||
sha256: str
|
||||
byte_length: int
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"upload_id": self.upload_id,
|
||||
"filename": self.filename,
|
||||
"format": self.format,
|
||||
"sha256": self.sha256,
|
||||
"byte_length": self.byte_length,
|
||||
}
|
||||
|
||||
|
||||
class GaussianPipelineGateway:
|
||||
"""TUS and JSON client with bounded responses and independent digest checks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
token_file: Path,
|
||||
*,
|
||||
timeout_seconds: float = 30.0,
|
||||
chunk_bytes: int = DEFAULT_CHUNK_BYTES,
|
||||
transport: httpx.BaseTransport | None = None,
|
||||
) -> None:
|
||||
self.endpoint = _endpoint(endpoint)
|
||||
self.token_file = token_file.expanduser().absolute()
|
||||
self.token = _read_token(self.token_file)
|
||||
if timeout_seconds <= 0:
|
||||
raise GaussianPipelineConfigurationError("provider timeout must be positive")
|
||||
if chunk_bytes <= 0 or chunk_bytes > 64 * 1024 * 1024:
|
||||
raise GaussianPipelineConfigurationError("provider upload chunk size is invalid")
|
||||
self.chunk_bytes = chunk_bytes
|
||||
self._client = httpx.Client(
|
||||
base_url=self.endpoint,
|
||||
headers={"Authorization": f"Bearer {self.token}"},
|
||||
timeout=httpx.Timeout(timeout_seconds),
|
||||
transport=transport,
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> GaussianPipelineGateway:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: object) -> None:
|
||||
self.close()
|
||||
|
||||
def capabilities(self) -> dict[str, Any]:
|
||||
document = self._json("GET", "/v1/capabilities")
|
||||
if (
|
||||
document.get("schema_version") != CAPABILITIES_SCHEMA
|
||||
or document.get("service") != "ndc-gaussian-pipeline"
|
||||
or document.get("api_version") != "gaussian-pipeline.api/v1"
|
||||
):
|
||||
raise GaussianPipelineGatewayError("Gaussian provider capabilities do not match v1")
|
||||
_validate_runtime_provenance(document)
|
||||
return document
|
||||
|
||||
def upload_source(
|
||||
self,
|
||||
source_path: Path,
|
||||
*,
|
||||
filename: str,
|
||||
source_format: str,
|
||||
sha256: str,
|
||||
) -> GaussianSourceUpload:
|
||||
source_candidate = source_path.expanduser().absolute()
|
||||
if source_candidate.is_symlink() or not source_candidate.is_file():
|
||||
raise GaussianPipelineIntegrityError("Gaussian source must be one regular file")
|
||||
source = source_candidate.resolve()
|
||||
if source_format not in {"lcc", "lcc2"}:
|
||||
raise GaussianPipelineIntegrityError("Gaussian source format must be lcc or lcc2")
|
||||
if Path(filename).name != filename or not filename.lower().endswith(f".{source_format}"):
|
||||
raise GaussianPipelineIntegrityError("Gaussian source filename is unsafe")
|
||||
if SHA256_PATTERN.fullmatch(sha256) is None:
|
||||
raise GaussianPipelineIntegrityError("Gaussian source digest is invalid")
|
||||
byte_length = source.stat().st_size
|
||||
if byte_length <= 0:
|
||||
raise GaussianPipelineIntegrityError("Gaussian source is empty")
|
||||
if _sha256(source) != sha256:
|
||||
raise GaussianPipelineIntegrityError("Gaussian source digest does not match")
|
||||
|
||||
metadata = _tus_metadata(
|
||||
{"filename": filename, "format": source_format, "sha256": sha256}
|
||||
)
|
||||
try:
|
||||
response = self._client.post(
|
||||
"/v1/uploads",
|
||||
headers={
|
||||
"Tus-Resumable": TUS_VERSION,
|
||||
"Upload-Length": str(byte_length),
|
||||
"Upload-Metadata": metadata,
|
||||
},
|
||||
content=b"",
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPError as exc:
|
||||
raise _unavailable("Gaussian upload could not be created", exc) from exc
|
||||
location = response.headers.get("Location")
|
||||
if location is None:
|
||||
raise GaussianPipelineGatewayError("Gaussian upload response omitted Location")
|
||||
upload_url = urljoin(str(response.request.url), location)
|
||||
if not _is_provider_upload_url(self.endpoint, upload_url):
|
||||
raise GaussianPipelineGatewayError("Gaussian upload Location escaped the provider")
|
||||
upload_id = unquote(urlparse(upload_url).path.rsplit("/", 1)[-1])
|
||||
if SAFE_UPLOAD_ID.fullmatch(upload_id) is None:
|
||||
raise GaussianPipelineGatewayError("Gaussian upload id is invalid")
|
||||
self._send_file(source, upload_url, byte_length)
|
||||
return GaussianSourceUpload(
|
||||
upload_id=upload_id,
|
||||
filename=filename,
|
||||
format=source_format,
|
||||
sha256=sha256,
|
||||
byte_length=byte_length,
|
||||
)
|
||||
|
||||
def submit_build(self, document: Mapping[str, object]) -> dict[str, Any]:
|
||||
if document.get("schema_version") != BUILD_REQUEST_SCHEMA:
|
||||
raise GaussianPipelineGatewayError("Gaussian build request schema is invalid")
|
||||
result = self._json("POST", "/v1/jobs", document=document)
|
||||
if result.get("schema_version") != JOB_SCHEMA:
|
||||
raise GaussianPipelineGatewayError("Gaussian job response does not match v1")
|
||||
return result
|
||||
|
||||
def get_job(self, job_id: str) -> dict[str, Any]:
|
||||
_safe_id(job_id, "job id")
|
||||
result = self._json("GET", f"/v1/jobs/{quote(job_id, safe='')}")
|
||||
if result.get("schema_version") != JOB_SCHEMA or result.get("job_id") != job_id:
|
||||
raise GaussianPipelineGatewayError("Gaussian job identity does not match")
|
||||
return result
|
||||
|
||||
def get_result(self, job_id: str) -> dict[str, Any]:
|
||||
_safe_id(job_id, "job id")
|
||||
result = self._json("GET", f"/v1/jobs/{quote(job_id, safe='')}/result")
|
||||
if result.get("schema_version") != RESULT_SCHEMA or result.get("job_id") != job_id:
|
||||
raise GaussianPipelineGatewayError("Gaussian result identity does not match")
|
||||
_validate_runtime_provenance(result)
|
||||
return result
|
||||
|
||||
def download_artifact(
|
||||
self,
|
||||
job_id: str,
|
||||
descriptor: Mapping[str, object],
|
||||
destination: Path,
|
||||
) -> Path:
|
||||
_safe_id(job_id, "job id")
|
||||
logical_path = descriptor.get("logical_path")
|
||||
expected_sha256 = descriptor.get("sha256")
|
||||
expected_bytes = descriptor.get("byte_length")
|
||||
if (
|
||||
not isinstance(logical_path, str)
|
||||
or not logical_path
|
||||
or logical_path.startswith("/")
|
||||
or "\\" in logical_path
|
||||
or any(part in {"", ".", ".."} for part in logical_path.split("/"))
|
||||
):
|
||||
raise GaussianPipelineGatewayError("Gaussian artifact logical path is invalid")
|
||||
if (
|
||||
not isinstance(expected_sha256, str)
|
||||
or SHA256_PATTERN.fullmatch(expected_sha256) is None
|
||||
):
|
||||
raise GaussianPipelineGatewayError("Gaussian artifact digest is invalid")
|
||||
if (
|
||||
not isinstance(expected_bytes, int)
|
||||
or isinstance(expected_bytes, bool)
|
||||
or expected_bytes < 0
|
||||
):
|
||||
raise GaussianPipelineGatewayError("Gaussian artifact byte length is invalid")
|
||||
target = destination.expanduser().resolve()
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = target.with_name(f".{target.name}.partial-{os.getpid()}")
|
||||
digest = hashlib.sha256()
|
||||
byte_length = 0
|
||||
temporary_created = False
|
||||
try:
|
||||
with self._client.stream(
|
||||
"GET",
|
||||
f"/v1/jobs/{quote(job_id, safe='')}/artifacts/{quote(logical_path, safe='/')}",
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
with temporary.open("xb") as output:
|
||||
temporary_created = True
|
||||
for chunk in response.iter_bytes(chunk_size=1024 * 1024):
|
||||
output.write(chunk)
|
||||
digest.update(chunk)
|
||||
byte_length += len(chunk)
|
||||
if byte_length != expected_bytes or digest.hexdigest() != expected_sha256:
|
||||
raise GaussianPipelineIntegrityError(
|
||||
"Gaussian artifact bytes do not match the result manifest"
|
||||
)
|
||||
temporary.replace(target)
|
||||
return target
|
||||
except httpx.HTTPError as exc:
|
||||
raise _unavailable("Gaussian artifact download failed", exc) from exc
|
||||
finally:
|
||||
if temporary_created:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
def _send_file(self, source: Path, upload_url: str, byte_length: int) -> None:
|
||||
offset = self._upload_offset(upload_url, byte_length)
|
||||
retries = 0
|
||||
with source.open("rb") as input_file:
|
||||
while offset < byte_length:
|
||||
input_file.seek(offset)
|
||||
chunk = input_file.read(min(self.chunk_bytes, byte_length - offset))
|
||||
if not chunk:
|
||||
raise GaussianPipelineIntegrityError(
|
||||
"Gaussian source ended before upload length"
|
||||
)
|
||||
try:
|
||||
response = self._client.patch(
|
||||
upload_url,
|
||||
headers={
|
||||
"Tus-Resumable": TUS_VERSION,
|
||||
"Upload-Offset": str(offset),
|
||||
"Content-Type": "application/offset+octet-stream",
|
||||
},
|
||||
content=chunk,
|
||||
)
|
||||
response.raise_for_status()
|
||||
next_offset = _offset(response.headers.get("Upload-Offset"), byte_length)
|
||||
if next_offset != offset + len(chunk):
|
||||
raise GaussianPipelineGatewayError(
|
||||
"Gaussian upload returned a non-contiguous offset"
|
||||
)
|
||||
offset = next_offset
|
||||
retries = 0
|
||||
except httpx.TransportError as exc:
|
||||
retries += 1
|
||||
if retries > MAX_RETRIES:
|
||||
raise _unavailable(
|
||||
"Gaussian upload failed after resume attempts", exc
|
||||
) from exc
|
||||
offset = self._upload_offset(upload_url, byte_length)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code == 409 and retries < MAX_RETRIES:
|
||||
retries += 1
|
||||
offset = self._upload_offset(upload_url, byte_length)
|
||||
continue
|
||||
raise _unavailable("Gaussian upload was rejected", exc) from exc
|
||||
|
||||
def _upload_offset(self, upload_url: str, byte_length: int) -> int:
|
||||
try:
|
||||
response = self._client.head(
|
||||
upload_url,
|
||||
headers={"Tus-Resumable": TUS_VERSION},
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPError as exc:
|
||||
raise _unavailable("Gaussian upload offset is unavailable", exc) from exc
|
||||
return _offset(response.headers.get("Upload-Offset"), byte_length)
|
||||
|
||||
def _json(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
document: Mapping[str, object] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
response = self._client.request(method, path, json=document)
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPError as exc:
|
||||
raise _unavailable("Gaussian provider request failed", exc) from exc
|
||||
if len(response.content) > MAX_JSON_RESPONSE_BYTES:
|
||||
raise GaussianPipelineGatewayError("Gaussian provider JSON response is too large")
|
||||
try:
|
||||
value = response.json()
|
||||
except json.JSONDecodeError as exc:
|
||||
raise GaussianPipelineGatewayError("Gaussian provider response is not JSON") from exc
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise GaussianPipelineGatewayError("Gaussian provider response must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def configured_gaussian_pipeline_gateway() -> GaussianPipelineGateway | None:
|
||||
endpoint = os.environ.get("MISSIONCORE_GAUSSIAN_ENDPOINT")
|
||||
token_file = os.environ.get("MISSIONCORE_GAUSSIAN_TOKEN_FILE")
|
||||
if endpoint is None and token_file is None:
|
||||
return None
|
||||
if not endpoint or not token_file:
|
||||
raise GaussianPipelineConfigurationError(
|
||||
"MISSIONCORE_GAUSSIAN_ENDPOINT and MISSIONCORE_GAUSSIAN_TOKEN_FILE must be set together"
|
||||
)
|
||||
return GaussianPipelineGateway(endpoint, Path(token_file))
|
||||
|
||||
|
||||
def _endpoint(value: str) -> str:
|
||||
parsed = urlparse(value)
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"}
|
||||
or not parsed.hostname
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.path not in {"", "/"}
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise GaussianPipelineConfigurationError("Gaussian provider endpoint is invalid")
|
||||
return value.rstrip("/")
|
||||
|
||||
|
||||
def _is_provider_upload_url(endpoint: str, upload_url: str) -> bool:
|
||||
provider = urlparse(endpoint)
|
||||
upload = urlparse(upload_url)
|
||||
return (
|
||||
upload.scheme == provider.scheme
|
||||
and upload.hostname == provider.hostname
|
||||
and upload.port == provider.port
|
||||
and upload.username is None
|
||||
and upload.password is None
|
||||
and upload.path.startswith("/v1/uploads/")
|
||||
and not upload.params
|
||||
and not upload.query
|
||||
and not upload.fragment
|
||||
)
|
||||
|
||||
|
||||
def _read_token(token_file: Path) -> str:
|
||||
try:
|
||||
metadata = token_file.lstat()
|
||||
if token_file.is_symlink() or not token_file.is_file():
|
||||
raise GaussianPipelineConfigurationError("Gaussian token must be one regular file")
|
||||
token = token_file.read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
raise GaussianPipelineConfigurationError("Gaussian token file is unavailable") from exc
|
||||
if metadata.st_size > 4096 or not 32 <= len(token) <= 4096:
|
||||
raise GaussianPipelineConfigurationError("Gaussian token length is invalid")
|
||||
return token
|
||||
|
||||
|
||||
def _tus_metadata(values: Mapping[str, str]) -> str:
|
||||
return ",".join(
|
||||
f"{key} {base64.b64encode(value.encode('utf-8')).decode('ascii')}"
|
||||
for key, value in values.items()
|
||||
)
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _offset(value: str | None, byte_length: int) -> int:
|
||||
if value is None or not value.isdigit():
|
||||
raise GaussianPipelineGatewayError("Gaussian upload offset is invalid")
|
||||
offset = int(value)
|
||||
if not 0 <= offset <= byte_length:
|
||||
raise GaussianPipelineGatewayError("Gaussian upload offset is outside the source")
|
||||
return offset
|
||||
|
||||
|
||||
def _safe_id(value: str, label: str) -> None:
|
||||
if SAFE_UPLOAD_ID.fullmatch(value) is None:
|
||||
raise GaussianPipelineGatewayError(f"Gaussian {label} is invalid")
|
||||
|
||||
|
||||
def _validate_runtime_provenance(document: Mapping[str, object]) -> None:
|
||||
runtime = document.get("runtime")
|
||||
if not isinstance(runtime, dict):
|
||||
raise GaussianPipelineGatewayError("Gaussian runtime provenance is unavailable")
|
||||
source_revision = runtime.get("source_revision")
|
||||
image_digest = runtime.get("image_digest")
|
||||
if (
|
||||
not isinstance(source_revision, str)
|
||||
or SOURCE_REVISION_PATTERN.fullmatch(source_revision) is None
|
||||
or not isinstance(image_digest, str)
|
||||
or IMAGE_DIGEST_PATTERN.fullmatch(image_digest) is None
|
||||
):
|
||||
raise GaussianPipelineGatewayError("Gaussian runtime provenance is invalid")
|
||||
|
||||
|
||||
def _unavailable(message: str, error: httpx.HTTPError) -> GaussianPipelineGatewayError:
|
||||
if isinstance(error, httpx.TransportError):
|
||||
return GaussianPipelineUnavailableError(message)
|
||||
return GaussianPipelineGatewayError(message)
|
||||
@@ -128,6 +128,7 @@ from k1link.web.m48_object_quality_api import build_m48_object_quality_router
|
||||
from k1link.web.m48s_fixed_class_detector_lab_api import (
|
||||
build_m48s_fixed_class_detector_lab_router,
|
||||
)
|
||||
from k1link.web.m48t_risk_quality_lab_api import build_m48t_risk_quality_lab_router
|
||||
from k1link.web.map_api import (
|
||||
MapGatewayConfiguration,
|
||||
MapGatewayProxy,
|
||||
@@ -151,6 +152,7 @@ from k1link.web.runtime_readiness import (
|
||||
build_runtime_readiness,
|
||||
)
|
||||
from k1link.web.session_api import build_session_router
|
||||
from k1link.web.simulation_world_provider_api import build_simulation_world_provider_router
|
||||
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
||||
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
|
||||
|
||||
@@ -964,6 +966,17 @@ app.include_router(
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_m48t_risk_quality_lab_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m48t-risk-quality"
|
||||
/ "lab-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_e47_semantic_slam_router(
|
||||
root_provider=lambda: (
|
||||
@@ -1296,6 +1309,7 @@ app.include_router(
|
||||
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||
)
|
||||
)
|
||||
app.include_router(build_simulation_world_provider_router())
|
||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||
app.include_router(
|
||||
build_viewer_diagnostics_router(
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
"""Read-only API for the sealed M4.8T quality and temporal LAB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from k1link.laboratory.evidence_registry import LaboratoryEvidenceDefinition
|
||||
from k1link.laboratory.evidence_report import (
|
||||
LaboratoryEvidenceReportError,
|
||||
verify_laboratory_evidence_result,
|
||||
)
|
||||
from k1link.laboratory.m48t_risk_quality_lab import (
|
||||
CATALOG_SCHEMA,
|
||||
LAB_SCHEMA,
|
||||
REPORT_SCHEMA,
|
||||
RESULT_PREFIX,
|
||||
)
|
||||
from k1link.perception.fixed_class_detector_tournament import false_authority
|
||||
|
||||
RootProvider = Callable[[], Path | None]
|
||||
RESULT_ID: Final = re.compile(rf"^{re.escape(RESULT_PREFIX)}[a-f0-9]{{64}}$")
|
||||
CASE_ID: Final = re.compile(r"^[0-9]{12}$")
|
||||
VIEW_SCHEMA: Final = "missioncore.m48t-risk-quality-temporal-view/v1"
|
||||
CATALOG_VIEW_SCHEMA: Final = "missioncore.m48t-risk-quality-temporal-catalog/v1"
|
||||
_DEFINITION: Final = LaboratoryEvidenceDefinition(
|
||||
work_id="m48t-risk-quality-temporal",
|
||||
runtime_relative_root=PurePosixPath("m48t-risk-quality/lab-results"),
|
||||
result_id_prefix="m48t-risk-quality-temporal-lab",
|
||||
document_name="manifest.json",
|
||||
result_schema_version=LAB_SCHEMA,
|
||||
)
|
||||
|
||||
|
||||
def build_m48t_risk_quality_lab_router(
|
||||
*, root_provider: RootProvider = lambda: None
|
||||
) -> APIRouter:
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/laboratory/m48t/risk-quality",
|
||||
tags=["laboratory"],
|
||||
)
|
||||
|
||||
@router.get("/results")
|
||||
def list_results(limit: int = Query(default=1, ge=1, le=10)) -> dict[str, object]:
|
||||
root = _configured_root(root_provider)
|
||||
if root is None:
|
||||
return _empty_catalog(False)
|
||||
candidates = _candidates(root)
|
||||
items: list[dict[str, object]] = []
|
||||
invalid_total = 0
|
||||
for candidate in candidates:
|
||||
try:
|
||||
items.append(_project_result(candidate))
|
||||
except RuntimeError:
|
||||
invalid_total += 1
|
||||
items.sort(
|
||||
key=lambda item: (str(item["created_at_utc"]), str(item["result_id"])),
|
||||
reverse=True,
|
||||
)
|
||||
return {
|
||||
"schema_version": CATALOG_VIEW_SCHEMA,
|
||||
"configured": True,
|
||||
"items": items[:limit],
|
||||
"candidate_total": len(candidates),
|
||||
"invalid_total": invalid_total,
|
||||
"access": "read-only",
|
||||
}
|
||||
|
||||
@router.get("/results/{result_id}")
|
||||
def get_result(result_id: str) -> dict[str, object]:
|
||||
try:
|
||||
return _project_result(_resolve_candidate(root_provider, result_id))
|
||||
except RuntimeError:
|
||||
raise HTTPException(status_code=404, detail="M4.8T result not found") from None
|
||||
|
||||
@router.get("/results/{result_id}/review/{case_id}.jpg")
|
||||
def get_review_image(result_id: str, case_id: str) -> FileResponse:
|
||||
if CASE_ID.fullmatch(case_id) is None:
|
||||
raise HTTPException(status_code=404, detail="M4.8T review case not found")
|
||||
try:
|
||||
loaded = _load_result(_resolve_candidate(root_provider, result_id))
|
||||
except RuntimeError:
|
||||
raise HTTPException(status_code=404, detail="M4.8T result not found") from None
|
||||
descriptor = next(
|
||||
(
|
||||
item
|
||||
for item in loaded["catalog"]["cases"]
|
||||
if isinstance(item, dict) and item.get("case_id") == case_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not isinstance(descriptor, dict):
|
||||
raise HTTPException(status_code=404, detail="M4.8T review case not found")
|
||||
candidate = loaded["root"]
|
||||
path = (candidate / str(descriptor["path"])).resolve()
|
||||
if (
|
||||
not path.is_relative_to(candidate)
|
||||
or path.is_symlink()
|
||||
or not path.is_file()
|
||||
or descriptor.get("byte_length") != path.stat().st_size
|
||||
or descriptor.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise HTTPException(status_code=404, detail="M4.8T review case not found")
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type="image/jpeg",
|
||||
headers={
|
||||
"Cache-Control": "public, max-age=31536000, immutable",
|
||||
"ETag": f'"{descriptor["sha256"]}"',
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _project_result(candidate: Path) -> dict[str, object]:
|
||||
loaded = _load_result(candidate)
|
||||
manifest = loaded["manifest"]
|
||||
report = loaded["report"]
|
||||
catalog = loaded["catalog"]
|
||||
result_id = str(manifest["result_id"])
|
||||
review_cases = [
|
||||
{
|
||||
"case_id": item["case_id"],
|
||||
"image_id": item["image_id"],
|
||||
"image_url": (
|
||||
f"/api/v1/laboratory/m48t/risk-quality/results/{result_id}"
|
||||
f"/review/{item['case_id']}.jpg"
|
||||
),
|
||||
"media_type": item["media_type"],
|
||||
"byte_length": item["byte_length"],
|
||||
"sha256": item["sha256"],
|
||||
}
|
||||
for item in catalog["cases"]
|
||||
]
|
||||
return {
|
||||
"schema_version": VIEW_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"created_at_utc": manifest["created_at_utc"],
|
||||
"status": manifest["status"],
|
||||
"source": copy.deepcopy(report["source"]),
|
||||
"configuration": copy.deepcopy(report["configuration"]),
|
||||
"method": copy.deepcopy(report["method"]),
|
||||
"execution": copy.deepcopy(report["execution"]),
|
||||
"metrics": copy.deepcopy(report["metrics"]),
|
||||
"acceptance": copy.deepcopy(report["acceptance"]),
|
||||
"decision": copy.deepcopy(report["decision"]),
|
||||
"limitations": copy.deepcopy(report["limitations"]),
|
||||
"review": {
|
||||
"legend": copy.deepcopy(catalog["legend"]),
|
||||
"cases": review_cases,
|
||||
},
|
||||
"ground_truth": False,
|
||||
"authority": copy.deepcopy(report["authority"]),
|
||||
"access": "read-only",
|
||||
}
|
||||
|
||||
|
||||
def _load_result(candidate: Path) -> dict[str, Any]:
|
||||
if (
|
||||
not candidate.is_dir()
|
||||
or candidate.is_symlink()
|
||||
or RESULT_ID.fullmatch(candidate.name) is None
|
||||
):
|
||||
raise RuntimeError("M4.8T result candidate is invalid")
|
||||
try:
|
||||
verify_laboratory_evidence_result(_DEFINITION, candidate)
|
||||
manifest = _read_object(candidate / "manifest.json")
|
||||
report = _read_object(candidate / "report.json")
|
||||
catalog = _read_object(candidate / "catalog.json")
|
||||
except (LaboratoryEvidenceReportError, OSError, ValueError) as exc:
|
||||
raise RuntimeError("M4.8T result integrity failed") from exc
|
||||
identity = manifest.get("identity")
|
||||
if (
|
||||
manifest.get("schema_version") != LAB_SCHEMA
|
||||
or manifest.get("result_id") != candidate.name
|
||||
or manifest.get("status")
|
||||
!= "complete-quality-gate-failed-temporal-invariant-passed"
|
||||
or manifest.get("completed") is not True
|
||||
or manifest.get("bounded_question_accepted") is not False
|
||||
or manifest.get("ground_truth") is not False
|
||||
or not isinstance(manifest.get("created_at_utc"), str)
|
||||
or not isinstance(identity, dict)
|
||||
or manifest.get("identity_sha256") != _canonical_sha256(identity)
|
||||
or not candidate.name.endswith(str(manifest.get("identity_sha256")))
|
||||
or identity.get("authority") != false_authority()
|
||||
or manifest.get("authority") != false_authority()
|
||||
or report.get("schema_version") != REPORT_SCHEMA
|
||||
or report.get("result_id") != candidate.name
|
||||
or report.get("authority") != false_authority()
|
||||
or report.get("decision", {}).get("quality_accepted") is not False
|
||||
or report.get("decision", {}).get("temporal_invariant_passed") is not True
|
||||
or catalog.get("schema_version") != CATALOG_SCHEMA
|
||||
or catalog.get("result_id") != candidate.name
|
||||
or catalog.get("case_count") != 16
|
||||
or not isinstance(catalog.get("cases"), list)
|
||||
or len(catalog["cases"]) != 16
|
||||
or any(not _valid_case(item) for item in catalog["cases"])
|
||||
):
|
||||
raise RuntimeError("M4.8T result contract changed")
|
||||
return {"root": candidate, "manifest": manifest, "report": report, "catalog": catalog}
|
||||
|
||||
|
||||
def _valid_case(value: object) -> bool:
|
||||
return (
|
||||
isinstance(value, dict)
|
||||
and isinstance(value.get("case_id"), str)
|
||||
and CASE_ID.fullmatch(value["case_id"]) is not None
|
||||
and value.get("image_id") == int(value["case_id"])
|
||||
and value.get("path") == f"review/review-{value['case_id']}.jpg"
|
||||
and value.get("media_type") == "image/jpeg"
|
||||
and isinstance(value.get("byte_length"), int)
|
||||
and value["byte_length"] > 0
|
||||
and isinstance(value.get("sha256"), str)
|
||||
and re.fullmatch(r"[a-f0-9]{64}", value["sha256"]) is not None
|
||||
)
|
||||
|
||||
|
||||
def _resolve_candidate(provider: RootProvider, result_id: str) -> Path:
|
||||
if RESULT_ID.fullmatch(result_id) is None:
|
||||
raise HTTPException(status_code=404, detail="M4.8T result not found")
|
||||
root = _configured_root(provider)
|
||||
if root is None:
|
||||
raise HTTPException(status_code=404, detail="M4.8T result not found")
|
||||
candidate = (root / result_id).resolve()
|
||||
if candidate.parent != root or candidate.is_symlink() or not candidate.is_dir():
|
||||
raise HTTPException(status_code=404, detail="M4.8T result not found")
|
||||
return candidate
|
||||
|
||||
|
||||
def _configured_root(provider: RootProvider) -> Path | None:
|
||||
value = provider()
|
||||
if value is None:
|
||||
return None
|
||||
candidate = value.expanduser().absolute()
|
||||
if candidate.is_symlink():
|
||||
return None
|
||||
try:
|
||||
root = candidate.resolve(strict=True)
|
||||
except OSError:
|
||||
return None
|
||||
return root if root.is_dir() else None
|
||||
|
||||
|
||||
def _candidates(root: Path) -> list[Path]:
|
||||
return sorted(
|
||||
(
|
||||
item
|
||||
for item in root.iterdir()
|
||||
if item.is_dir() and not item.is_symlink() and RESULT_ID.fullmatch(item.name)
|
||||
),
|
||||
key=lambda item: item.stat().st_mtime_ns,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def _empty_catalog(configured: bool) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": CATALOG_VIEW_SCHEMA,
|
||||
"configured": configured,
|
||||
"items": [],
|
||||
"candidate_total": 0,
|
||||
"invalid_total": 0,
|
||||
"access": "read-only",
|
||||
}
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text("utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("JSON evidence must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _canonical_sha256(value: object) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Read-only Mission Core projection of the external Gaussian build provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from k1link.simulation.gaussian_pipeline_gateway import (
|
||||
GaussianPipelineGateway,
|
||||
GaussianPipelineGatewayError,
|
||||
configured_gaussian_pipeline_gateway,
|
||||
)
|
||||
|
||||
ProviderFactory = Callable[[], GaussianPipelineGateway | None]
|
||||
|
||||
|
||||
class SimulationWorldProviderStatus(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
schema_version: Literal["missioncore.simulation-world-provider-status/v1"] = (
|
||||
"missioncore.simulation-world-provider-status/v1"
|
||||
)
|
||||
provider_id: Literal["gaussian-pipeline"] = "gaussian-pipeline"
|
||||
configured: bool
|
||||
state: Literal["ready", "unavailable"]
|
||||
capabilities: dict[str, Any] | None
|
||||
|
||||
|
||||
def build_simulation_world_provider_router(
|
||||
provider_factory: ProviderFactory = configured_gaussian_pipeline_gateway,
|
||||
) -> APIRouter:
|
||||
router = APIRouter()
|
||||
|
||||
@router.get(
|
||||
"/api/v1/simulation-worlds/provider",
|
||||
response_model=SimulationWorldProviderStatus,
|
||||
)
|
||||
def provider_status() -> SimulationWorldProviderStatus:
|
||||
provider: GaussianPipelineGateway | None = None
|
||||
try:
|
||||
provider = provider_factory()
|
||||
if provider is None:
|
||||
return SimulationWorldProviderStatus(
|
||||
configured=False,
|
||||
state="unavailable",
|
||||
capabilities=None,
|
||||
)
|
||||
return SimulationWorldProviderStatus(
|
||||
configured=True,
|
||||
state="ready",
|
||||
capabilities=provider.capabilities(),
|
||||
)
|
||||
except GaussianPipelineGatewayError:
|
||||
return SimulationWorldProviderStatus(
|
||||
configured=True,
|
||||
state="unavailable",
|
||||
capabilities=None,
|
||||
)
|
||||
finally:
|
||||
if provider is not None:
|
||||
provider.close()
|
||||
|
||||
return router
|
||||
@@ -0,0 +1,237 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from k1link.simulation.gaussian_pipeline_gateway import (
|
||||
BUILD_REQUEST_SCHEMA,
|
||||
GaussianPipelineGateway,
|
||||
GaussianPipelineGatewayError,
|
||||
GaussianPipelineIntegrityError,
|
||||
)
|
||||
|
||||
|
||||
def _token_file(tmp_path: Path) -> Path:
|
||||
token = tmp_path / "gaussian.token"
|
||||
token.write_text("t" * 64, encoding="utf-8")
|
||||
return token
|
||||
|
||||
|
||||
def test_gateway_uploads_with_tus_and_reads_provider_contract(tmp_path: Path) -> None:
|
||||
uploaded = bytearray()
|
||||
metadata: dict[str, str] = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.headers["authorization"] == f"Bearer {'t' * 64}"
|
||||
if request.method == "GET" and request.url.path == "/v1/capabilities":
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"schema_version": "gaussian-pipeline.capabilities/v1",
|
||||
"service": "ndc-gaussian-pipeline",
|
||||
"api_version": "gaussian-pipeline.api/v1",
|
||||
"provider": {"id": "playcanvas-splat-transform", "version": "3.3.3"},
|
||||
"runtime": {
|
||||
"source_revision": "a" * 40,
|
||||
"image_digest": f"sha256:{'b' * 64}",
|
||||
},
|
||||
},
|
||||
)
|
||||
if request.method == "POST" and request.url.path == "/v1/uploads":
|
||||
for item in request.headers["upload-metadata"].split(","):
|
||||
key, encoded = item.split(" ", 1)
|
||||
metadata[key] = base64.b64decode(encoded).decode("utf-8")
|
||||
return httpx.Response(201, headers={"Location": "/v1/uploads/upload-001"})
|
||||
if request.method == "HEAD" and request.url.path == "/v1/uploads/upload-001":
|
||||
return httpx.Response(200, headers={"Upload-Offset": str(len(uploaded))})
|
||||
if request.method == "PATCH" and request.url.path == "/v1/uploads/upload-001":
|
||||
assert int(request.headers["upload-offset"]) == len(uploaded)
|
||||
uploaded.extend(request.content)
|
||||
return httpx.Response(204, headers={"Upload-Offset": str(len(uploaded))})
|
||||
return httpx.Response(404)
|
||||
|
||||
source = tmp_path / "yard.lcc2"
|
||||
source.write_bytes(b"portable-gaussian-source")
|
||||
digest = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
with GaussianPipelineGateway(
|
||||
"http://gaussian.test",
|
||||
_token_file(tmp_path),
|
||||
chunk_bytes=5,
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as gateway:
|
||||
assert gateway.capabilities()["service"] == "ndc-gaussian-pipeline"
|
||||
descriptor = gateway.upload_source(
|
||||
source,
|
||||
filename="yard.lcc2",
|
||||
source_format="lcc2",
|
||||
sha256=digest,
|
||||
)
|
||||
|
||||
assert bytes(uploaded) == source.read_bytes()
|
||||
assert metadata == {"filename": "yard.lcc2", "format": "lcc2", "sha256": digest}
|
||||
assert descriptor.to_dict() == {
|
||||
"upload_id": "upload-001",
|
||||
"filename": "yard.lcc2",
|
||||
"format": "lcc2",
|
||||
"sha256": digest,
|
||||
"byte_length": len(uploaded),
|
||||
}
|
||||
|
||||
|
||||
def test_gateway_rejects_local_source_digest_mismatch(tmp_path: Path) -> None:
|
||||
source = tmp_path / "yard.lcc"
|
||||
source.write_bytes(b"source")
|
||||
with (
|
||||
GaussianPipelineGateway(
|
||||
"http://gaussian.test",
|
||||
_token_file(tmp_path),
|
||||
transport=httpx.MockTransport(lambda _request: httpx.Response(500)),
|
||||
) as gateway,
|
||||
pytest.raises(GaussianPipelineIntegrityError, match="digest does not match"),
|
||||
):
|
||||
gateway.upload_source(
|
||||
source,
|
||||
filename="yard.lcc",
|
||||
source_format="lcc",
|
||||
sha256="a" * 64,
|
||||
)
|
||||
|
||||
|
||||
def test_gateway_rejects_upload_location_outside_provider(tmp_path: Path) -> None:
|
||||
source = tmp_path / "yard.lcc"
|
||||
source.write_bytes(b"source")
|
||||
digest = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
|
||||
def handler(_request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
201,
|
||||
headers={"Location": "https://attacker.invalid/v1/uploads/stolen"},
|
||||
)
|
||||
|
||||
with (
|
||||
GaussianPipelineGateway(
|
||||
"http://gaussian.test",
|
||||
_token_file(tmp_path),
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as gateway,
|
||||
pytest.raises(GaussianPipelineGatewayError, match="escaped the provider"),
|
||||
):
|
||||
gateway.upload_source(
|
||||
source,
|
||||
filename="yard.lcc",
|
||||
source_format="lcc",
|
||||
sha256=digest,
|
||||
)
|
||||
|
||||
|
||||
def test_gateway_rejects_symlink_source_and_token(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source.lcc"
|
||||
source.write_bytes(b"source")
|
||||
source_link = tmp_path / "source-link.lcc"
|
||||
source_link.symlink_to(source)
|
||||
token = _token_file(tmp_path)
|
||||
token_link = tmp_path / "token-link"
|
||||
token_link.symlink_to(token)
|
||||
|
||||
with pytest.raises(GaussianPipelineGatewayError, match="token must be one regular file"):
|
||||
GaussianPipelineGateway("http://gaussian.test", token_link)
|
||||
|
||||
digest = hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
with (
|
||||
GaussianPipelineGateway(
|
||||
"http://gaussian.test",
|
||||
token,
|
||||
transport=httpx.MockTransport(lambda _request: httpx.Response(500)),
|
||||
) as gateway,
|
||||
pytest.raises(GaussianPipelineIntegrityError, match="one regular file"),
|
||||
):
|
||||
gateway.upload_source(
|
||||
source_link,
|
||||
filename="source.lcc",
|
||||
source_format="lcc",
|
||||
sha256=digest,
|
||||
)
|
||||
|
||||
|
||||
def test_gateway_submits_tracks_and_imports_digest_bound_artifact(tmp_path: Path) -> None:
|
||||
artifact = b"preview-sog"
|
||||
artifact_sha = hashlib.sha256(artifact).hexdigest()
|
||||
job_id = "gsp-20260825200000-deadbeef"
|
||||
request_document: dict[str, object] = {
|
||||
"schema_version": BUILD_REQUEST_SCHEMA,
|
||||
"idempotency_key": "missioncore-build-01",
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.method == "POST" and request.url.path == "/v1/jobs":
|
||||
assert json.loads(request.content) == request_document
|
||||
return httpx.Response(
|
||||
202,
|
||||
json={"schema_version": "gaussian-pipeline.job/v1", "job_id": job_id},
|
||||
)
|
||||
if request.method == "GET" and request.url.path == f"/v1/jobs/{job_id}":
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"schema_version": "gaussian-pipeline.job/v1", "job_id": job_id},
|
||||
)
|
||||
if request.method == "GET" and request.url.path == f"/v1/jobs/{job_id}/result":
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"schema_version": "gaussian-pipeline.build-result/v1",
|
||||
"job_id": job_id,
|
||||
"runtime": {
|
||||
"source_revision": "a" * 40,
|
||||
"image_digest": f"sha256:{'b' * 64}",
|
||||
},
|
||||
},
|
||||
)
|
||||
if request.method == "GET" and request.url.path.endswith("/artifacts/preview.sog"):
|
||||
return httpx.Response(200, content=artifact)
|
||||
return httpx.Response(404)
|
||||
|
||||
with GaussianPipelineGateway(
|
||||
"http://gaussian.test",
|
||||
_token_file(tmp_path),
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as gateway:
|
||||
assert gateway.submit_build(request_document)["job_id"] == job_id
|
||||
assert gateway.get_job(job_id)["job_id"] == job_id
|
||||
assert gateway.get_result(job_id)["job_id"] == job_id
|
||||
destination = gateway.download_artifact(
|
||||
job_id,
|
||||
{
|
||||
"logical_path": "preview.sog",
|
||||
"sha256": artifact_sha,
|
||||
"byte_length": len(artifact),
|
||||
},
|
||||
tmp_path / "import" / "preview.sog",
|
||||
)
|
||||
assert destination.read_bytes() == artifact
|
||||
|
||||
|
||||
def test_gateway_rejects_capabilities_without_runtime_provenance(tmp_path: Path) -> None:
|
||||
def handler(_request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"schema_version": "gaussian-pipeline.capabilities/v1",
|
||||
"service": "ndc-gaussian-pipeline",
|
||||
"api_version": "gaussian-pipeline.api/v1",
|
||||
},
|
||||
)
|
||||
|
||||
with (
|
||||
GaussianPipelineGateway(
|
||||
"http://gaussian.test",
|
||||
_token_file(tmp_path),
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as gateway,
|
||||
pytest.raises(GaussianPipelineGatewayError, match="provenance is unavailable"),
|
||||
):
|
||||
gateway.capabilities()
|
||||
@@ -127,7 +127,7 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None:
|
||||
repository_root / "config" / "laboratories"
|
||||
)
|
||||
|
||||
assert len(registry.definitions) == 37
|
||||
assert len(registry.definitions) == 38
|
||||
assert {item.work_id for item in registry.definitions} >= {
|
||||
"e31-source-binding",
|
||||
"e46j-raw-fisheye-realtime",
|
||||
@@ -142,6 +142,7 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None:
|
||||
"m48-object-centric-quality",
|
||||
"m48-small-static-passage-regression",
|
||||
"m48s-fixed-class-detector",
|
||||
"m48t-risk-quality-temporal",
|
||||
}
|
||||
m48 = next(
|
||||
item for item in registry.definitions
|
||||
|
||||
@@ -99,6 +99,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
"e46j-raw-fisheye-realtime",
|
||||
"e47-semantic-slam-shadow",
|
||||
"m48s-fixed-class-detector",
|
||||
"m48t-risk-quality-temporal",
|
||||
}
|
||||
by_work_id = {row.work_id: row for row in execution.definitions}
|
||||
assert by_work_id["m48-small-static-passage-regression"].evidence_contract == (
|
||||
@@ -111,10 +112,17 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
assert by_work_id["e47-semantic-slam-shadow"].isolation == "bounded-adapter"
|
||||
assert by_work_id["m48s-fixed-class-detector"].lifecycle == "experimental"
|
||||
assert by_work_id["m48s-fixed-class-detector"].isolation == "bounded-adapter"
|
||||
assert by_work_id["m48t-risk-quality-temporal"].lifecycle == "experimental"
|
||||
assert by_work_id["m48t-risk-quality-temporal"].isolation == "bounded-adapter"
|
||||
assert all(
|
||||
row.lifecycle == "canonical"
|
||||
for row in execution.definitions
|
||||
if row.work_id not in {"e47-semantic-slam-shadow", "m48s-fixed-class-detector"}
|
||||
if row.work_id
|
||||
not in {
|
||||
"e47-semantic-slam-shadow",
|
||||
"m48s-fixed-class-detector",
|
||||
"m48t-risk-quality-temporal",
|
||||
}
|
||||
)
|
||||
assert len(execution.definitions) + len(execution.legacy_work_ids) == len(
|
||||
evidence.definitions
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
root / "config" / "laboratory-value-review.json"
|
||||
)
|
||||
|
||||
assert len(registry.entries) == 36
|
||||
assert len(registry.entries) == 37
|
||||
assert {entry.catalog_id for entry in registry.entries} >= {
|
||||
"e28-local-surface",
|
||||
"e46d-temporal-failure-audit",
|
||||
@@ -89,4 +89,5 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
"l34f-adjudicated-reference",
|
||||
"m4-replay-threat",
|
||||
"m48s-fixed-class-detector",
|
||||
"m48t-risk-quality-temporal",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.perception.m48t_risk_quality import (
|
||||
CocoRiskImage,
|
||||
M48TRiskQualityError,
|
||||
RiskPrediction,
|
||||
RiskTruth,
|
||||
load_coco_risk_truth,
|
||||
load_m48t_risk_quality_profile,
|
||||
score_risk_quality,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROFILE_PATH = REPOSITORY_ROOT / "config/perception/m48t-risk-quality-temporal-v1.json"
|
||||
|
||||
|
||||
def _truth(
|
||||
annotation_id: int,
|
||||
class_name: str,
|
||||
family: str,
|
||||
bbox: tuple[float, float, float, float],
|
||||
*,
|
||||
size_band: str = "medium",
|
||||
) -> RiskTruth:
|
||||
return RiskTruth(
|
||||
image_id=1,
|
||||
annotation_id=annotation_id,
|
||||
class_name=class_name,
|
||||
family=family,
|
||||
bbox_xyxy=bbox,
|
||||
projected_area_pixels=(bbox[2] - bbox[0]) * (bbox[3] - bbox[1]),
|
||||
size_band=size_band,
|
||||
)
|
||||
|
||||
|
||||
def _prediction(
|
||||
prediction_id: str,
|
||||
class_name: str,
|
||||
family: str,
|
||||
bbox: tuple[float, float, float, float],
|
||||
score: float = 0.9,
|
||||
) -> RiskPrediction:
|
||||
return RiskPrediction(
|
||||
image_id=1,
|
||||
prediction_id=prediction_id,
|
||||
class_name=class_name,
|
||||
family=family,
|
||||
score=score,
|
||||
bbox_xyxy=bbox,
|
||||
)
|
||||
|
||||
|
||||
def test_m48t_profile_pins_candidate_and_class_independent_temporal_policy() -> None:
|
||||
profile = load_m48t_risk_quality_profile(PROFILE_PATH)
|
||||
|
||||
assert profile.minimum_score == 0.25
|
||||
assert profile.model_id == "rf_detr_large:1"
|
||||
assert profile.class_to_family["dog"] == "animal"
|
||||
assert profile.temporal.initial_confirmation_observations == 2
|
||||
assert profile.temporal.switch_confirmation_observations == 3
|
||||
|
||||
|
||||
def test_m48t_profile_rejects_candidate_threshold_tuning(tmp_path: Path) -> None:
|
||||
document = json.loads(PROFILE_PATH.read_text("utf-8"))
|
||||
document["candidate"]["minimum_score"] = 0.2
|
||||
changed = tmp_path / "changed.json"
|
||||
changed.write_text(json.dumps(document), "utf-8")
|
||||
|
||||
with pytest.raises(M48TRiskQualityError, match="tuned in place"):
|
||||
load_m48t_risk_quality_profile(changed)
|
||||
|
||||
|
||||
def test_coco_truth_is_projected_to_actual_source_contract_and_filtered(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = load_m48t_risk_quality_profile(PROFILE_PATH)
|
||||
annotations = {
|
||||
"images": [{"id": 1, "file_name": "one.jpg", "width": 400, "height": 300}],
|
||||
"categories": [
|
||||
{"id": index, "name": class_name}
|
||||
for index, class_name in enumerate(profile.class_to_family, start=1)
|
||||
],
|
||||
"annotations": [
|
||||
{"id": 1, "image_id": 1, "category_id": 1, "bbox": [10, 20, 20, 30], "iscrowd": 0},
|
||||
{"id": 2, "image_id": 1, "category_id": 1, "bbox": [1, 1, 1, 1], "iscrowd": 0},
|
||||
{"id": 3, "image_id": 1, "category_id": 1, "bbox": [10, 20, 20, 30], "iscrowd": 1},
|
||||
],
|
||||
}
|
||||
path = tmp_path / "instances.json"
|
||||
path.write_text(json.dumps(annotations), "utf-8")
|
||||
|
||||
images, truth = load_coco_risk_truth(path, profile)
|
||||
|
||||
assert images == (CocoRiskImage(image_id=1, file_name="one.jpg", width=400, height=300),)
|
||||
assert len(truth) == 1
|
||||
assert truth[0].bbox_xyxy == (20.0, 40.0, 60.0, 100.0)
|
||||
assert truth[0].projected_area_pixels == 2400.0
|
||||
|
||||
|
||||
def test_quality_separates_exact_class_family_and_failure_buckets() -> None:
|
||||
profile = load_m48t_risk_quality_profile(PROFILE_PATH)
|
||||
images = (CocoRiskImage(image_id=1, file_name="one.jpg", width=800, height=600),)
|
||||
truth = (
|
||||
_truth(1, "person", "person", (10.0, 10.0, 110.0, 210.0), size_band="large"),
|
||||
_truth(2, "dog", "animal", (200.0, 100.0, 260.0, 170.0)),
|
||||
_truth(3, "car", "vehicle", (400.0, 200.0, 600.0, 350.0), size_band="large"),
|
||||
_truth(4, "bicycle", "light-road-user", (650.0, 200.0, 760.0, 350.0)),
|
||||
)
|
||||
predictions = (
|
||||
_prediction("p1", "person", "person", (10.0, 10.0, 110.0, 210.0)),
|
||||
_prediction("p2", "cat", "animal", (200.0, 100.0, 260.0, 170.0)),
|
||||
_prediction("p3", "car", "vehicle", (520.0, 300.0, 700.0, 450.0)),
|
||||
_prediction("p4", "truck", "vehicle", (300.0, 20.0, 390.0, 100.0)),
|
||||
)
|
||||
|
||||
result = score_risk_quality(
|
||||
images=images,
|
||||
truth=truth,
|
||||
predictions=predictions,
|
||||
profile=profile,
|
||||
)
|
||||
|
||||
assert result.report["counts"] == {
|
||||
"predictions": 4,
|
||||
"true_positive": 1,
|
||||
"false_positive": 3,
|
||||
"false_negative": 3,
|
||||
"empty_prediction_risk_images": 0,
|
||||
}
|
||||
metrics = result.report["metrics"]
|
||||
assert isinstance(metrics, dict)
|
||||
families = metrics["families"]
|
||||
assert isinstance(families, dict)
|
||||
assert families["animal"]["exact_class_recall"] == 0.0
|
||||
assert families["animal"]["family_recall"] == 1.0
|
||||
buckets = result.report["failure_buckets"]
|
||||
assert buckets["same-family-class-confusion"] == 1
|
||||
assert buckets["localization"] == 1
|
||||
assert buckets["missed"] == 1
|
||||
assert buckets["unmatched-prediction"] == 3
|
||||
assert result.report["quality_gates"]["passed"] is False
|
||||
assert result.report["authority"]["candidate_accepted"] is False
|
||||
|
||||
|
||||
def test_quality_rejects_predictions_below_frozen_threshold() -> None:
|
||||
profile = load_m48t_risk_quality_profile(PROFILE_PATH)
|
||||
images = (CocoRiskImage(image_id=1, file_name="one.jpg", width=800, height=600),)
|
||||
|
||||
with pytest.raises(M48TRiskQualityError, match="escaped the frozen score"):
|
||||
score_risk_quality(
|
||||
images=images,
|
||||
truth=(_truth(1, "person", "person", (10.0, 10.0, 100.0, 200.0)),),
|
||||
predictions=(
|
||||
_prediction(
|
||||
"p1",
|
||||
"person",
|
||||
"person",
|
||||
(10.0, 10.0, 100.0, 200.0),
|
||||
score=0.24,
|
||||
),
|
||||
),
|
||||
profile=profile,
|
||||
)
|
||||
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.laboratory.m48t_risk_quality_lab import (
|
||||
CATALOG_SCHEMA,
|
||||
LAB_SCHEMA,
|
||||
REPORT_SCHEMA,
|
||||
RESULT_PREFIX,
|
||||
)
|
||||
from k1link.perception.fixed_class_detector_tournament import canonical_json, false_authority
|
||||
from k1link.web.m48t_risk_quality_lab_api import build_m48t_risk_quality_lab_router
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.write_bytes(canonical_json(value) + b"\n")
|
||||
|
||||
|
||||
def _descriptor(path: Path, root: Path, role: str, media_type: str) -> dict[str, object]:
|
||||
return {
|
||||
"role": role,
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
"media_type": media_type,
|
||||
"schema_version": None,
|
||||
}
|
||||
|
||||
|
||||
def _fixture(tmp_path: Path) -> tuple[TestClient, Path, str]:
|
||||
root = tmp_path / "results"
|
||||
root.mkdir()
|
||||
identity = {"schema_version": LAB_SCHEMA, "authority": false_authority()}
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = RESULT_PREFIX + identity_sha256
|
||||
result_root = root / result_id
|
||||
review_root = result_root / "review"
|
||||
review_root.mkdir(parents=True)
|
||||
cases = []
|
||||
image_paths = []
|
||||
for index in range(16):
|
||||
case_id = f"{index + 1:012d}"
|
||||
image_path = review_root / f"review-{case_id}.jpg"
|
||||
image_path.write_bytes(b"jpeg" + bytes([index]))
|
||||
image_paths.append(image_path)
|
||||
cases.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"image_id": index + 1,
|
||||
"path": f"review/{image_path.name}",
|
||||
"media_type": "image/jpeg",
|
||||
"byte_length": image_path.stat().st_size,
|
||||
"sha256": hashlib.sha256(image_path.read_bytes()).hexdigest(),
|
||||
}
|
||||
)
|
||||
catalog = {
|
||||
"schema_version": CATALOG_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"case_count": 16,
|
||||
"legend": {"ground_truth": "green", "rf_detr_prediction": "yellow"},
|
||||
"cases": cases,
|
||||
}
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"source": {
|
||||
"quality": {
|
||||
"dataset_id": "coco-2017-val",
|
||||
"risk_images": 3348,
|
||||
"truth_instances": 16060,
|
||||
},
|
||||
"temporal": {"source_id": "RAVNOVES00", "frames": 4481},
|
||||
},
|
||||
"configuration": {},
|
||||
"method": {"schema_version": "missioncore.laboratory-method/v1"},
|
||||
"execution": {},
|
||||
"metrics": {},
|
||||
"acceptance": {},
|
||||
"decision": {
|
||||
"quality_accepted": False,
|
||||
"temporal_invariant_passed": True,
|
||||
},
|
||||
"limitations": [],
|
||||
"authority": false_authority(),
|
||||
}
|
||||
catalog_path = result_root / "catalog.json"
|
||||
report_path = result_root / "report.json"
|
||||
_write_json(catalog_path, catalog)
|
||||
_write_json(report_path, report)
|
||||
artifacts = [
|
||||
_descriptor(catalog_path, result_root, "visual-evidence-catalog", "application/json"),
|
||||
_descriptor(report_path, result_root, "laboratory-report", "application/json"),
|
||||
*[
|
||||
_descriptor(path, result_root, "visual-evidence-independent-truth-review", "image/jpeg")
|
||||
for path in image_paths
|
||||
],
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": LAB_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": "2026-08-25T17:38:01.505739Z",
|
||||
"status": "complete-quality-gate-failed-temporal-invariant-passed",
|
||||
"completed": True,
|
||||
"bounded_question_accepted": False,
|
||||
"ground_truth": False,
|
||||
"authority": false_authority(),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(result_root / "manifest.json", manifest)
|
||||
app = FastAPI()
|
||||
app.include_router(build_m48t_risk_quality_lab_router(root_provider=lambda: root))
|
||||
return TestClient(app), result_root, result_id
|
||||
|
||||
|
||||
def test_m48t_lab_api_projects_failed_quality_and_verified_review(tmp_path: Path) -> None:
|
||||
client, result_root, result_id = _fixture(tmp_path)
|
||||
|
||||
catalog = client.get("/api/v1/laboratory/m48t/risk-quality/results")
|
||||
assert catalog.status_code == 200
|
||||
assert catalog.json()["items"][0]["result_id"] == result_id
|
||||
assert catalog.json()["invalid_total"] == 0
|
||||
|
||||
result = client.get(f"/api/v1/laboratory/m48t/risk-quality/results/{result_id}")
|
||||
assert result.status_code == 200
|
||||
assert result.json()["decision"]["quality_accepted"] is False
|
||||
assert result.json()["decision"]["temporal_invariant_passed"] is True
|
||||
assert len(result.json()["review"]["cases"]) == 16
|
||||
|
||||
case_id = "000000000001"
|
||||
image = client.get(
|
||||
f"/api/v1/laboratory/m48t/risk-quality/results/{result_id}/review/{case_id}.jpg"
|
||||
)
|
||||
assert image.status_code == 200
|
||||
assert image.headers["content-type"] == "image/jpeg"
|
||||
assert image.content == (result_root / f"review/review-{case_id}.jpg").read_bytes()
|
||||
|
||||
|
||||
def test_m48t_lab_api_fails_closed_after_visual_tamper(tmp_path: Path) -> None:
|
||||
client, result_root, result_id = _fixture(tmp_path)
|
||||
(result_root / "review/review-000000000001.jpg").write_bytes(b"tampered")
|
||||
|
||||
response = client.get(f"/api/v1/laboratory/m48t/risk-quality/results/{result_id}")
|
||||
assert response.status_code == 404
|
||||
catalog = client.get("/api/v1/laboratory/m48t/risk-quality/results")
|
||||
assert catalog.json()["items"] == []
|
||||
assert catalog.json()["invalid_total"] == 1
|
||||
@@ -0,0 +1,104 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.perception.m48t_risk_quality import (
|
||||
BoundedTemporalSemanticIdentity,
|
||||
M48TRiskQualityError,
|
||||
TemporalSemanticObservation,
|
||||
load_m48t_risk_quality_profile,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
PROFILE_PATH = REPOSITORY_ROOT / "config/perception/m48t-risk-quality-temporal-v1.json"
|
||||
|
||||
|
||||
def _observation(
|
||||
time_ns: int,
|
||||
raw_class: str | None,
|
||||
*,
|
||||
component_id: str = "temporal-000001",
|
||||
currentness: str = "current",
|
||||
) -> TemporalSemanticObservation:
|
||||
return TemporalSemanticObservation(
|
||||
component_id=component_id,
|
||||
evidence_time_ns=time_ns,
|
||||
raw_class_name=raw_class,
|
||||
currentness=currentness,
|
||||
)
|
||||
|
||||
|
||||
def test_initial_class_requires_two_observations_and_identity_stays_geometry_owned() -> None:
|
||||
stabilizer = BoundedTemporalSemanticIdentity(load_m48t_risk_quality_profile(PROFILE_PATH))
|
||||
|
||||
first = stabilizer.update(_observation(0, "person"))
|
||||
second = stabilizer.update(_observation(100_000_000, "person"))
|
||||
|
||||
assert first.resolution == "pending"
|
||||
assert first.selected_class_name is None
|
||||
assert second.resolution == "confirmed"
|
||||
assert second.selected_class_name == "person"
|
||||
assert second.component_id == "temporal-000001"
|
||||
assert second.association_uses_semantic_class is False
|
||||
assert second.occupancy_uses_semantic_class is False
|
||||
|
||||
|
||||
def test_cross_family_switch_falls_back_unknown_until_third_confirmation() -> None:
|
||||
stabilizer = BoundedTemporalSemanticIdentity(load_m48t_risk_quality_profile(PROFILE_PATH))
|
||||
stabilizer.update(_observation(0, "person"))
|
||||
stabilizer.update(_observation(100_000_000, "person"))
|
||||
|
||||
first_conflict = stabilizer.update(_observation(200_000_000, "car"))
|
||||
second_conflict = stabilizer.update(_observation(300_000_000, "car"))
|
||||
switched = stabilizer.update(_observation(400_000_000, "car"))
|
||||
|
||||
assert first_conflict.resolution == "conflict"
|
||||
assert first_conflict.selected_class_name is None
|
||||
assert second_conflict.resolution == "conflict"
|
||||
assert switched.resolution == "confirmed"
|
||||
assert switched.selected_class_name == "car"
|
||||
assert stabilizer.snapshot().class_switches == 1
|
||||
|
||||
|
||||
def test_same_family_switch_holds_previous_class_until_confirmed() -> None:
|
||||
stabilizer = BoundedTemporalSemanticIdentity(load_m48t_risk_quality_profile(PROFILE_PATH))
|
||||
stabilizer.update(_observation(0, "dog"))
|
||||
stabilizer.update(_observation(100_000_000, "dog"))
|
||||
|
||||
pending = stabilizer.update(_observation(200_000_000, "cat"))
|
||||
stabilizer.update(_observation(300_000_000, "cat"))
|
||||
switched = stabilizer.update(_observation(400_000_000, "cat"))
|
||||
|
||||
assert pending.resolution == "pending"
|
||||
assert pending.selected_class_name == "dog"
|
||||
assert switched.selected_class_name == "cat"
|
||||
|
||||
|
||||
def test_semantic_hold_is_bounded_then_degrades_to_unknown() -> None:
|
||||
stabilizer = BoundedTemporalSemanticIdentity(load_m48t_risk_quality_profile(PROFILE_PATH))
|
||||
stabilizer.update(_observation(0, "person"))
|
||||
stabilizer.update(_observation(100_000_000, "person"))
|
||||
|
||||
held = stabilizer.update(_observation(300_000_000, None, currentness="held"))
|
||||
unknown = stabilizer.update(_observation(500_000_001, None, currentness="held"))
|
||||
|
||||
assert held.resolution == "held"
|
||||
assert held.selected_class_name == "person"
|
||||
assert unknown.resolution == "unknown"
|
||||
assert unknown.selected_class_name is None
|
||||
|
||||
|
||||
def test_explicit_expiry_removes_state_and_out_of_order_evidence_is_rejected() -> None:
|
||||
stabilizer = BoundedTemporalSemanticIdentity(load_m48t_risk_quality_profile(PROFILE_PATH))
|
||||
stabilizer.update(_observation(100, "person"))
|
||||
with pytest.raises(M48TRiskQualityError, match="moved backwards"):
|
||||
stabilizer.update(_observation(99, "person"))
|
||||
|
||||
expired = stabilizer.update(_observation(200, None, currentness="expired"))
|
||||
restarted = stabilizer.update(_observation(300, "person"))
|
||||
|
||||
assert expired.resolution == "expired"
|
||||
assert restarted.resolution == "pending"
|
||||
assert stabilizer.snapshot().active_components == 1
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUNNER_PATH = REPOSITORY_ROOT / "experiments/perception/run_m48t_upstream_parity_worker.py"
|
||||
PROFILE_PATH = REPOSITORY_ROOT / "config/perception/m48t-upstream-parity-v1.json"
|
||||
|
||||
|
||||
def load_runner() -> ModuleType:
|
||||
specification = importlib.util.spec_from_file_location("m48t_upstream_parity", RUNNER_PATH)
|
||||
assert specification is not None and specification.loader is not None
|
||||
module = importlib.util.module_from_spec(specification)
|
||||
specification.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_upstream_parity_profile_freezes_official_and_deployed_providers() -> None:
|
||||
profile = json.loads(PROFILE_PATH.read_text("utf-8"))
|
||||
|
||||
assert profile["model"]["package_version"] == "1.9.4"
|
||||
assert profile["model"]["resolution"] == [704, 704]
|
||||
assert profile["dataset"]["image_count"] == 5000
|
||||
assert profile["providers"]["pytorch"]["confidence_prefilter"] == 0.0
|
||||
assert profile["providers"]["tensorrt"]["confidence_prefilter"] == 0.0
|
||||
assert profile["authority"]["navigation_or_safety_accepted"] is False
|
||||
|
||||
|
||||
def test_tensorrt_decoder_uses_sparse_coco_ids_and_original_image_geometry() -> None:
|
||||
runner = load_runner()
|
||||
boxes = np.zeros((1, 300, 4), dtype=np.float16)
|
||||
logits = np.full((1, 300, 91), -20, dtype=np.float16)
|
||||
boxes[0, 4] = np.asarray((0.5, 0.5, 0.2, 0.4), dtype=np.float16)
|
||||
logits[0, 4, 1] = np.float16(4.0) # COCO sparse id 1: person
|
||||
logits[0, 7, 12] = np.float16(5.0) # sparse gap: must never escape
|
||||
|
||||
rows = runner.decode_tensorrt_coco_rows(
|
||||
image_id=42,
|
||||
image_width=1000,
|
||||
image_height=500,
|
||||
boxes=boxes,
|
||||
logits=logits,
|
||||
maximum_detections=2,
|
||||
)
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["image_id"] == 42
|
||||
assert rows[0]["category_id"] == 1
|
||||
assert np.allclose(rows[0]["bbox"], [400.0244, 149.9756, 199.9512, 200.0488], atol=0.1)
|
||||
|
||||
|
||||
def test_parity_decision_localizes_upstream_and_deployment_failures() -> None:
|
||||
runner = load_runner()
|
||||
profile = json.loads(PROFILE_PATH.read_text("utf-8"))
|
||||
|
||||
passed = runner.build_parity_decision(
|
||||
profile=profile,
|
||||
pytorch_metrics={"ap_50_95": 0.565, "ap_50": 0.751},
|
||||
tensorrt_metrics={"ap_50_95": 0.563, "ap_50": 0.749},
|
||||
full_admission_run=True,
|
||||
)
|
||||
assert passed["passed"] is True
|
||||
assert passed["diagnosis"] == "upstream-and-tensorrt-parity-passed"
|
||||
|
||||
deployment_failure = runner.build_parity_decision(
|
||||
profile=profile,
|
||||
pytorch_metrics={"ap_50_95": 0.565, "ap_50": 0.751},
|
||||
tensorrt_metrics={"ap_50_95": 0.54, "ap_50": 0.72},
|
||||
full_admission_run=True,
|
||||
)
|
||||
assert deployment_failure["passed"] is False
|
||||
assert deployment_failure["diagnosis"] == "tensorrt-deployment-parity-failed"
|
||||
|
||||
smoke = runner.build_parity_decision(
|
||||
profile=profile,
|
||||
pytorch_metrics={"ap_50_95": 0.565, "ap_50": 0.751},
|
||||
tensorrt_metrics={"ap_50_95": 0.565, "ap_50": 0.751},
|
||||
full_admission_run=False,
|
||||
)
|
||||
assert smoke["passed"] is False
|
||||
assert smoke["diagnosis"] == "bounded-smoke-only"
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.simulation.gaussian_pipeline_gateway import GaussianPipelineGateway
|
||||
from k1link.web.simulation_world_provider_api import build_simulation_world_provider_router
|
||||
|
||||
|
||||
def _gateway(tmp_path: Path, status: int = 200) -> GaussianPipelineGateway:
|
||||
token = tmp_path / "token"
|
||||
token.write_text("x" * 64, encoding="utf-8")
|
||||
|
||||
def handler(_request: httpx.Request) -> httpx.Response:
|
||||
if status != 200:
|
||||
return httpx.Response(status)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"schema_version": "gaussian-pipeline.capabilities/v1",
|
||||
"service": "ndc-gaussian-pipeline",
|
||||
"api_version": "gaussian-pipeline.api/v1",
|
||||
"provider": {"id": "playcanvas-splat-transform", "version": "3.3.3"},
|
||||
"runtime": {
|
||||
"source_revision": "a" * 40,
|
||||
"image_digest": f"sha256:{'b' * 64}",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
return GaussianPipelineGateway(
|
||||
"http://gaussian.test",
|
||||
token,
|
||||
transport=httpx.MockTransport(handler),
|
||||
)
|
||||
|
||||
|
||||
def test_provider_status_is_explicitly_unconfigured() -> None:
|
||||
app = FastAPI()
|
||||
app.include_router(build_simulation_world_provider_router(lambda: None))
|
||||
response = TestClient(app).get("/api/v1/simulation-worlds/provider")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"schema_version": "missioncore.simulation-world-provider-status/v1",
|
||||
"provider_id": "gaussian-pipeline",
|
||||
"configured": False,
|
||||
"state": "unavailable",
|
||||
"capabilities": None,
|
||||
}
|
||||
|
||||
|
||||
def test_provider_status_projects_ready_capabilities(tmp_path: Path) -> None:
|
||||
app = FastAPI()
|
||||
app.include_router(build_simulation_world_provider_router(lambda: _gateway(tmp_path)))
|
||||
response = TestClient(app).get("/api/v1/simulation-worlds/provider")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["state"] == "ready"
|
||||
assert response.json()["capabilities"]["provider"]["version"] == "3.3.3"
|
||||
|
||||
|
||||
def test_provider_status_fails_closed_without_leaking_transport_error(tmp_path: Path) -> None:
|
||||
app = FastAPI()
|
||||
app.include_router(build_simulation_world_provider_router(lambda: _gateway(tmp_path, 503)))
|
||||
response = TestClient(app).get("/api/v1/simulation-worlds/provider")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["configured"] is True
|
||||
assert response.json()["state"] == "unavailable"
|
||||
assert response.json()["capabilities"] is None
|
||||
Reference in New Issue
Block a user