feat(lab): measure RAVNOVES00 R1 quality baseline
This commit is contained in:
@@ -132,6 +132,53 @@ export interface E37AcceptanceContractResult {
|
|||||||
access: "read-only";
|
access: "read-only";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface E38DimensionMetric {
|
||||||
|
correct: number;
|
||||||
|
incorrect: number;
|
||||||
|
total: number;
|
||||||
|
accuracy: number;
|
||||||
|
target: number;
|
||||||
|
passed: boolean;
|
||||||
|
byStratum: Readonly<Record<string, {
|
||||||
|
correct: number;
|
||||||
|
incorrect: number;
|
||||||
|
total: number;
|
||||||
|
accuracy: number;
|
||||||
|
}>>;
|
||||||
|
confusion: readonly {
|
||||||
|
reference: string;
|
||||||
|
prediction: string;
|
||||||
|
count: number;
|
||||||
|
}[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface E38PerceptionBaselineResult {
|
||||||
|
resultId: string;
|
||||||
|
createdAtUtc: string | null;
|
||||||
|
sourceSessionId: string;
|
||||||
|
sourceDisplayName: string;
|
||||||
|
status: "measured-r1-source-scoped-baseline";
|
||||||
|
profileId: string;
|
||||||
|
workerNode: string;
|
||||||
|
qualityGatePassed: boolean;
|
||||||
|
metrics: {
|
||||||
|
developmentItems: number;
|
||||||
|
validationItems: number;
|
||||||
|
terminalOutcomes: number;
|
||||||
|
accountingFraction: number;
|
||||||
|
falseFreeClaims: number;
|
||||||
|
highSeverityFailures: number;
|
||||||
|
dimensions: {
|
||||||
|
presence: E38DimensionMetric;
|
||||||
|
geometryAssociation: E38DimensionMetric;
|
||||||
|
freshness: E38DimensionMetric;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
blockingChecks: readonly string[];
|
||||||
|
limitations: readonly string[];
|
||||||
|
access: "read-only";
|
||||||
|
}
|
||||||
|
|
||||||
export interface AdvancedLaboratoryResults {
|
export interface AdvancedLaboratoryResults {
|
||||||
e31: E31LaboratoryResult | null;
|
e31: E31LaboratoryResult | null;
|
||||||
e32: E32LaboratoryResult | null;
|
e32: E32LaboratoryResult | null;
|
||||||
@@ -139,6 +186,7 @@ export interface AdvancedLaboratoryResults {
|
|||||||
e34: E34TemporalLayerResult | null;
|
e34: E34TemporalLayerResult | null;
|
||||||
e35: E35DegradationRecoveryResult | null;
|
e35: E35DegradationRecoveryResult | null;
|
||||||
e37: E37AcceptanceContractResult | null;
|
e37: E37AcceptanceContractResult | null;
|
||||||
|
e38: E38PerceptionBaselineResult | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class AdvancedLaboratoryContractError extends Error {
|
export class AdvancedLaboratoryContractError extends Error {
|
||||||
@@ -539,6 +587,147 @@ function parseE37(value: unknown): E37AcceptanceContractResult {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseE38Dimension(
|
||||||
|
value: unknown,
|
||||||
|
label: string,
|
||||||
|
): E38DimensionMetric {
|
||||||
|
const source = record(value, label);
|
||||||
|
const byStratumSource = record(source.by_stratum, `${label}.by_stratum`);
|
||||||
|
if (!Array.isArray(source.confusion)) {
|
||||||
|
throw new AdvancedLaboratoryContractError(
|
||||||
|
`${label}.confusion: ожидался массив.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
correct: integerValue(source.correct, `${label}.correct`),
|
||||||
|
incorrect: integerValue(source.incorrect, `${label}.incorrect`),
|
||||||
|
total: integerValue(source.total, `${label}.total`),
|
||||||
|
accuracy: numberValue(source.accuracy, `${label}.accuracy`),
|
||||||
|
target: numberValue(source.target, `${label}.target`),
|
||||||
|
passed: booleanValue(source.passed, `${label}.passed`),
|
||||||
|
byStratum: Object.fromEntries(
|
||||||
|
Object.entries(byStratumSource).map(([stratum, value]) => {
|
||||||
|
const metric = record(value, `${label}.by_stratum.${stratum}`);
|
||||||
|
return [
|
||||||
|
stratum,
|
||||||
|
{
|
||||||
|
correct: integerValue(
|
||||||
|
metric.correct,
|
||||||
|
`${label}.by_stratum.${stratum}.correct`,
|
||||||
|
),
|
||||||
|
incorrect: integerValue(
|
||||||
|
metric.incorrect,
|
||||||
|
`${label}.by_stratum.${stratum}.incorrect`,
|
||||||
|
),
|
||||||
|
total: integerValue(
|
||||||
|
metric.total,
|
||||||
|
`${label}.by_stratum.${stratum}.total`,
|
||||||
|
),
|
||||||
|
accuracy: numberValue(
|
||||||
|
metric.accuracy,
|
||||||
|
`${label}.by_stratum.${stratum}.accuracy`,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
confusion: source.confusion.map((value, index) => {
|
||||||
|
const row = record(value, `${label}.confusion[${index}]`);
|
||||||
|
return {
|
||||||
|
reference: stringValue(
|
||||||
|
row.reference,
|
||||||
|
`${label}.confusion[${index}].reference`,
|
||||||
|
),
|
||||||
|
prediction: stringValue(
|
||||||
|
row.prediction,
|
||||||
|
`${label}.confusion[${index}].prediction`,
|
||||||
|
),
|
||||||
|
count: integerValue(
|
||||||
|
row.count,
|
||||||
|
`${label}.confusion[${index}].count`,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseE38(value: unknown): E38PerceptionBaselineResult {
|
||||||
|
const item = record(value, "E38");
|
||||||
|
const metrics = record(item.metrics, "E38.metrics");
|
||||||
|
const dimensions = record(metrics.dimensions, "E38.metrics.dimensions");
|
||||||
|
diagnosticAuthority(item.authority, "E38.authority");
|
||||||
|
return {
|
||||||
|
resultId: contentId(
|
||||||
|
item.result_id,
|
||||||
|
"e38-perception-baseline",
|
||||||
|
"E38.result_id",
|
||||||
|
),
|
||||||
|
createdAtUtc: optionalString(item.created_at_utc, "E38.created_at_utc"),
|
||||||
|
sourceSessionId: stringValue(
|
||||||
|
item.source_session_id,
|
||||||
|
"E38.source_session_id",
|
||||||
|
),
|
||||||
|
sourceDisplayName: stringValue(
|
||||||
|
item.source_display_name,
|
||||||
|
"E38.source_display_name",
|
||||||
|
),
|
||||||
|
status: exactString(
|
||||||
|
item.status,
|
||||||
|
"measured-r1-source-scoped-baseline",
|
||||||
|
"E38.status",
|
||||||
|
),
|
||||||
|
profileId: stringValue(item.profile_id, "E38.profile_id"),
|
||||||
|
workerNode: stringValue(item.worker_node, "E38.worker_node"),
|
||||||
|
qualityGatePassed: booleanValue(
|
||||||
|
item.quality_gate_passed,
|
||||||
|
"E38.quality_gate_passed",
|
||||||
|
),
|
||||||
|
metrics: {
|
||||||
|
developmentItems: integerValue(
|
||||||
|
metrics.development_items,
|
||||||
|
"E38.metrics.development_items",
|
||||||
|
),
|
||||||
|
validationItems: integerValue(
|
||||||
|
metrics.validation_items,
|
||||||
|
"E38.metrics.validation_items",
|
||||||
|
),
|
||||||
|
terminalOutcomes: integerValue(
|
||||||
|
metrics.terminal_outcomes,
|
||||||
|
"E38.metrics.terminal_outcomes",
|
||||||
|
),
|
||||||
|
accountingFraction: numberValue(
|
||||||
|
metrics.accounting_fraction,
|
||||||
|
"E38.metrics.accounting_fraction",
|
||||||
|
),
|
||||||
|
falseFreeClaims: integerValue(
|
||||||
|
metrics.false_free_claims,
|
||||||
|
"E38.metrics.false_free_claims",
|
||||||
|
),
|
||||||
|
highSeverityFailures: integerValue(
|
||||||
|
metrics.high_severity_failures,
|
||||||
|
"E38.metrics.high_severity_failures",
|
||||||
|
),
|
||||||
|
dimensions: {
|
||||||
|
presence: parseE38Dimension(
|
||||||
|
dimensions.presence,
|
||||||
|
"E38.metrics.dimensions.presence",
|
||||||
|
),
|
||||||
|
geometryAssociation: parseE38Dimension(
|
||||||
|
dimensions.geometry_association,
|
||||||
|
"E38.metrics.dimensions.geometry_association",
|
||||||
|
),
|
||||||
|
freshness: parseE38Dimension(
|
||||||
|
dimensions.freshness,
|
||||||
|
"E38.metrics.dimensions.freshness",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
blockingChecks: strings(item.blocking_checks, "E38.blocking_checks"),
|
||||||
|
limitations: strings(item.limitations, "E38.limitations"),
|
||||||
|
access: exactString(item.access, "read-only", "E38.access"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchOne<T>(
|
async function fetchOne<T>(
|
||||||
path: string,
|
path: string,
|
||||||
parser: (value: unknown) => T,
|
parser: (value: unknown) => T,
|
||||||
@@ -563,15 +752,16 @@ export async function fetchAdvancedLaboratoryResults({
|
|||||||
fetcher?: LaboratoryFetch;
|
fetcher?: LaboratoryFetch;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
} = {}): Promise<AdvancedLaboratoryResults> {
|
} = {}): Promise<AdvancedLaboratoryResults> {
|
||||||
const [e31, e32, e33, e34, e35, e37] = await Promise.all([
|
const [e31, e32, e33, e34, e35, e37, e38] = await Promise.all([
|
||||||
fetchOne("/api/v1/laboratory/e31/results?limit=1", parseE31, fetcher, signal),
|
fetchOne("/api/v1/laboratory/e31/results?limit=1", parseE31, fetcher, signal),
|
||||||
fetchOne("/api/v1/laboratory/e32/results?limit=1", parseE32, fetcher, signal),
|
fetchOne("/api/v1/laboratory/e32/results?limit=1", parseE32, fetcher, signal),
|
||||||
fetchOne("/api/v1/laboratory/e33/results?limit=1", parseE33, fetcher, signal),
|
fetchOne("/api/v1/laboratory/e33/results?limit=1", parseE33, fetcher, signal),
|
||||||
fetchE34TemporalLayerResult({ fetcher, signal }),
|
fetchE34TemporalLayerResult({ fetcher, signal }),
|
||||||
fetchE35DegradationRecoveryResult({ fetcher, signal }),
|
fetchE35DegradationRecoveryResult({ fetcher, signal }),
|
||||||
fetchOne("/api/v1/laboratory/e37/results?limit=1", parseE37, fetcher, signal),
|
fetchOne("/api/v1/laboratory/e37/results?limit=1", parseE37, fetcher, signal),
|
||||||
|
fetchOne("/api/v1/laboratory/e38/results?limit=1", parseE38, fetcher, signal),
|
||||||
]);
|
]);
|
||||||
return { e31, e32, e33, e34, e35, e37 };
|
return { e31, e32, e33, e34, e35, e37, e38 };
|
||||||
}
|
}
|
||||||
import {
|
import {
|
||||||
fetchE34TemporalLayerResult,
|
fetchE34TemporalLayerResult,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { E33Result } from "./E33Result";
|
|||||||
import { E34Result } from "./E34Result";
|
import { E34Result } from "./E34Result";
|
||||||
import { E35Result } from "./E35Result";
|
import { E35Result } from "./E35Result";
|
||||||
import { E37Result } from "./E37Result";
|
import { E37Result } from "./E37Result";
|
||||||
|
import { E38Result } from "./E38Result";
|
||||||
import { RecordedReplayEvidence } from "./RecordedReplayEvidence";
|
import { RecordedReplayEvidence } from "./RecordedReplayEvidence";
|
||||||
|
|
||||||
export type AdvancedLaboratoryWorkId =
|
export type AdvancedLaboratoryWorkId =
|
||||||
@@ -18,7 +19,8 @@ export type AdvancedLaboratoryWorkId =
|
|||||||
| "e33-worker-shadow"
|
| "e33-worker-shadow"
|
||||||
| "e34-temporal-layer"
|
| "e34-temporal-layer"
|
||||||
| "e35-degradation-recovery"
|
| "e35-degradation-recovery"
|
||||||
| "e37-ravnoves-acceptance";
|
| "e37-ravnoves-acceptance"
|
||||||
|
| "e38-perception-baseline";
|
||||||
|
|
||||||
type LaboratoryWorkspaceProps = WorkspaceRendererProps & {
|
type LaboratoryWorkspaceProps = WorkspaceRendererProps & {
|
||||||
SpatialView: ComponentType<WorkspaceRendererProps>;
|
SpatialView: ComponentType<WorkspaceRendererProps>;
|
||||||
@@ -34,6 +36,7 @@ export function isAdvancedLaboratoryWorkId(
|
|||||||
|| value === "e34-temporal-layer"
|
|| value === "e34-temporal-layer"
|
||||||
|| value === "e35-degradation-recovery"
|
|| value === "e35-degradation-recovery"
|
||||||
|| value === "e37-ravnoves-acceptance"
|
|| value === "e37-ravnoves-acceptance"
|
||||||
|
|| value === "e38-perception-baseline"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,6 +81,12 @@ export function advancedLaboratoryWorkOptions(
|
|||||||
label: "LAB E37 · RAVNOVES00 acceptance R0",
|
label: "LAB E37 · RAVNOVES00 acceptance R0",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
if (results.e38) {
|
||||||
|
options.push({
|
||||||
|
id: "e38-perception-baseline",
|
||||||
|
label: "LAB E38 · perception quality R1",
|
||||||
|
});
|
||||||
|
}
|
||||||
return options;
|
return options;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,6 +124,9 @@ export function AdvancedLaboratoryResult({
|
|||||||
failedSessionId: string | null;
|
failedSessionId: string | null;
|
||||||
replayError: string | null;
|
replayError: string | null;
|
||||||
}) {
|
}) {
|
||||||
|
if (workId === "e38-perception-baseline" && results.e38) {
|
||||||
|
return <E38Result rigLabel={rigLabel} result={results.e38} />;
|
||||||
|
}
|
||||||
if (workId === "e37-ravnoves-acceptance" && results.e37) {
|
if (workId === "e37-ravnoves-acceptance" && results.e37) {
|
||||||
return <E37Result rigLabel={rigLabel} result={results.e37} />;
|
return <E37Result rigLabel={rigLabel} result={results.e37} />;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,188 @@
|
|||||||
|
import {
|
||||||
|
LaboratoryEvidence,
|
||||||
|
LaboratoryResultSummary,
|
||||||
|
LaboratorySummary,
|
||||||
|
LaboratoryWorkTemplate,
|
||||||
|
} from "../../components/laboratory/LaboratoryPresentation";
|
||||||
|
import type {
|
||||||
|
E38DimensionMetric,
|
||||||
|
E38PerceptionBaselineResult,
|
||||||
|
} from "../../core/laboratory/advancedResults";
|
||||||
|
import { formatNumber } from "../../presentation";
|
||||||
|
|
||||||
|
function percent(value: number): string {
|
||||||
|
return `${(value * 100).toLocaleString("ru-RU", {
|
||||||
|
maximumFractionDigits: 1,
|
||||||
|
})}%`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function worstStratum(metric: E38DimensionMetric): string {
|
||||||
|
const [name, value] = Object.entries(metric.byStratum)
|
||||||
|
.sort((left, right) => left[1].accuracy - right[1].accuracy)[0] ?? [];
|
||||||
|
return name && value
|
||||||
|
? `${name}: ${percent(value.accuracy)}`
|
||||||
|
: "нет данных";
|
||||||
|
}
|
||||||
|
|
||||||
|
function DimensionEvidence({
|
||||||
|
label,
|
||||||
|
metric,
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
metric: E38DimensionMetric;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<span>{label}</span>
|
||||||
|
<strong>{percent(metric.accuracy)}</strong>
|
||||||
|
<small>
|
||||||
|
{`${formatNumber(metric.correct, 0)} / ${formatNumber(metric.total, 0)}`}
|
||||||
|
{" · слабее всего: "}
|
||||||
|
{worstStratum(metric)}
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function E38Result({
|
||||||
|
rigLabel,
|
||||||
|
result,
|
||||||
|
}: {
|
||||||
|
rigLabel: string;
|
||||||
|
result: E38PerceptionBaselineResult;
|
||||||
|
}) {
|
||||||
|
const metrics = result.metrics;
|
||||||
|
const dimensions = metrics.dimensions;
|
||||||
|
const gateLabel = result.qualityGatePassed
|
||||||
|
? "R1 gate пройден"
|
||||||
|
: "R1 требует улучшения";
|
||||||
|
return (
|
||||||
|
<LaboratoryWorkTemplate
|
||||||
|
summary={(
|
||||||
|
<LaboratorySummary
|
||||||
|
title="LAB E38 · первый baseline качества R1"
|
||||||
|
description="На неизменяемом validation-наборе E37 отдельно измерены распознавание присутствия, принадлежность геометрии и свежесть состояния. Модель обучалась только на development-разделе; validation использован только для итоговой оценки."
|
||||||
|
status={gateLabel}
|
||||||
|
statusTone={result.qualityGatePassed ? "success" : "warning"}
|
||||||
|
facts={[
|
||||||
|
{
|
||||||
|
label: "Конфигурация",
|
||||||
|
value: `${rigLabel} · ${result.sourceDisplayName}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Обучение",
|
||||||
|
value: `${formatNumber(metrics.developmentItems, 0)} development`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Проверка",
|
||||||
|
value: `${formatNumber(metrics.validationItems, 0)} validation`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Исполнение",
|
||||||
|
value: `Worker 006 · ${result.workerNode}`,
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
brief={{
|
||||||
|
question: "Какое качество уже даёт текущий source-scoped сенсорный контур на замороженном holdout и какие узлы реально не дотягивают до 90%?",
|
||||||
|
approach: "Три неглубоких детерминированных CART-модели обучены на 340 development-кейсах по фактическим признакам E29/E30. Замороженные 146 validation-кейсов не участвовали в подборе деревьев и использованы только для оценки.",
|
||||||
|
principalResult: `Freshness достиг ${percent(dimensions.freshness.accuracy)} и прошёл порог. Presence — ${percent(dimensions.presence.accuracy)}, geometry association — ${percent(dimensions.geometryAssociation.accuracy)}; оба узла пока ниже 90%.`,
|
||||||
|
limitation: "Это инженерно размеченный baseline одного RAVNOVES00, а не независимый ground truth или доказательство переноса на другой маршрут, риг и камеру.",
|
||||||
|
}}
|
||||||
|
method={{
|
||||||
|
completeness: "complete",
|
||||||
|
executionClass: "deterministic",
|
||||||
|
pipelineId: result.profileId,
|
||||||
|
components: [
|
||||||
|
{
|
||||||
|
kind: "source",
|
||||||
|
name: "E37 frozen acceptance contract",
|
||||||
|
version: `${formatNumber(metrics.validationItems, 0)} sealed validation items`,
|
||||||
|
role: "неизменяемый denominator и три раздельные reference-метрики",
|
||||||
|
identitySha256: null,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
kind: "algorithm",
|
||||||
|
name: "Development-only shallow CART",
|
||||||
|
version: "dimension-specific deterministic trees",
|
||||||
|
role: "baseline без обучения на validation и без смешивания метрик",
|
||||||
|
identitySha256: null,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
kind: "runtime",
|
||||||
|
name: "Worker 006",
|
||||||
|
version: result.workerNode,
|
||||||
|
role: "изолированное исполнение pinned package без командных полномочий",
|
||||||
|
identitySha256: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
evidence={(
|
||||||
|
<LaboratoryEvidence
|
||||||
|
eyebrow="ЗАМОРОЖЕННЫЙ VALIDATION"
|
||||||
|
title="Три независимые метрики и слабые страты"
|
||||||
|
kind="diagnostic-model"
|
||||||
|
>
|
||||||
|
<div className="laboratory-result-metrics">
|
||||||
|
<DimensionEvidence
|
||||||
|
label="Presence"
|
||||||
|
metric={dimensions.presence}
|
||||||
|
/>
|
||||||
|
<DimensionEvidence
|
||||||
|
label="Geometry association"
|
||||||
|
metric={dimensions.geometryAssociation}
|
||||||
|
/>
|
||||||
|
<DimensionEvidence
|
||||||
|
label="Freshness"
|
||||||
|
metric={dimensions.freshness}
|
||||||
|
/>
|
||||||
|
<div>
|
||||||
|
<span>High severity</span>
|
||||||
|
<strong>{formatNumber(metrics.highSeverityFailures, 0)}</strong>
|
||||||
|
<small>
|
||||||
|
{"блокирующие ошибки независимо от среднего процента"}
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</LaboratoryEvidence>
|
||||||
|
)}
|
||||||
|
result={(
|
||||||
|
<LaboratoryResultSummary
|
||||||
|
title={result.qualityGatePassed
|
||||||
|
? "Все три метрики достигли source-scoped цели R1"
|
||||||
|
: "Baseline измерен; presence и geometry остаются открыты"}
|
||||||
|
status={gateLabel}
|
||||||
|
statusTone={result.qualityGatePassed ? "success" : "warning"}
|
||||||
|
metrics={[
|
||||||
|
{
|
||||||
|
label: "Presence",
|
||||||
|
value: percent(dimensions.presence.accuracy),
|
||||||
|
hint: `${formatNumber(dimensions.presence.incorrect, 0)} ошибок · цель 90%`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Geometry association",
|
||||||
|
value: percent(dimensions.geometryAssociation.accuracy),
|
||||||
|
hint: `${formatNumber(dimensions.geometryAssociation.incorrect, 0)} ошибок · цель 90%`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Freshness",
|
||||||
|
value: percent(dimensions.freshness.accuracy),
|
||||||
|
hint: `${formatNumber(dimensions.freshness.incorrect, 0)} ошибок · цель 90%`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Учёт / false free",
|
||||||
|
value: `${percent(metrics.accountingFraction)} / ${formatNumber(metrics.falseFreeClaims, 0)}`,
|
||||||
|
hint: `${formatNumber(metrics.terminalOutcomes, 0)} terminal outcomes`,
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
conclusion={{
|
||||||
|
proved: `На sealed validation текущая логика честно закрывает freshness с ${percent(dimensions.freshness.accuracy)}, сохраняет 100% учёта и не публикует false-free claims.`,
|
||||||
|
notProved: `Presence (${percent(dimensions.presence.accuracy)}) и geometry association (${percent(dimensions.geometryAssociation.accuracy)}) не достигли 90%; зафиксировано ${formatNumber(metrics.highSeverityFailures, 0)} high-severity ошибок. R1 и выпуск не приняты.`,
|
||||||
|
decision: "Сохранить E38 как первый измеренный baseline. Следующую итерацию R1 направить на detector/background decision и различение class-bearing объекта от независимой geometry-only среды, не меняя validation.",
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -70,6 +70,7 @@ const EMPTY_ADVANCED_RESULTS: AdvancedLaboratoryResults = {
|
|||||||
e34: null,
|
e34: null,
|
||||||
e35: null,
|
e35: null,
|
||||||
e37: null,
|
e37: null,
|
||||||
|
e38: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
function laboratoryWorkOrdinal(value: string): number {
|
function laboratoryWorkOrdinal(value: string): number {
|
||||||
|
|||||||
@@ -355,6 +355,83 @@ function e37() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function e38Dimension({
|
||||||
|
accuracy,
|
||||||
|
correct,
|
||||||
|
incorrect,
|
||||||
|
passed,
|
||||||
|
}) {
|
||||||
|
return {
|
||||||
|
correct,
|
||||||
|
incorrect,
|
||||||
|
total: 146,
|
||||||
|
accuracy,
|
||||||
|
target: 0.9,
|
||||||
|
passed,
|
||||||
|
by_stratum: {
|
||||||
|
agree: {
|
||||||
|
correct: 25,
|
||||||
|
incorrect: 4,
|
||||||
|
total: 29,
|
||||||
|
accuracy: 25 / 29,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
confusion: [{
|
||||||
|
reference: "object-present",
|
||||||
|
prediction: "object-present",
|
||||||
|
count: correct,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function e38() {
|
||||||
|
return {
|
||||||
|
result_id: `e38-perception-baseline-${"8".repeat(64)}`,
|
||||||
|
created_at_utc: "2026-07-28T02:50:00Z",
|
||||||
|
source_session_id: "20260720T065719Z_viewer_live",
|
||||||
|
source_display_name: "RAVNOVES00",
|
||||||
|
status: "measured-r1-source-scoped-baseline",
|
||||||
|
profile_id: "e38-ravnoves00-r1-development-cart/v1",
|
||||||
|
worker_node: "DESKTOP-OPJ8J04",
|
||||||
|
quality_gate_passed: false,
|
||||||
|
metrics: {
|
||||||
|
development_items: 340,
|
||||||
|
validation_items: 146,
|
||||||
|
terminal_outcomes: 146,
|
||||||
|
accounting_fraction: 1,
|
||||||
|
false_free_claims: 0,
|
||||||
|
high_severity_failures: 14,
|
||||||
|
dimensions: {
|
||||||
|
presence: e38Dimension({
|
||||||
|
accuracy: 0.821918,
|
||||||
|
correct: 120,
|
||||||
|
incorrect: 26,
|
||||||
|
passed: false,
|
||||||
|
}),
|
||||||
|
geometry_association: e38Dimension({
|
||||||
|
accuracy: 0.815068,
|
||||||
|
correct: 119,
|
||||||
|
incorrect: 27,
|
||||||
|
passed: false,
|
||||||
|
}),
|
||||||
|
freshness: e38Dimension({
|
||||||
|
accuracy: 0.952055,
|
||||||
|
correct: 139,
|
||||||
|
incorrect: 7,
|
||||||
|
passed: true,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
blocking_checks: [
|
||||||
|
"presence_target_reached",
|
||||||
|
"geometry_association_target_reached",
|
||||||
|
],
|
||||||
|
limitations: ["source-scoped"],
|
||||||
|
authority,
|
||||||
|
access: "read-only",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
server = await createServer({
|
server = await createServer({
|
||||||
appType: "custom",
|
appType: "custom",
|
||||||
@@ -371,9 +448,9 @@ after(async () => {
|
|||||||
await server?.close();
|
await server?.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("decodes E31–E37 from separate read-only catalogs", async () => {
|
test("decodes E31–E38 from separate read-only catalogs", async () => {
|
||||||
const requests = [];
|
const requests = [];
|
||||||
const items = [e31(), e32(), e33(), e34(), e35(), e37()];
|
const items = [e31(), e32(), e33(), e34(), e35(), e37(), e38()];
|
||||||
const decoded = await fetchAdvancedLaboratoryResults({
|
const decoded = await fetchAdvancedLaboratoryResults({
|
||||||
fetcher: async (input, init) => {
|
fetcher: async (input, init) => {
|
||||||
requests.push({ input: String(input), method: init?.method });
|
requests.push({ input: String(input), method: init?.method });
|
||||||
@@ -396,6 +473,9 @@ test("decodes E31–E37 from separate read-only catalogs", async () => {
|
|||||||
assert.equal(decoded.e37.metrics.reviewedItems, 486);
|
assert.equal(decoded.e37.metrics.reviewedItems, 486);
|
||||||
assert.equal(decoded.e37.metrics.validationItems, 146);
|
assert.equal(decoded.e37.metrics.validationItems, 146);
|
||||||
assert.equal(decoded.e37.qualityTargetEvaluated, false);
|
assert.equal(decoded.e37.qualityTargetEvaluated, false);
|
||||||
|
assert.equal(decoded.e38.metrics.dimensions.presence.accuracy, 0.821918);
|
||||||
|
assert.equal(decoded.e38.metrics.dimensions.freshness.passed, true);
|
||||||
|
assert.equal(decoded.e38.qualityGatePassed, false);
|
||||||
assert.deepEqual(requests, [
|
assert.deepEqual(requests, [
|
||||||
{ input: "/api/v1/laboratory/e31/results?limit=1", method: "GET" },
|
{ input: "/api/v1/laboratory/e31/results?limit=1", method: "GET" },
|
||||||
{ input: "/api/v1/laboratory/e32/results?limit=1", method: "GET" },
|
{ input: "/api/v1/laboratory/e32/results?limit=1", method: "GET" },
|
||||||
@@ -403,6 +483,7 @@ test("decodes E31–E37 from separate read-only catalogs", async () => {
|
|||||||
{ input: "/api/v1/laboratory/e34/results?limit=1", method: "GET" },
|
{ input: "/api/v1/laboratory/e34/results?limit=1", method: "GET" },
|
||||||
{ input: "/api/v1/laboratory/e35/results?limit=1", method: "GET" },
|
{ input: "/api/v1/laboratory/e35/results?limit=1", method: "GET" },
|
||||||
{ input: "/api/v1/laboratory/e37/results?limit=1", method: "GET" },
|
{ input: "/api/v1/laboratory/e37/results?limit=1", method: "GET" },
|
||||||
|
{ input: "/api/v1/laboratory/e38/results?limit=1", method: "GET" },
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -418,7 +499,8 @@ test("rejects authority escalation in an accepted-looking result", async () => {
|
|||||||
: String(input).includes("/e32/") ? e32()
|
: String(input).includes("/e32/") ? e32()
|
||||||
: String(input).includes("/e33/") ? e33()
|
: String(input).includes("/e33/") ? e33()
|
||||||
: String(input).includes("/e34/") ? e34()
|
: String(input).includes("/e34/") ? e34()
|
||||||
: String(input).includes("/e35/") ? e35() : e37(),
|
: String(input).includes("/e35/") ? e35()
|
||||||
|
: String(input).includes("/e37/") ? e37() : e38(),
|
||||||
)), { status: 200 }),
|
)), { status: 200 }),
|
||||||
}),
|
}),
|
||||||
AdvancedLaboratoryContractError,
|
AdvancedLaboratoryContractError,
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ const e35ResultUrl = new URL(
|
|||||||
"../src/workspaces/laboratory/E35Result.tsx",
|
"../src/workspaces/laboratory/E35Result.tsx",
|
||||||
import.meta.url,
|
import.meta.url,
|
||||||
);
|
);
|
||||||
|
const e38ResultUrl = new URL(
|
||||||
|
"../src/workspaces/laboratory/E38Result.tsx",
|
||||||
|
import.meta.url,
|
||||||
|
);
|
||||||
const e35StylesUrl = new URL(
|
const e35StylesUrl = new URL(
|
||||||
"../src/styles/e35-degradation-recovery.css",
|
"../src/styles/e35-degradation-recovery.css",
|
||||||
import.meta.url,
|
import.meta.url,
|
||||||
@@ -217,6 +221,24 @@ test("E35 extends the canonical LAB with fault and recovery evidence", async ()
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("E38 reports the frozen R1 baseline through the canonical LAB anatomy", async () => {
|
||||||
|
const [e38Source, advancedSource] = await Promise.all([
|
||||||
|
readFile(e38ResultUrl, "utf8"),
|
||||||
|
readFile(advancedLaboratoryResultUrl, "utf8"),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.match(e38Source, /<LaboratoryWorkTemplate/);
|
||||||
|
assert.match(e38Source, /<LaboratoryEvidence/);
|
||||||
|
assert.match(e38Source, /<LaboratoryResultSummary/);
|
||||||
|
assert.match(e38Source, /validation использован только для итоговой оценки/);
|
||||||
|
assert.match(e38Source, /Presence —/);
|
||||||
|
assert.match(e38Source, /R1 и выпуск не приняты/);
|
||||||
|
assert.match(e38Source, /не меняя validation/);
|
||||||
|
assert.doesNotMatch(e38Source, /className="laboratory-(?:summary|result-summary)"/);
|
||||||
|
assert.match(advancedSource, /id: "e38-perception-baseline"/);
|
||||||
|
assert.match(advancedSource, /<E38Result/);
|
||||||
|
});
|
||||||
|
|
||||||
test("the primary point-cloud viewer restores from fullscreen on Escape", async () => {
|
test("the primary point-cloud viewer restores from fullscreen on Escape", async () => {
|
||||||
const workspacesSource = await readFile(workspacesUrl, "utf8");
|
const workspacesSource = await readFile(workspacesUrl, "utf8");
|
||||||
|
|
||||||
|
|||||||
@@ -239,6 +239,15 @@ mount identity is bound to that source session. A9 transfer therefore remains
|
|||||||
unexecuted without publishing an empty LAB. ADR 0029 retains the audit.
|
unexecuted without publishing an empty LAB. ADR 0029 retains the audit.
|
||||||
ADR 0030 now defers TEST007/E36 and makes RAVNOVES00 source-scoped quality and
|
ADR 0030 now defers TEST007/E36 and makes RAVNOVES00 source-scoped quality and
|
||||||
product closure the active critical path.
|
product closure the active critical path.
|
||||||
|
R0 is now closed by E37 immutable result
|
||||||
|
`e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344`.
|
||||||
|
The first R1 baseline is E38 immutable Worker 006 result
|
||||||
|
`e38-perception-baseline-a272f82988cd9a7e071fad94c3e9fb49daf804fdcca523f853445fd3113a62b1`.
|
||||||
|
Its sealed 146-item validation measures presence at 82.2%, geometry
|
||||||
|
association at 81.5% and freshness at 95.2%, with 100% accounting, zero
|
||||||
|
false-free claims and 14 high-severity failures. R1 therefore remains open;
|
||||||
|
the next bounded iteration targets detector/background presence and
|
||||||
|
geometry-only association without changing the E37 validation set.
|
||||||
|
|
||||||
- [x] Reproduce all 4,489 immutable E29 frames with the exact frozen profile
|
- [x] Reproduce all 4,489 immutable E29 frames with the exact frozen profile
|
||||||
before applying E31/E30 changes.
|
before applying E31/E30 changes.
|
||||||
@@ -256,7 +265,7 @@ product closure the active critical path.
|
|||||||
contract without creating an empty LAB.
|
contract without creating an empty LAB.
|
||||||
- [x] Defer TEST007, E36 and new collection until the RAVNOVES00 reference
|
- [x] Defer TEST007, E36 and new collection until the RAVNOVES00 reference
|
||||||
release candidate is accepted.
|
release candidate is accepted.
|
||||||
- [ ] Freeze the RAVNOVES00 task ontology, reviewed denominator and
|
- [x] Freeze the RAVNOVES00 task ontology, reviewed denominator and
|
||||||
development/validation split.
|
development/validation split.
|
||||||
- [ ] Reach `>= 90%` separately for presence, geometry association and
|
- [ ] Reach `>= 90%` separately for presence, geometry association and
|
||||||
freshness decisions while retaining complete accounting and zero false-free
|
freshness decisions while retaining complete accounting and zero false-free
|
||||||
|
|||||||
@@ -61,11 +61,28 @@ failures remain blocking even when an aggregate percentage passes.
|
|||||||
|
|
||||||
R0–R4 are the active path. R5 is intentionally deferred.
|
R0–R4 are the active path. R5 is intentionally deferred.
|
||||||
|
|
||||||
|
## Current evidence
|
||||||
|
|
||||||
|
R0 is closed by immutable result
|
||||||
|
`e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344`.
|
||||||
|
It freezes 486 reviewed RAVNOVES00 items as 340 development and 146 sealed
|
||||||
|
validation cases, with separate presence, geometry-association and freshness
|
||||||
|
references.
|
||||||
|
|
||||||
|
The first R1 measurement is immutable result
|
||||||
|
`e38-perception-baseline-a272f82988cd9a7e071fad94c3e9fb49daf804fdcca523f853445fd3113a62b1`.
|
||||||
|
It was trained only on the development split and executed on Worker 006.
|
||||||
|
Freshness passes at 95.2%; evidence accounting is 100% and false-free claims
|
||||||
|
remain zero. Presence is 82.2%, geometry association is 81.5% and 14
|
||||||
|
high-severity validation cases fail, so R1 is not accepted. The next R1
|
||||||
|
iteration is bounded to detector/background presence, geometry-only
|
||||||
|
association and the camera-only freshness tail without changing validation.
|
||||||
|
|
||||||
## Experiment rules
|
## Experiment rules
|
||||||
|
|
||||||
1. RAVNOVES00 remains immutable and is never relabelled as a LAB result.
|
1. RAVNOVES00 remains immutable and is never relabelled as a LAB result.
|
||||||
2. Every iteration creates a new LAB identity, method profile and conclusion.
|
2. Every iteration creates a new LAB identity, method profile and conclusion.
|
||||||
3. Existing E29–E35 results remain historical evidence and are not overwritten.
|
3. Existing LAB results remain historical evidence and are not overwritten.
|
||||||
4. Metrics are computed from server-owned artifacts; the UI does not invent
|
4. Metrics are computed from server-owned artifacts; the UI does not invent
|
||||||
acceptance state.
|
acceptance state.
|
||||||
5. The compact LAB summary explains the task, method, relevant models,
|
5. The compact LAB summary explains the task, method, relevant models,
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
# LAB E38 — RAVNOVES00 R1 perception-quality baseline
|
||||||
|
|
||||||
|
Date: 2026-07-28
|
||||||
|
|
||||||
|
Status: completed baseline; R1 quality gate not passed
|
||||||
|
|
||||||
|
## Why this LAB exists
|
||||||
|
|
||||||
|
E37 froze the RAVNOVES00 acceptance ontology, reviewed denominator and
|
||||||
|
development/validation split. E38 is the first planned R1 measurement against
|
||||||
|
that contract. It answers three separate questions:
|
||||||
|
|
||||||
|
1. Does the current source-scoped evidence support that a task-relevant object
|
||||||
|
or occupied environment is present?
|
||||||
|
2. Is current geometry associated with the correct object, rejected as
|
||||||
|
non-object support, or retained as an independent occupied component?
|
||||||
|
3. Is evidence freshness represented as current, stale or unavailable?
|
||||||
|
|
||||||
|
E38 does not evaluate navigation, planning, commands, safety or transfer to
|
||||||
|
another route or rig.
|
||||||
|
|
||||||
|
## Immutable inputs
|
||||||
|
|
||||||
|
| Input | Identity |
|
||||||
|
| --- | --- |
|
||||||
|
| Physical source | `20260720T065719Z_viewer_live` (`RAVNOVES00`) |
|
||||||
|
| Frozen E37 contract | `e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344` |
|
||||||
|
| E37 item digest | `c6e4f474f59867ce0cc86825193043245c227106400cc216a2dac0645dca54ab` |
|
||||||
|
| E30 materialization | `e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a` |
|
||||||
|
| E30 item digest | `827b498c7b7ab520cc3998e288134ebb5a2e8ebba0532d5cc3905e1a4460b0a7` |
|
||||||
|
| E38 profile | `e38-ravnoves00-r1-development-cart/v1` |
|
||||||
|
| Profile digest | `f34012fd6fedd2a8c3e5c6866cb2e57da9cb64bd6e798772da2c338b8a9b96c1` |
|
||||||
|
|
||||||
|
The E37 denominator contains 486 reviewed items: 340 development items and
|
||||||
|
146 sealed validation items. Validation labels are not used for fitting or
|
||||||
|
tree selection.
|
||||||
|
|
||||||
|
## Method
|
||||||
|
|
||||||
|
E38 joins each immutable E37 item with its immutable E30 materialization and
|
||||||
|
derives deterministic source features. Three dimension-specific shallow CART
|
||||||
|
models are trained only on the development partition:
|
||||||
|
|
||||||
|
- presence: maximum depth 5, minimum leaf 4;
|
||||||
|
- geometry association: maximum depth 5, minimum leaf 4;
|
||||||
|
- freshness: maximum depth 3, minimum leaf 6 and source-time features enabled.
|
||||||
|
|
||||||
|
Each validation item receives one terminal prediction for each dimension.
|
||||||
|
Targets are evaluated separately: presence `>= 90%`, geometry association
|
||||||
|
`>= 90%`, freshness `>= 90%`, accounting `100%`, false-free claims `0` and
|
||||||
|
high-severity failures `0`.
|
||||||
|
|
||||||
|
This is an engineering-labelled source-scoped baseline, not an independent
|
||||||
|
external ground-truth benchmark.
|
||||||
|
|
||||||
|
## Worker execution
|
||||||
|
|
||||||
|
| Property | Value |
|
||||||
|
| --- | --- |
|
||||||
|
| Worker | `DESKTOP-OPJ8J04` (`Worker 006`) |
|
||||||
|
| Worker package | `e38-worker-package-a8b2313e07bd4d1de04db86de7728cbc21f89ae5259b2ee0adfb297eea6974c3` |
|
||||||
|
| Container | `ndc-mission-core-e38-baseline` |
|
||||||
|
| Pinned image | `nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794` |
|
||||||
|
| Network | disabled |
|
||||||
|
| Limits | 2 CPU, 512 MiB memory, 128 PIDs |
|
||||||
|
| Filesystem | read-only runtime, explicit output mount, 64 MiB no-exec tmpfs |
|
||||||
|
| Authority | commands false; navigation/safety acceptance false |
|
||||||
|
|
||||||
|
The result was copied back from the Worker 006 D-volume and independently
|
||||||
|
validated against its manifest and artifact digests.
|
||||||
|
|
||||||
|
## Result
|
||||||
|
|
||||||
|
Official result:
|
||||||
|
|
||||||
|
`e38-perception-baseline-a272f82988cd9a7e071fad94c3e9fb49daf804fdcca523f853445fd3113a62b1`
|
||||||
|
|
||||||
|
| Dimension | Correct | Accuracy | Target | Gate |
|
||||||
|
| --- | ---: | ---: | ---: | --- |
|
||||||
|
| Presence | 120 / 146 | 82.2% | >= 90% | fail |
|
||||||
|
| Geometry association | 119 / 146 | 81.5% | >= 90% | fail |
|
||||||
|
| Freshness | 139 / 146 | 95.2% | >= 90% | pass |
|
||||||
|
| Evidence accounting | 146 / 146 | 100% | 100% | pass |
|
||||||
|
| False-free claims | 0 | — | 0 | pass |
|
||||||
|
| High-severity failures | 14 | — | 0 | fail |
|
||||||
|
|
||||||
|
The R1 quality gate is not passed.
|
||||||
|
|
||||||
|
### Weakest strata
|
||||||
|
|
||||||
|
- Presence is weakest on `geometry-only`: 25 / 38, or 65.8%.
|
||||||
|
- Geometry association is weakest on `geometry-only`: 26 / 38, or 68.4%.
|
||||||
|
- Freshness is weakest on `camera-only`: 34 / 39, or 87.2%.
|
||||||
|
|
||||||
|
Presence errors are dominated by background/noise being promoted to
|
||||||
|
object-present and real objects being reduced to occupied environment.
|
||||||
|
Geometry errors are dominated by object-associated evidence being retained as
|
||||||
|
independent occupied geometry.
|
||||||
|
|
||||||
|
## What E38 proves
|
||||||
|
|
||||||
|
- RAVNOVES00 now has a reproducible first R1 baseline over a frozen holdout.
|
||||||
|
- Freshness already exceeds the source-scoped 90% target.
|
||||||
|
- All 146 validation cases receive terminal outcomes.
|
||||||
|
- No missing or rejected evidence is converted into asserted free space.
|
||||||
|
- The exact worker, immutable package, input identities, model and predictions
|
||||||
|
are retained.
|
||||||
|
|
||||||
|
## What E38 does not prove
|
||||||
|
|
||||||
|
- Presence and geometry association have not reached 90%.
|
||||||
|
- Fourteen high-severity validation items remain blocking.
|
||||||
|
- E38 does not prove performance on another route, camera, mount or K1 unit.
|
||||||
|
- E38 grants no navigation, command, planning or safety authority.
|
||||||
|
|
||||||
|
## Decision and next iteration
|
||||||
|
|
||||||
|
Keep E38 unchanged as the first measured R1 baseline. The next R1 LAB must
|
||||||
|
focus on:
|
||||||
|
|
||||||
|
1. detector/background presence decisions;
|
||||||
|
2. separating class-bearing objects from independent `geometry-only`
|
||||||
|
occupied environment;
|
||||||
|
3. the camera-only freshness tail;
|
||||||
|
4. eliminating high-severity failures.
|
||||||
|
|
||||||
|
The 146 validation items remain frozen. Any model or rule change is developed
|
||||||
|
only on the 340 development items and is published as a new LAB identity
|
||||||
|
before the validation gate is evaluated again.
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"authority": {
|
||||||
|
"commands_enabled": false,
|
||||||
|
"navigation_or_safety_accepted": false
|
||||||
|
},
|
||||||
|
"model": {
|
||||||
|
"dimensions": {
|
||||||
|
"freshness": {
|
||||||
|
"include_source_time": true,
|
||||||
|
"max_depth": 3,
|
||||||
|
"min_leaf": 6
|
||||||
|
},
|
||||||
|
"geometry_association": {
|
||||||
|
"include_source_time": false,
|
||||||
|
"max_depth": 5,
|
||||||
|
"min_leaf": 4
|
||||||
|
},
|
||||||
|
"presence": {
|
||||||
|
"include_source_time": false,
|
||||||
|
"max_depth": 5,
|
||||||
|
"min_leaf": 4
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"type": "deterministic-shallow-cart"
|
||||||
|
},
|
||||||
|
"profile_id": "e38-ravnoves00-r1-development-cart/v1",
|
||||||
|
"schema_version": "missioncore.e38-perception-baseline-profile/v1",
|
||||||
|
"source": {
|
||||||
|
"acceptance_result_id": "e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344",
|
||||||
|
"display_name": "RAVNOVES00",
|
||||||
|
"materialization_id": "e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a",
|
||||||
|
"session_id": "20260720T065719Z_viewer_live"
|
||||||
|
},
|
||||||
|
"targets": {
|
||||||
|
"accounting_target": 1.0,
|
||||||
|
"freshness_target": 0.9,
|
||||||
|
"geometry_association_target": 0.9,
|
||||||
|
"maximum_false_free_claims": 0,
|
||||||
|
"presence_target": 0.9
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,292 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build a minimal immutable E38 package for Worker 006."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from k1link.compute.e38_perception_baseline import (
|
||||||
|
E38_PACKAGE_SCHEMA,
|
||||||
|
E38_PROFILE_SCHEMA,
|
||||||
|
)
|
||||||
|
|
||||||
|
_RUNTIME_FILES = {
|
||||||
|
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
|
||||||
|
"runtime/k1link/compute/__init__.py": None,
|
||||||
|
"runtime/k1link/compute/e37_acceptance_contract.py": (
|
||||||
|
"src/k1link/compute/e37_acceptance_contract.py"
|
||||||
|
),
|
||||||
|
"runtime/k1link/compute/e38_perception_baseline.py": (
|
||||||
|
"src/k1link/compute/e38_perception_baseline.py"
|
||||||
|
),
|
||||||
|
"runtime/run_e38_perception_baseline.py": (
|
||||||
|
"experiments/perception/worker/run_e38_perception_baseline.py"
|
||||||
|
),
|
||||||
|
"runtime/Invoke-E38PerceptionBaseline.ps1": (
|
||||||
|
"experiments/perception/worker/Invoke-E38PerceptionBaseline.ps1"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
_GENERATED_COMPUTE_INIT = (
|
||||||
|
'"""Minimal E38 worker projection; import contract modules explicitly."""\n'
|
||||||
|
)
|
||||||
|
_INPUT_FILES = {
|
||||||
|
"acceptance": (
|
||||||
|
"manifest.json",
|
||||||
|
"acceptance-items.jsonl",
|
||||||
|
"acceptance-contract.json",
|
||||||
|
"run-report.json",
|
||||||
|
),
|
||||||
|
"materialization": ("manifest.json", "materialized-items.jsonl"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class E38WorkerPackageError(RuntimeError):
|
||||||
|
"""The E38 package source or immutable package is invalid."""
|
||||||
|
|
||||||
|
|
||||||
|
def build_e38_worker_package(
|
||||||
|
*,
|
||||||
|
repository_root: Path,
|
||||||
|
acceptance_root: Path,
|
||||||
|
materialization_root: Path,
|
||||||
|
profile_path: Path,
|
||||||
|
output_root: Path,
|
||||||
|
) -> Path:
|
||||||
|
"""Build or verify one content-addressed E38 worker package."""
|
||||||
|
|
||||||
|
repository = repository_root.resolve(strict=True)
|
||||||
|
profile_source = profile_path.resolve(strict=True)
|
||||||
|
profile = _read_json(profile_source)
|
||||||
|
if profile.get("schema_version") != E38_PROFILE_SCHEMA:
|
||||||
|
raise E38WorkerPackageError("E38 package profile is incompatible")
|
||||||
|
roots = {
|
||||||
|
"acceptance": acceptance_root.resolve(strict=True),
|
||||||
|
"materialization": materialization_root.resolve(strict=True),
|
||||||
|
}
|
||||||
|
expected_ids = {
|
||||||
|
"acceptance": profile["source"]["acceptance_result_id"],
|
||||||
|
"materialization": profile["source"]["materialization_id"],
|
||||||
|
}
|
||||||
|
sources: dict[str, Path | None] = {}
|
||||||
|
for target, relative in _RUNTIME_FILES.items():
|
||||||
|
source = None if relative is None else repository / relative
|
||||||
|
if source is not None and (not source.is_file() or source.is_symlink()):
|
||||||
|
raise E38WorkerPackageError(f"E38 runtime source is invalid: {relative}")
|
||||||
|
sources[target] = source
|
||||||
|
sources["profile.json"] = profile_source
|
||||||
|
for kind, filenames in _INPUT_FILES.items():
|
||||||
|
root = roots[kind]
|
||||||
|
if root.name != expected_ids[kind]:
|
||||||
|
raise E38WorkerPackageError(f"E38 {kind} identity changed")
|
||||||
|
for filename in filenames:
|
||||||
|
source = root / filename
|
||||||
|
if not source.is_file() or source.is_symlink():
|
||||||
|
raise E38WorkerPackageError(f"E38 {kind} artifact is invalid")
|
||||||
|
sources[f"input/{kind}/{root.name}/{filename}"] = source
|
||||||
|
|
||||||
|
descriptors = []
|
||||||
|
for relative, source in sorted(sources.items()):
|
||||||
|
payload = (
|
||||||
|
_GENERATED_COMPUTE_INIT.encode()
|
||||||
|
if source is None
|
||||||
|
else source.read_bytes()
|
||||||
|
)
|
||||||
|
descriptors.append(
|
||||||
|
{
|
||||||
|
"path": relative,
|
||||||
|
"byte_length": len(payload),
|
||||||
|
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
identity = {
|
||||||
|
"schema_version": E38_PACKAGE_SCHEMA,
|
||||||
|
"classification": "immutable-ravnoves00-r1-worker-input",
|
||||||
|
"source_ids": expected_ids,
|
||||||
|
"profile_sha256": _sha256(profile_source),
|
||||||
|
"artifact_paths": [row["path"] for row in descriptors],
|
||||||
|
"source_artifacts": descriptors,
|
||||||
|
"authority": {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||||
|
package_id = f"e38-worker-package-{identity_sha256}"
|
||||||
|
output = output_root.expanduser().absolute()
|
||||||
|
output.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
destination = output / package_id
|
||||||
|
if destination.exists():
|
||||||
|
validate_e38_worker_package(destination)
|
||||||
|
return destination
|
||||||
|
|
||||||
|
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
|
||||||
|
staging.mkdir(mode=0o700, exist_ok=False)
|
||||||
|
try:
|
||||||
|
for relative, source in sources.items():
|
||||||
|
target = staging / relative
|
||||||
|
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
if source is None:
|
||||||
|
target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8")
|
||||||
|
else:
|
||||||
|
shutil.copyfile(source, target)
|
||||||
|
artifacts = [
|
||||||
|
{
|
||||||
|
"kind": relative,
|
||||||
|
"path": relative,
|
||||||
|
"byte_length": (staging / relative).stat().st_size,
|
||||||
|
"sha256": _sha256(staging / relative),
|
||||||
|
}
|
||||||
|
for relative in sorted(sources)
|
||||||
|
]
|
||||||
|
manifest = {
|
||||||
|
"schema_version": E38_PACKAGE_SCHEMA,
|
||||||
|
"package_id": package_id,
|
||||||
|
"identity_sha256": identity_sha256,
|
||||||
|
"identity": identity,
|
||||||
|
"created_at_utc": datetime.now(UTC)
|
||||||
|
.isoformat(timespec="milliseconds")
|
||||||
|
.replace("+00:00", "Z"),
|
||||||
|
"artifacts": artifacts,
|
||||||
|
}
|
||||||
|
_write_json(staging / "manifest.json", manifest)
|
||||||
|
validate_e38_worker_package(staging, allow_staging=True)
|
||||||
|
os.replace(staging, destination)
|
||||||
|
except BaseException:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
raise
|
||||||
|
validate_e38_worker_package(destination)
|
||||||
|
return destination
|
||||||
|
|
||||||
|
|
||||||
|
def validate_e38_worker_package(
|
||||||
|
root: Path,
|
||||||
|
*,
|
||||||
|
allow_staging: bool = False,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Validate package identity, exact file set, and every member digest."""
|
||||||
|
|
||||||
|
resolved = root.resolve(strict=True)
|
||||||
|
manifest = _read_json(resolved / "manifest.json")
|
||||||
|
identity = manifest.get("identity")
|
||||||
|
identity_sha256 = manifest.get("identity_sha256")
|
||||||
|
package_id = manifest.get("package_id")
|
||||||
|
artifacts = manifest.get("artifacts")
|
||||||
|
expected_name = (
|
||||||
|
isinstance(package_id, str)
|
||||||
|
and (
|
||||||
|
resolved.name == package_id
|
||||||
|
or (
|
||||||
|
allow_staging
|
||||||
|
and resolved.name.startswith(f".{package_id}.")
|
||||||
|
and resolved.name.endswith(".tmp")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
manifest.get("schema_version") != E38_PACKAGE_SCHEMA
|
||||||
|
or not isinstance(identity, dict)
|
||||||
|
or not isinstance(identity_sha256, str)
|
||||||
|
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||||
|
or package_id != f"e38-worker-package-{identity_sha256}"
|
||||||
|
or not expected_name
|
||||||
|
or not isinstance(artifacts, list)
|
||||||
|
):
|
||||||
|
raise E38WorkerPackageError("E38 worker package identity is invalid")
|
||||||
|
expected_paths = set(identity.get("artifact_paths", []))
|
||||||
|
actual_paths = {
|
||||||
|
path.relative_to(resolved).as_posix()
|
||||||
|
for path in resolved.rglob("*")
|
||||||
|
if path.is_file()
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
not expected_paths
|
||||||
|
or actual_paths != expected_paths | {"manifest.json"}
|
||||||
|
or len(artifacts) != len(expected_paths)
|
||||||
|
):
|
||||||
|
raise E38WorkerPackageError("E38 worker package file set changed")
|
||||||
|
observed: set[str] = set()
|
||||||
|
for row in artifacts:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
raise E38WorkerPackageError("E38 worker package artifact is invalid")
|
||||||
|
relative = row.get("path")
|
||||||
|
path = resolved / str(relative)
|
||||||
|
if (
|
||||||
|
not isinstance(relative, str)
|
||||||
|
or relative not in expected_paths
|
||||||
|
or relative in observed
|
||||||
|
or Path(relative).is_absolute()
|
||||||
|
or ".." in Path(relative).parts
|
||||||
|
or not path.is_file()
|
||||||
|
or path.is_symlink()
|
||||||
|
or row.get("kind") != relative
|
||||||
|
or row.get("byte_length") != path.stat().st_size
|
||||||
|
or row.get("sha256") != _sha256(path)
|
||||||
|
):
|
||||||
|
raise E38WorkerPackageError("E38 worker package artifact changed")
|
||||||
|
observed.add(relative)
|
||||||
|
if observed != expected_paths:
|
||||||
|
raise E38WorkerPackageError("E38 worker package coverage changed")
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
while chunk := stream.read(1024 * 1024):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_json(path: Path) -> dict[str, Any]:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise E38WorkerPackageError(f"JSON object expected: {path.name}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _write_json(path: Path, value: object) -> None:
|
||||||
|
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||||
|
json.dump(value, stream, indent=2, sort_keys=True)
|
||||||
|
stream.write("\n")
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--repository-root", type=Path, required=True)
|
||||||
|
parser.add_argument("--acceptance", type=Path, required=True)
|
||||||
|
parser.add_argument("--materialization", type=Path, required=True)
|
||||||
|
parser.add_argument("--profile", type=Path, required=True)
|
||||||
|
parser.add_argument("--output-root", type=Path, required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
package = build_e38_worker_package(
|
||||||
|
repository_root=args.repository_root,
|
||||||
|
acceptance_root=args.acceptance,
|
||||||
|
materialization_root=args.materialization,
|
||||||
|
profile_path=args.profile,
|
||||||
|
output_root=args.output_root,
|
||||||
|
)
|
||||||
|
print(package)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build the first RAVNOVES00 R1 perception-quality baseline."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from k1link.compute.e38_perception_baseline import (
|
||||||
|
build_e38_perception_baseline,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--acceptance", type=Path, required=True)
|
||||||
|
parser.add_argument("--materialization", type=Path, required=True)
|
||||||
|
parser.add_argument("--profile", type=Path, required=True)
|
||||||
|
parser.add_argument("--output-root", type=Path, required=True)
|
||||||
|
parser.add_argument("--worker-node", default=os.environ.get("COMPUTERNAME"))
|
||||||
|
args = parser.parse_args()
|
||||||
|
result = build_e38_perception_baseline(
|
||||||
|
acceptance_root=args.acceptance,
|
||||||
|
materialization_root=args.materialization,
|
||||||
|
profile_path=args.profile,
|
||||||
|
output_root=args.output_root,
|
||||||
|
worker_node=args.worker_node,
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"result_id": result.result_id,
|
||||||
|
"result_root": str(result.result_root),
|
||||||
|
"quality_gate_passed": result.quality_gate_passed,
|
||||||
|
"metrics": result.report["metrics"],
|
||||||
|
"blocking_checks": result.report["quality_gate"][
|
||||||
|
"blocking_checks"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[string]$PackageRoot,
|
||||||
|
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\e38-baseline",
|
||||||
|
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||||
|
[ValidateRange(1, 1000)]
|
||||||
|
[int]$FreeGiBFloor = 300
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$ProgressPreference = "SilentlyContinue"
|
||||||
|
|
||||||
|
function Assert-LastExitCode([string]$Operation) {
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||||
|
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||||
|
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||||
|
if (
|
||||||
|
-not $item.PSIsContainer -or
|
||||||
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||||
|
$root -ine "D:"
|
||||||
|
) {
|
||||||
|
throw "$Label must be a real D: directory"
|
||||||
|
}
|
||||||
|
return $item.FullName
|
||||||
|
}
|
||||||
|
|
||||||
|
function Convert-ToDockerPath([string]$Path) {
|
||||||
|
return $Path.Replace("\", "/")
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-FreeSpace([string]$Phase) {
|
||||||
|
$free = [int64](Get-PSDrive -Name D).Free
|
||||||
|
$floor = [int64]$FreeGiBFloor * 1GB
|
||||||
|
Write-Host (
|
||||||
|
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
|
||||||
|
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
|
||||||
|
)
|
||||||
|
if ($free -lt ($floor + 1GB)) {
|
||||||
|
throw "D: lacks the guarded E38 reserve during $Phase"
|
||||||
|
}
|
||||||
|
return $free
|
||||||
|
}
|
||||||
|
|
||||||
|
$package = Resolve-DDirectory $PackageRoot "E38 package"
|
||||||
|
$packageManifestPath = Join-Path $package "manifest.json"
|
||||||
|
if (-not (Test-Path -LiteralPath $packageManifestPath -PathType Leaf)) {
|
||||||
|
throw "E38 package manifest is missing"
|
||||||
|
}
|
||||||
|
$packageManifest = Get-Content -LiteralPath $packageManifestPath -Raw |
|
||||||
|
ConvertFrom-Json
|
||||||
|
if (
|
||||||
|
$packageManifest.schema_version -ne "missioncore.e38-worker-package/v1" -or
|
||||||
|
$packageManifest.package_id -ne (Split-Path $package -Leaf) -or
|
||||||
|
$packageManifest.package_id -notmatch "^e38-worker-package-[a-f0-9]{64}$"
|
||||||
|
) {
|
||||||
|
throw "E38 package manifest is incompatible"
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not (Test-Path -LiteralPath $OutputRoot)) {
|
||||||
|
$null = New-Item -ItemType Directory -Path $OutputRoot
|
||||||
|
}
|
||||||
|
$output = Resolve-DDirectory $OutputRoot "E38 output root"
|
||||||
|
$freeBefore = Assert-FreeSpace "preflight"
|
||||||
|
|
||||||
|
& docker image inspect $ContainerImage *> $null
|
||||||
|
Assert-LastExitCode "Pinned E38 container image inspection"
|
||||||
|
|
||||||
|
$dockerPackage = Convert-ToDockerPath $package
|
||||||
|
$dockerOutput = Convert-ToDockerPath $output
|
||||||
|
$packageName = Split-Path $package -Leaf
|
||||||
|
$containerPackage = "/opt/e38-input/$packageName"
|
||||||
|
$command = @(
|
||||||
|
"run", "--rm",
|
||||||
|
"--name", "ndc-mission-core-e38-baseline",
|
||||||
|
"--network", "none",
|
||||||
|
"--read-only",
|
||||||
|
"--security-opt", "no-new-privileges:true",
|
||||||
|
"--cap-drop", "ALL",
|
||||||
|
"--pids-limit", "128",
|
||||||
|
"--memory", "512m",
|
||||||
|
"--memory-swap", "512m",
|
||||||
|
"--cpus", "2",
|
||||||
|
"--tmpfs", "/tmp:rw,noexec,nosuid,size=64m",
|
||||||
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||||
|
"-e", ("PYTHONPATH={0}/runtime" -f $containerPackage),
|
||||||
|
"-e", ("E38_WORKER_NODE={0}" -f $env:COMPUTERNAME),
|
||||||
|
"-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage),
|
||||||
|
"-v", ("{0}:/output:rw" -f $dockerOutput),
|
||||||
|
"--entrypoint", "python3",
|
||||||
|
$ContainerImage,
|
||||||
|
("{0}/runtime/run_e38_perception_baseline.py" -f $containerPackage),
|
||||||
|
"--package", $containerPackage,
|
||||||
|
"--output-root", "/output"
|
||||||
|
)
|
||||||
|
|
||||||
|
Write-Output ("PACKAGE_ID={0}" -f $packageManifest.package_id)
|
||||||
|
Write-Output ("PACKAGE_IDENTITY_SHA256={0}" -f $packageManifest.identity_sha256)
|
||||||
|
Write-Output ("CONTAINER_IMAGE={0}" -f $ContainerImage)
|
||||||
|
& docker @command
|
||||||
|
Assert-LastExitCode "E38 perception baseline"
|
||||||
|
|
||||||
|
$matches = @(
|
||||||
|
Get-ChildItem -LiteralPath $output -Directory -Filter "e38-perception-baseline-*" |
|
||||||
|
Where-Object {
|
||||||
|
$manifestPath = Join-Path $_.FullName "manifest.json"
|
||||||
|
if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
$manifest = Get-Content -LiteralPath $manifestPath -Raw |
|
||||||
|
ConvertFrom-Json
|
||||||
|
return (
|
||||||
|
$manifest.schema_version -eq
|
||||||
|
"missioncore.e38-perception-baseline/v1" -and
|
||||||
|
$manifest.acceptance_state -eq
|
||||||
|
"completed-r1-source-scoped-baseline" -and
|
||||||
|
$manifest.identity.execution.worker_node -eq $env:COMPUTERNAME
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if ($matches.Count -ne 1) {
|
||||||
|
throw "E38 immutable result could not be resolved uniquely"
|
||||||
|
}
|
||||||
|
$resultRoot = $matches[0].FullName
|
||||||
|
$resultManifest = Get-Content -LiteralPath (
|
||||||
|
Join-Path $resultRoot "manifest.json"
|
||||||
|
) -Raw | ConvertFrom-Json
|
||||||
|
$freeAfter = Assert-FreeSpace "completed"
|
||||||
|
Write-Output ("RESULT_ROOT={0}" -f $resultRoot)
|
||||||
|
Write-Output ("RESULT_ID={0}" -f $resultManifest.result_id)
|
||||||
|
Write-Output ("QUALITY_GATE_PASSED={0}" -f $resultManifest.quality_gate_passed)
|
||||||
|
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||||
|
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Execute a packaged E38 baseline inside the pinned Worker 006 container."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from k1link.compute.e38_perception_baseline import (
|
||||||
|
build_e38_perception_baseline,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--package", type=Path, required=True)
|
||||||
|
parser.add_argument("--output-root", type=Path, required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
package = args.package.resolve(strict=True)
|
||||||
|
manifest = json.loads((package / "manifest.json").read_text(encoding="utf-8"))
|
||||||
|
source = manifest["identity"]["source_ids"]
|
||||||
|
result = build_e38_perception_baseline(
|
||||||
|
acceptance_root=package / "input" / "acceptance" / source["acceptance"],
|
||||||
|
materialization_root=(
|
||||||
|
package / "input" / "materialization" / source["materialization"]
|
||||||
|
),
|
||||||
|
profile_path=package / "profile.json",
|
||||||
|
output_root=args.output_root,
|
||||||
|
worker_node=os.environ.get("E38_WORKER_NODE"),
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"result_id": result.result_id,
|
||||||
|
"quality_gate_passed": result.quality_gate_passed,
|
||||||
|
"metrics": result.report["metrics"],
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,830 @@
|
|||||||
|
"""Development-trained RAVNOVES00 R1 perception-quality baseline.
|
||||||
|
|
||||||
|
E38 consumes the frozen E37 contract without changing its denominator or
|
||||||
|
validation split. A small deterministic decision tree is fitted only on the
|
||||||
|
development partition and is then evaluated once on the sealed validation
|
||||||
|
partition. The result remains source-scoped and diagnostic.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import uuid
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Final
|
||||||
|
|
||||||
|
from k1link.compute.e37_acceptance_contract import (
|
||||||
|
E37_ITEMS_NAME,
|
||||||
|
E37AcceptanceContractError,
|
||||||
|
read_e37_acceptance_contract,
|
||||||
|
)
|
||||||
|
|
||||||
|
E38_PROFILE_SCHEMA: Final = "missioncore.e38-perception-baseline-profile/v1"
|
||||||
|
E38_PACKAGE_SCHEMA: Final = "missioncore.e38-worker-package/v1"
|
||||||
|
E38_RESULT_SCHEMA: Final = "missioncore.e38-perception-baseline/v1"
|
||||||
|
E38_PREDICTION_SCHEMA: Final = "missioncore.e38-perception-prediction/v1"
|
||||||
|
E38_MODEL_SCHEMA: Final = "missioncore.e38-development-cart-model/v1"
|
||||||
|
E38_REPORT_SCHEMA: Final = "missioncore.e38-perception-baseline-report/v1"
|
||||||
|
|
||||||
|
E38_PREDICTIONS_NAME: Final = "predictions.jsonl"
|
||||||
|
E38_MODEL_NAME: Final = "development-model.json"
|
||||||
|
E38_REPORT_NAME: Final = "run-report.json"
|
||||||
|
E38_MANIFEST_NAME: Final = "manifest.json"
|
||||||
|
|
||||||
|
_MATERIALIZATION_SCHEMA: Final = "missioncore.e30-evidence-materialization/v2"
|
||||||
|
_SOURCE_SESSION_ID: Final = "20260720T065719Z_viewer_live"
|
||||||
|
_SOURCE_DISPLAY_NAME: Final = "RAVNOVES00"
|
||||||
|
_DIMENSIONS: Final = ("presence", "geometry_association", "freshness")
|
||||||
|
_CATEGORICAL_FEATURES: Final = {
|
||||||
|
"stratum": ("agree", "camera-only", "conflict", "geometry-only", "unknown"),
|
||||||
|
"range": ("near", "middle", "far", "unavailable"),
|
||||||
|
"geometry": (
|
||||||
|
"agree",
|
||||||
|
"conflict",
|
||||||
|
"single-source-camera",
|
||||||
|
"single-source-geometry",
|
||||||
|
"unknown",
|
||||||
|
"unavailable",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
_AUTHORITY: Final = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class E38PerceptionBaselineError(RuntimeError):
|
||||||
|
"""The E38 profile, immutable inputs, or result is invalid."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class E38PerceptionBaseline:
|
||||||
|
result_id: str
|
||||||
|
result_root: Path
|
||||||
|
manifest: dict[str, Any]
|
||||||
|
report: dict[str, Any]
|
||||||
|
model: dict[str, Any]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def quality_gate_passed(self) -> bool:
|
||||||
|
return self.report.get("quality_gate", {}).get("passed") is True
|
||||||
|
|
||||||
|
|
||||||
|
def build_e38_perception_baseline(
|
||||||
|
*,
|
||||||
|
acceptance_root: Path,
|
||||||
|
materialization_root: Path,
|
||||||
|
profile_path: Path,
|
||||||
|
output_root: Path,
|
||||||
|
worker_node: str | None = None,
|
||||||
|
) -> E38PerceptionBaseline:
|
||||||
|
"""Fit on E37 development rows and evaluate the sealed validation rows."""
|
||||||
|
|
||||||
|
profile_file = profile_path.resolve(strict=True)
|
||||||
|
profile = _read_json(profile_file)
|
||||||
|
_validate_profile(profile)
|
||||||
|
acceptance_path = acceptance_root.resolve(strict=True)
|
||||||
|
materialization_path = materialization_root.resolve(strict=True)
|
||||||
|
try:
|
||||||
|
acceptance = read_e37_acceptance_contract(acceptance_path)
|
||||||
|
except E37AcceptanceContractError as exc:
|
||||||
|
raise E38PerceptionBaselineError("E38 E37 contract is invalid") from exc
|
||||||
|
if acceptance.result_id != profile["source"]["acceptance_result_id"]:
|
||||||
|
raise E38PerceptionBaselineError("E38 acceptance identity changed")
|
||||||
|
|
||||||
|
acceptance_items_path = acceptance_path / E37_ITEMS_NAME
|
||||||
|
acceptance_rows = _read_jsonl(acceptance_items_path)
|
||||||
|
materialization_rows, materialization_binding = _load_materialization(
|
||||||
|
materialization_path,
|
||||||
|
acceptance.manifest,
|
||||||
|
profile,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
len(acceptance_rows) != 486
|
||||||
|
or len(materialization_rows) != 486
|
||||||
|
or {row.get("item_id") for row in acceptance_rows}
|
||||||
|
!= {row.get("item_id") for row in materialization_rows}
|
||||||
|
):
|
||||||
|
raise E38PerceptionBaselineError("E38 denominator accounting differs")
|
||||||
|
|
||||||
|
identity = {
|
||||||
|
"schema_version": E38_RESULT_SCHEMA,
|
||||||
|
"source": {
|
||||||
|
"session_id": _SOURCE_SESSION_ID,
|
||||||
|
"display_name": _SOURCE_DISPLAY_NAME,
|
||||||
|
"acceptance_result_id": acceptance.result_id,
|
||||||
|
"acceptance_identity_sha256": acceptance.manifest["identity_sha256"],
|
||||||
|
"acceptance_items_sha256": _sha256(acceptance_items_path),
|
||||||
|
**materialization_binding,
|
||||||
|
},
|
||||||
|
"profile": {
|
||||||
|
"profile_id": profile["profile_id"],
|
||||||
|
"sha256": _sha256(profile_file),
|
||||||
|
},
|
||||||
|
"execution": {
|
||||||
|
"class": "deterministic-development-trained-validation-evaluation",
|
||||||
|
"worker_node": worker_node or "unbound-local",
|
||||||
|
},
|
||||||
|
"authority": _AUTHORITY,
|
||||||
|
}
|
||||||
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||||
|
result_id = f"e38-perception-baseline-{identity_sha256}"
|
||||||
|
destination = output_root.expanduser().absolute() / result_id
|
||||||
|
if destination.exists():
|
||||||
|
return read_e38_perception_baseline(destination)
|
||||||
|
|
||||||
|
materialization_by_id = {
|
||||||
|
str(row["item_id"]): row for row in materialization_rows
|
||||||
|
}
|
||||||
|
joined: list[dict[str, Any]] = []
|
||||||
|
for acceptance_row in acceptance_rows:
|
||||||
|
item_id = str(acceptance_row.get("item_id"))
|
||||||
|
reference = acceptance_row.get("reference")
|
||||||
|
if (
|
||||||
|
acceptance_row.get("split") not in {"development", "validation"}
|
||||||
|
or acceptance_row.get("severity") not in {"standard", "medium", "high"}
|
||||||
|
or not isinstance(reference, dict)
|
||||||
|
or set(reference) != set(_DIMENSIONS)
|
||||||
|
):
|
||||||
|
raise E38PerceptionBaselineError("E38 acceptance row is invalid")
|
||||||
|
joined.append(
|
||||||
|
{
|
||||||
|
"acceptance": acceptance_row,
|
||||||
|
"features": _feature_vector(
|
||||||
|
acceptance_row,
|
||||||
|
materialization_by_id[item_id],
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
model_dimensions: dict[str, Any] = {}
|
||||||
|
for dimension in _DIMENSIONS:
|
||||||
|
dimension_profile = profile["model"]["dimensions"][dimension]
|
||||||
|
feature_names = _dimension_features(
|
||||||
|
joined,
|
||||||
|
include_source_time=dimension_profile["include_source_time"],
|
||||||
|
)
|
||||||
|
development = [
|
||||||
|
(
|
||||||
|
_selected_features(row["features"], feature_names),
|
||||||
|
str(row["acceptance"]["reference"][dimension]),
|
||||||
|
)
|
||||||
|
for row in joined
|
||||||
|
if row["acceptance"]["split"] == "development"
|
||||||
|
]
|
||||||
|
tree = _train_tree(
|
||||||
|
development,
|
||||||
|
feature_names=feature_names,
|
||||||
|
max_depth=int(dimension_profile["max_depth"]),
|
||||||
|
min_leaf=int(dimension_profile["min_leaf"]),
|
||||||
|
)
|
||||||
|
model_dimensions[dimension] = {
|
||||||
|
"feature_names": feature_names,
|
||||||
|
"max_depth": dimension_profile["max_depth"],
|
||||||
|
"min_leaf": dimension_profile["min_leaf"],
|
||||||
|
"tree": tree,
|
||||||
|
}
|
||||||
|
|
||||||
|
predictions: list[dict[str, Any]] = []
|
||||||
|
for row in joined:
|
||||||
|
acceptance_row = row["acceptance"]
|
||||||
|
predicted = {
|
||||||
|
dimension: _predict_tree(
|
||||||
|
model_dimensions[dimension]["tree"],
|
||||||
|
row["features"],
|
||||||
|
)
|
||||||
|
for dimension in _DIMENSIONS
|
||||||
|
}
|
||||||
|
predictions.append(
|
||||||
|
{
|
||||||
|
"schema_version": E38_PREDICTION_SCHEMA,
|
||||||
|
"sequence": acceptance_row["sequence"],
|
||||||
|
"item_id": acceptance_row["item_id"],
|
||||||
|
"review_key": acceptance_row["review_key"],
|
||||||
|
"source_frame_index": acceptance_row["source_frame_index"],
|
||||||
|
"source_stratum": acceptance_row["source_stratum"],
|
||||||
|
"severity": acceptance_row["severity"],
|
||||||
|
"split": acceptance_row["split"],
|
||||||
|
"prediction": predicted,
|
||||||
|
"reference": acceptance_row["reference"],
|
||||||
|
"scored": acceptance_row["split"] == "validation",
|
||||||
|
"authority": _AUTHORITY,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
validation_predictions = [
|
||||||
|
row for row in predictions if row["split"] == "validation"
|
||||||
|
]
|
||||||
|
development_count = sum(row["split"] == "development" for row in predictions)
|
||||||
|
validation_count = len(validation_predictions)
|
||||||
|
if development_count != 340 or validation_count != 146:
|
||||||
|
raise E38PerceptionBaselineError("E38 frozen split changed")
|
||||||
|
dimension_metrics = {
|
||||||
|
dimension: _dimension_metrics(
|
||||||
|
validation_predictions,
|
||||||
|
dimension=dimension,
|
||||||
|
target=float(profile["targets"][f"{dimension}_target"]),
|
||||||
|
)
|
||||||
|
for dimension in _DIMENSIONS
|
||||||
|
}
|
||||||
|
high_severity_failures = sum(
|
||||||
|
row["severity"] == "high"
|
||||||
|
and any(
|
||||||
|
row["prediction"][dimension] != row["reference"][dimension]
|
||||||
|
for dimension in _DIMENSIONS
|
||||||
|
)
|
||||||
|
for row in validation_predictions
|
||||||
|
)
|
||||||
|
accounting_fraction = (
|
||||||
|
len(validation_predictions) / validation_count if validation_count else 0.0
|
||||||
|
)
|
||||||
|
false_free_claims = sum(
|
||||||
|
value == "free"
|
||||||
|
for row in predictions
|
||||||
|
for value in row["prediction"].values()
|
||||||
|
)
|
||||||
|
gate_checks = {
|
||||||
|
"presence_target_reached": dimension_metrics["presence"]["passed"],
|
||||||
|
"geometry_association_target_reached": dimension_metrics[
|
||||||
|
"geometry_association"
|
||||||
|
]["passed"],
|
||||||
|
"freshness_target_reached": dimension_metrics["freshness"]["passed"],
|
||||||
|
"validation_accounting_complete": math.isclose(accounting_fraction, 1.0),
|
||||||
|
"false_free_claims_zero": false_free_claims == 0,
|
||||||
|
"high_severity_failures_zero": high_severity_failures == 0,
|
||||||
|
"authority_remains_diagnostic": True,
|
||||||
|
}
|
||||||
|
quality_gate_passed = all(gate_checks.values())
|
||||||
|
model_document = {
|
||||||
|
"schema_version": E38_MODEL_SCHEMA,
|
||||||
|
"profile_id": profile["profile_id"],
|
||||||
|
"training_split": "development",
|
||||||
|
"training_items": development_count,
|
||||||
|
"validation_labels_used_for_training": False,
|
||||||
|
"dimensions": model_dimensions,
|
||||||
|
"authority": _AUTHORITY,
|
||||||
|
}
|
||||||
|
report = {
|
||||||
|
"schema_version": E38_REPORT_SCHEMA,
|
||||||
|
"result_id": result_id,
|
||||||
|
"identity_sha256": identity_sha256,
|
||||||
|
"status": "measured-r1-source-scoped-baseline",
|
||||||
|
"source_session_id": _SOURCE_SESSION_ID,
|
||||||
|
"source_display_name": _SOURCE_DISPLAY_NAME,
|
||||||
|
"profile_id": profile["profile_id"],
|
||||||
|
"execution": identity["execution"],
|
||||||
|
"metrics": {
|
||||||
|
"development_items": development_count,
|
||||||
|
"validation_items": validation_count,
|
||||||
|
"terminal_outcomes": validation_count,
|
||||||
|
"accounting_fraction": round(accounting_fraction, 6),
|
||||||
|
"false_free_claims": false_free_claims,
|
||||||
|
"high_severity_failures": high_severity_failures,
|
||||||
|
"dimensions": dimension_metrics,
|
||||||
|
},
|
||||||
|
"quality_gate": {
|
||||||
|
"passed": quality_gate_passed,
|
||||||
|
"checks": gate_checks,
|
||||||
|
"blocking_checks": [
|
||||||
|
name for name, passed in gate_checks.items() if not passed
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"decision": {
|
||||||
|
"r1_baseline_measured": True,
|
||||||
|
"accepted_for_release": quality_gate_passed,
|
||||||
|
"next_gate": (
|
||||||
|
"R2 source-scoped temporal product state"
|
||||||
|
if quality_gate_passed
|
||||||
|
else "R1 detector presence and geometry-association improvement"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"limitations": [
|
||||||
|
(
|
||||||
|
"the model is trained and evaluated only on the source-scoped "
|
||||||
|
"RAVNOVES00 engineering-reviewed contract"
|
||||||
|
),
|
||||||
|
"validation labels are evaluation-only and never used to fit a tree",
|
||||||
|
(
|
||||||
|
"the result is not independent ground truth and does not prove "
|
||||||
|
"another route, camera, rig or mount"
|
||||||
|
),
|
||||||
|
"navigation, command and safety authority remain false",
|
||||||
|
],
|
||||||
|
"authority": _AUTHORITY,
|
||||||
|
}
|
||||||
|
|
||||||
|
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
|
||||||
|
staging.mkdir(mode=0o700, exist_ok=False)
|
||||||
|
try:
|
||||||
|
_write_jsonl(staging / E38_PREDICTIONS_NAME, predictions)
|
||||||
|
_write_json(staging / E38_MODEL_NAME, model_document)
|
||||||
|
_write_json(staging / E38_REPORT_NAME, report)
|
||||||
|
artifacts = [
|
||||||
|
_artifact(staging / E38_PREDICTIONS_NAME, "sealed-evaluation"),
|
||||||
|
_artifact(staging / E38_MODEL_NAME, "development-trained-model"),
|
||||||
|
_artifact(staging / E38_REPORT_NAME, "quality-report"),
|
||||||
|
]
|
||||||
|
manifest = {
|
||||||
|
"schema_version": E38_RESULT_SCHEMA,
|
||||||
|
"result_id": result_id,
|
||||||
|
"identity_sha256": identity_sha256,
|
||||||
|
"identity": identity,
|
||||||
|
"created_at_utc": _utc_now(),
|
||||||
|
"acceptance_state": "completed-r1-source-scoped-baseline",
|
||||||
|
"quality_gate_passed": quality_gate_passed,
|
||||||
|
"artifacts": artifacts,
|
||||||
|
}
|
||||||
|
_write_json(staging / E38_MANIFEST_NAME, manifest)
|
||||||
|
os.replace(staging, destination)
|
||||||
|
except BaseException:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
raise
|
||||||
|
return read_e38_perception_baseline(destination)
|
||||||
|
|
||||||
|
|
||||||
|
def read_e38_perception_baseline(root: Path) -> E38PerceptionBaseline:
|
||||||
|
"""Validate and read one immutable E38 result."""
|
||||||
|
|
||||||
|
resolved = root.resolve(strict=True)
|
||||||
|
manifest = _read_json(resolved / E38_MANIFEST_NAME)
|
||||||
|
identity = manifest.get("identity")
|
||||||
|
identity_sha256 = manifest.get("identity_sha256")
|
||||||
|
if (
|
||||||
|
manifest.get("schema_version") != E38_RESULT_SCHEMA
|
||||||
|
or not isinstance(identity, dict)
|
||||||
|
or not isinstance(identity_sha256, str)
|
||||||
|
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||||
|
or manifest.get("result_id") != f"e38-perception-baseline-{identity_sha256}"
|
||||||
|
or resolved.name != manifest.get("result_id")
|
||||||
|
or manifest.get("acceptance_state")
|
||||||
|
!= "completed-r1-source-scoped-baseline"
|
||||||
|
or identity.get("authority") != _AUTHORITY
|
||||||
|
):
|
||||||
|
raise E38PerceptionBaselineError("E38 result identity is invalid")
|
||||||
|
expected = {
|
||||||
|
E38_PREDICTIONS_NAME: "sealed-evaluation",
|
||||||
|
E38_MODEL_NAME: "development-trained-model",
|
||||||
|
E38_REPORT_NAME: "quality-report",
|
||||||
|
}
|
||||||
|
artifacts = manifest.get("artifacts")
|
||||||
|
if not isinstance(artifacts, list) or len(artifacts) != len(expected):
|
||||||
|
raise E38PerceptionBaselineError("E38 artifact catalog is invalid")
|
||||||
|
for row in artifacts:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
raise E38PerceptionBaselineError("E38 artifact descriptor is invalid")
|
||||||
|
name = row.get("path")
|
||||||
|
path = resolved / str(name)
|
||||||
|
if (
|
||||||
|
name not in expected
|
||||||
|
or row.get("role") != expected[name]
|
||||||
|
or not path.is_file()
|
||||||
|
or path.is_symlink()
|
||||||
|
or row.get("byte_length") != path.stat().st_size
|
||||||
|
or row.get("sha256") != _sha256(path)
|
||||||
|
):
|
||||||
|
raise E38PerceptionBaselineError("E38 artifact content changed")
|
||||||
|
report = _read_json(resolved / E38_REPORT_NAME)
|
||||||
|
model = _read_json(resolved / E38_MODEL_NAME)
|
||||||
|
if (
|
||||||
|
report.get("schema_version") != E38_REPORT_SCHEMA
|
||||||
|
or report.get("result_id") != resolved.name
|
||||||
|
or report.get("identity_sha256") != identity_sha256
|
||||||
|
or report.get("status") != "measured-r1-source-scoped-baseline"
|
||||||
|
or model.get("schema_version") != E38_MODEL_SCHEMA
|
||||||
|
or model.get("validation_labels_used_for_training") is not False
|
||||||
|
or manifest.get("quality_gate_passed")
|
||||||
|
is not report.get("quality_gate", {}).get("passed")
|
||||||
|
):
|
||||||
|
raise E38PerceptionBaselineError("E38 report or model is invalid")
|
||||||
|
return E38PerceptionBaseline(
|
||||||
|
result_id=resolved.name,
|
||||||
|
result_root=resolved,
|
||||||
|
manifest=manifest,
|
||||||
|
report=report,
|
||||||
|
model=model,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_materialization(
|
||||||
|
root: Path,
|
||||||
|
acceptance_manifest: dict[str, Any],
|
||||||
|
profile: dict[str, Any],
|
||||||
|
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||||
|
manifest_path = root / "manifest.json"
|
||||||
|
index_path = root / "materialized-items.jsonl"
|
||||||
|
manifest = _read_json(manifest_path)
|
||||||
|
binding = (
|
||||||
|
acceptance_manifest.get("identity", {})
|
||||||
|
.get("reviewed_substrate", {})
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
manifest.get("schema_version") != _MATERIALIZATION_SCHEMA
|
||||||
|
or root.name != profile["source"]["materialization_id"]
|
||||||
|
or manifest.get("result_id") != root.name
|
||||||
|
or root.name != binding.get("materialization_id")
|
||||||
|
or manifest.get("identity_sha256")
|
||||||
|
!= binding.get("materialization_identity_sha256")
|
||||||
|
or _sha256(manifest_path)
|
||||||
|
!= binding.get("materialization_manifest_sha256")
|
||||||
|
or not index_path.is_file()
|
||||||
|
or index_path.is_symlink()
|
||||||
|
or _sha256(index_path) != binding.get("materialization_index_sha256")
|
||||||
|
):
|
||||||
|
raise E38PerceptionBaselineError("E38 materialization identity changed")
|
||||||
|
rows = _read_jsonl(index_path)
|
||||||
|
return rows, {
|
||||||
|
"materialization_id": root.name,
|
||||||
|
"materialization_identity_sha256": manifest["identity_sha256"],
|
||||||
|
"materialization_index_sha256": _sha256(index_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _feature_vector(
|
||||||
|
acceptance: dict[str, Any],
|
||||||
|
materialization: dict[str, Any],
|
||||||
|
) -> dict[str, float]:
|
||||||
|
snapshot = _object(materialization.get("e29_snapshot"), "E38 snapshot")
|
||||||
|
evidence = _object(
|
||||||
|
materialization.get("materialization"),
|
||||||
|
"E38 materialization evidence",
|
||||||
|
)
|
||||||
|
bounds = snapshot.get("bounds_map_xyz_m")
|
||||||
|
height = snapshot.get("height_range_m")
|
||||||
|
features = {
|
||||||
|
"detector_score": _number_or(evidence.get("detector_score"), -1.0),
|
||||||
|
"selected_points": _number_or(evidence.get("selected_point_count"), 0.0),
|
||||||
|
"rejected_points": _number_or(
|
||||||
|
evidence.get("rejected_candidate_point_count"),
|
||||||
|
0.0,
|
||||||
|
),
|
||||||
|
"candidate_points": _number_or(evidence.get("candidate_point_count"), 0.0),
|
||||||
|
"camera_front_points": _number_or(
|
||||||
|
evidence.get("camera_front_point_count"),
|
||||||
|
0.0,
|
||||||
|
),
|
||||||
|
"projected_points": _number_or(
|
||||||
|
evidence.get("projected_point_count"),
|
||||||
|
0.0,
|
||||||
|
),
|
||||||
|
"frame_points": _number_or(evidence.get("frame_point_count"), 0.0),
|
||||||
|
"point_count": _number_or(snapshot.get("point_count"), 0.0),
|
||||||
|
"voxel_count": _number_or(snapshot.get("voxel_count"), 0.0),
|
||||||
|
"nearest_range_m": _number_or(snapshot.get("nearest_range_m"), -1.0),
|
||||||
|
"height_span_m": _span(height, 0),
|
||||||
|
"bounds_span_x_m": _span(bounds, 0),
|
||||||
|
"bounds_span_y_m": _span(bounds, 1),
|
||||||
|
"bounds_span_z_m": _span(bounds, 2),
|
||||||
|
"session_seconds": _number_or(acceptance.get("session_seconds"), -1.0),
|
||||||
|
"source_frame_index": _number_or(
|
||||||
|
acceptance.get("source_frame_index"),
|
||||||
|
-1.0,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
categorical = {
|
||||||
|
"stratum": materialization.get("stratum"),
|
||||||
|
"range": materialization.get("range_bucket"),
|
||||||
|
"geometry": snapshot.get("geometry_status"),
|
||||||
|
}
|
||||||
|
for prefix, values in _CATEGORICAL_FEATURES.items():
|
||||||
|
observed = str(categorical[prefix])
|
||||||
|
for value in values:
|
||||||
|
features[f"{prefix}={value}"] = float(observed == value)
|
||||||
|
return features
|
||||||
|
|
||||||
|
|
||||||
|
def _dimension_features(
|
||||||
|
rows: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
include_source_time: bool,
|
||||||
|
) -> list[str]:
|
||||||
|
features = sorted(
|
||||||
|
{
|
||||||
|
name
|
||||||
|
for row in rows
|
||||||
|
for name in row["features"]
|
||||||
|
if include_source_time
|
||||||
|
or name not in {"session_seconds", "source_frame_index"}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if not features:
|
||||||
|
raise E38PerceptionBaselineError("E38 feature set is empty")
|
||||||
|
return features
|
||||||
|
|
||||||
|
|
||||||
|
def _selected_features(
|
||||||
|
values: dict[str, float],
|
||||||
|
names: list[str],
|
||||||
|
) -> dict[str, float]:
|
||||||
|
return {name: values.get(name, 0.0) for name in names}
|
||||||
|
|
||||||
|
|
||||||
|
def _train_tree(
|
||||||
|
rows: list[tuple[dict[str, float], str]],
|
||||||
|
*,
|
||||||
|
feature_names: list[str],
|
||||||
|
max_depth: int,
|
||||||
|
min_leaf: int,
|
||||||
|
depth: int = 0,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not rows:
|
||||||
|
raise E38PerceptionBaselineError("E38 tree has no training rows")
|
||||||
|
labels = [label for _, label in rows]
|
||||||
|
prediction = sorted(Counter(labels).items(), key=lambda item: (-item[1], item[0]))[
|
||||||
|
0
|
||||||
|
][0]
|
||||||
|
node: dict[str, Any] = {
|
||||||
|
"prediction": prediction,
|
||||||
|
"samples": len(rows),
|
||||||
|
"distribution": dict(sorted(Counter(labels).items())),
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
depth >= max_depth
|
||||||
|
or len(set(labels)) == 1
|
||||||
|
or len(rows) < 2 * min_leaf
|
||||||
|
):
|
||||||
|
return node
|
||||||
|
base_impurity = _gini(labels)
|
||||||
|
best: tuple[
|
||||||
|
float,
|
||||||
|
str,
|
||||||
|
float,
|
||||||
|
list[tuple[dict[str, float], str]],
|
||||||
|
list[tuple[dict[str, float], str]],
|
||||||
|
] | None = None
|
||||||
|
for feature in feature_names:
|
||||||
|
values = sorted({float(features.get(feature, 0.0)) for features, _ in rows})
|
||||||
|
for left_value, right_value in zip(values, values[1:], strict=False):
|
||||||
|
threshold = (left_value + right_value) / 2.0
|
||||||
|
left_rows = [
|
||||||
|
row for row in rows if float(row[0].get(feature, 0.0)) <= threshold
|
||||||
|
]
|
||||||
|
right_rows = [
|
||||||
|
row for row in rows if float(row[0].get(feature, 0.0)) > threshold
|
||||||
|
]
|
||||||
|
if len(left_rows) < min_leaf or len(right_rows) < min_leaf:
|
||||||
|
continue
|
||||||
|
impurity = (
|
||||||
|
len(left_rows) * _gini([label for _, label in left_rows])
|
||||||
|
+ len(right_rows) * _gini([label for _, label in right_rows])
|
||||||
|
) / len(rows)
|
||||||
|
gain = base_impurity - impurity
|
||||||
|
candidate = (gain, feature, threshold, left_rows, right_rows)
|
||||||
|
if best is None or _better_split(candidate, best):
|
||||||
|
best = candidate
|
||||||
|
if best is None or best[0] <= 1e-12:
|
||||||
|
return node
|
||||||
|
node.update(
|
||||||
|
{
|
||||||
|
"feature": best[1],
|
||||||
|
"threshold": round(best[2], 12),
|
||||||
|
"left": _train_tree(
|
||||||
|
best[3],
|
||||||
|
feature_names=feature_names,
|
||||||
|
max_depth=max_depth,
|
||||||
|
min_leaf=min_leaf,
|
||||||
|
depth=depth + 1,
|
||||||
|
),
|
||||||
|
"right": _train_tree(
|
||||||
|
best[4],
|
||||||
|
feature_names=feature_names,
|
||||||
|
max_depth=max_depth,
|
||||||
|
min_leaf=min_leaf,
|
||||||
|
depth=depth + 1,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return node
|
||||||
|
|
||||||
|
|
||||||
|
def _better_split(
|
||||||
|
candidate: tuple[float, str, float, Any, Any],
|
||||||
|
current: tuple[float, str, float, Any, Any],
|
||||||
|
) -> bool:
|
||||||
|
if candidate[0] > current[0] + 1e-12:
|
||||||
|
return True
|
||||||
|
if abs(candidate[0] - current[0]) <= 1e-12:
|
||||||
|
return (candidate[1], candidate[2]) < (current[1], current[2])
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _predict_tree(tree: dict[str, Any], features: dict[str, float]) -> str:
|
||||||
|
node = tree
|
||||||
|
while "feature" in node:
|
||||||
|
feature = str(node["feature"])
|
||||||
|
threshold = float(node["threshold"])
|
||||||
|
node = (
|
||||||
|
_object(node.get("left"), "E38 tree left")
|
||||||
|
if features.get(feature, 0.0) <= threshold
|
||||||
|
else _object(node.get("right"), "E38 tree right")
|
||||||
|
)
|
||||||
|
prediction = node.get("prediction")
|
||||||
|
if not isinstance(prediction, str) or not prediction:
|
||||||
|
raise E38PerceptionBaselineError("E38 tree prediction is invalid")
|
||||||
|
return prediction
|
||||||
|
|
||||||
|
|
||||||
|
def _dimension_metrics(
|
||||||
|
rows: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
dimension: str,
|
||||||
|
target: float,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
confusion: Counter[tuple[str, str]] = Counter()
|
||||||
|
stratum_counts: dict[str, Counter[str]] = defaultdict(Counter)
|
||||||
|
correct = 0
|
||||||
|
for row in rows:
|
||||||
|
reference = str(row["reference"][dimension])
|
||||||
|
prediction = str(row["prediction"][dimension])
|
||||||
|
confusion[(reference, prediction)] += 1
|
||||||
|
matched = reference == prediction
|
||||||
|
correct += matched
|
||||||
|
stratum_counts[str(row["source_stratum"])][
|
||||||
|
"correct" if matched else "incorrect"
|
||||||
|
] += 1
|
||||||
|
total = len(rows)
|
||||||
|
accuracy = correct / total if total else 0.0
|
||||||
|
return {
|
||||||
|
"correct": correct,
|
||||||
|
"incorrect": total - correct,
|
||||||
|
"total": total,
|
||||||
|
"accuracy": round(accuracy, 6),
|
||||||
|
"target": target,
|
||||||
|
"passed": accuracy >= target,
|
||||||
|
"confusion": [
|
||||||
|
{
|
||||||
|
"reference": reference,
|
||||||
|
"prediction": prediction,
|
||||||
|
"count": count,
|
||||||
|
}
|
||||||
|
for (reference, prediction), count in sorted(
|
||||||
|
confusion.items(),
|
||||||
|
key=lambda item: (-item[1], item[0]),
|
||||||
|
)
|
||||||
|
],
|
||||||
|
"by_stratum": {
|
||||||
|
stratum: {
|
||||||
|
"correct": counts["correct"],
|
||||||
|
"incorrect": counts["incorrect"],
|
||||||
|
"total": sum(counts.values()),
|
||||||
|
"accuracy": round(
|
||||||
|
counts["correct"] / sum(counts.values()),
|
||||||
|
6,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
for stratum, counts in sorted(stratum_counts.items())
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _gini(labels: list[str]) -> float:
|
||||||
|
counts = Counter(labels)
|
||||||
|
total = len(labels)
|
||||||
|
return 1.0 - sum((count / total) ** 2 for count in counts.values())
|
||||||
|
|
||||||
|
|
||||||
|
def _span(value: object, axis: int) -> float:
|
||||||
|
if not isinstance(value, list) or len(value) != 2:
|
||||||
|
return -1.0
|
||||||
|
if axis == 0 and all(isinstance(item, (int, float)) for item in value):
|
||||||
|
return float(value[1]) - float(value[0])
|
||||||
|
if not all(
|
||||||
|
isinstance(item, list)
|
||||||
|
and len(item) > axis
|
||||||
|
and isinstance(item[axis], (int, float))
|
||||||
|
for item in value
|
||||||
|
):
|
||||||
|
return -1.0
|
||||||
|
return float(value[1][axis]) - float(value[0][axis])
|
||||||
|
|
||||||
|
|
||||||
|
def _number_or(value: object, fallback: float) -> float:
|
||||||
|
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||||
|
parsed = float(value)
|
||||||
|
if math.isfinite(parsed):
|
||||||
|
return parsed
|
||||||
|
return fallback
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_profile(profile: dict[str, Any]) -> None:
|
||||||
|
source = _object(profile.get("source"), "E38 source")
|
||||||
|
model = _object(profile.get("model"), "E38 model")
|
||||||
|
dimensions = _object(model.get("dimensions"), "E38 model dimensions")
|
||||||
|
targets = _object(profile.get("targets"), "E38 targets")
|
||||||
|
if (
|
||||||
|
profile.get("schema_version") != E38_PROFILE_SCHEMA
|
||||||
|
or profile.get("profile_id")
|
||||||
|
!= "e38-ravnoves00-r1-development-cart/v1"
|
||||||
|
or source.get("session_id") != _SOURCE_SESSION_ID
|
||||||
|
or source.get("display_name") != _SOURCE_DISPLAY_NAME
|
||||||
|
or not isinstance(source.get("acceptance_result_id"), str)
|
||||||
|
or not str(source["acceptance_result_id"]).startswith(
|
||||||
|
"e37-ravnoves-acceptance-"
|
||||||
|
)
|
||||||
|
or not isinstance(source.get("materialization_id"), str)
|
||||||
|
or not str(source["materialization_id"]).startswith("e30-materialization-")
|
||||||
|
or model.get("type") != "deterministic-shallow-cart"
|
||||||
|
or set(dimensions) != set(_DIMENSIONS)
|
||||||
|
or any(
|
||||||
|
not isinstance(dimensions[name], dict)
|
||||||
|
or not isinstance(dimensions[name].get("include_source_time"), bool)
|
||||||
|
or not isinstance(dimensions[name].get("max_depth"), int)
|
||||||
|
or not 1 <= dimensions[name]["max_depth"] <= 8
|
||||||
|
or not isinstance(dimensions[name].get("min_leaf"), int)
|
||||||
|
or not 2 <= dimensions[name]["min_leaf"] <= 32
|
||||||
|
for name in _DIMENSIONS
|
||||||
|
)
|
||||||
|
or any(targets.get(f"{name}_target") != 0.9 for name in _DIMENSIONS)
|
||||||
|
or targets.get("accounting_target") != 1.0
|
||||||
|
or targets.get("maximum_false_free_claims") != 0
|
||||||
|
or profile.get("authority") != _AUTHORITY
|
||||||
|
):
|
||||||
|
raise E38PerceptionBaselineError("E38 profile contract changed")
|
||||||
|
|
||||||
|
|
||||||
|
def _artifact(path: Path, role: str) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"role": role,
|
||||||
|
"path": path.name,
|
||||||
|
"byte_length": path.stat().st_size,
|
||||||
|
"sha256": _sha256(path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _object(value: object, label: str) -> dict[str, Any]:
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise E38PerceptionBaselineError(f"{label} must be an object")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _read_json(path: Path) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise E38PerceptionBaselineError(f"invalid JSON: {path.name}") from exc
|
||||||
|
return _object(value, path.name)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
try:
|
||||||
|
with path.open("r", encoding="utf-8-sig") as stream:
|
||||||
|
for line in stream:
|
||||||
|
rows.append(_object(json.loads(line), path.name))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise E38PerceptionBaselineError(f"invalid JSONL: {path.name}") from exc
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _write_json(path: Path, value: object) -> None:
|
||||||
|
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||||
|
json.dump(value, stream, ensure_ascii=False, indent=2, sort_keys=True)
|
||||||
|
stream.write("\n")
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
|
||||||
|
|
||||||
|
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||||
|
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||||
|
for row in rows:
|
||||||
|
stream.write(
|
||||||
|
json.dumps(
|
||||||
|
row,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
stream.write("\n")
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
while chunk := stream.read(1024 * 1024):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now() -> str:
|
||||||
|
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||||
@@ -39,6 +39,11 @@ from k1link.compute.e37_acceptance_contract import (
|
|||||||
E37AcceptanceContractError,
|
E37AcceptanceContractError,
|
||||||
read_e37_acceptance_contract,
|
read_e37_acceptance_contract,
|
||||||
)
|
)
|
||||||
|
from k1link.compute.e38_perception_baseline import (
|
||||||
|
E38PerceptionBaseline,
|
||||||
|
E38PerceptionBaselineError,
|
||||||
|
read_e38_perception_baseline,
|
||||||
|
)
|
||||||
|
|
||||||
LABORATORY_ADVANCED_CATALOG_SCHEMA: Final = (
|
LABORATORY_ADVANCED_CATALOG_SCHEMA: Final = (
|
||||||
"missioncore.laboratory-advanced-catalog/v1"
|
"missioncore.laboratory-advanced-catalog/v1"
|
||||||
@@ -50,6 +55,7 @@ _E33_RESULT_ID = re.compile(r"^e33-worker-shadow-[a-f0-9]{64}$")
|
|||||||
_E34_RESULT_ID = re.compile(r"^e34-temporal-occupied-[a-f0-9]{64}$")
|
_E34_RESULT_ID = re.compile(r"^e34-temporal-occupied-[a-f0-9]{64}$")
|
||||||
_E35_RESULT_ID = re.compile(r"^e35-degradation-recovery-[a-f0-9]{64}$")
|
_E35_RESULT_ID = re.compile(r"^e35-degradation-recovery-[a-f0-9]{64}$")
|
||||||
_E37_RESULT_ID = re.compile(r"^e37-ravnoves-acceptance-[a-f0-9]{64}$")
|
_E37_RESULT_ID = re.compile(r"^e37-ravnoves-acceptance-[a-f0-9]{64}$")
|
||||||
|
_E38_RESULT_ID = re.compile(r"^e38-perception-baseline-[a-f0-9]{64}$")
|
||||||
|
|
||||||
RootProvider = Callable[[], Path | None]
|
RootProvider = Callable[[], Path | None]
|
||||||
|
|
||||||
@@ -118,6 +124,15 @@ def _read_e37_cached(
|
|||||||
return read_e37_acceptance_contract(Path(root_text))
|
return read_e37_acceptance_contract(Path(root_text))
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=16)
|
||||||
|
def _read_e38_cached(
|
||||||
|
root_text: str,
|
||||||
|
signature: tuple[int, ...],
|
||||||
|
) -> E38PerceptionBaseline:
|
||||||
|
del signature
|
||||||
|
return read_e38_perception_baseline(Path(root_text))
|
||||||
|
|
||||||
|
|
||||||
def _configured_root(provider: RootProvider) -> Path | None:
|
def _configured_root(provider: RootProvider) -> Path | None:
|
||||||
value = provider()
|
value = provider()
|
||||||
if value is None:
|
if value is None:
|
||||||
@@ -555,6 +570,40 @@ def _project_e37(result: E37AcceptanceContract) -> dict[str, object]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _project_e38(result: E38PerceptionBaseline) -> dict[str, object]:
|
||||||
|
identity = _object(result.manifest.get("identity"), "E38 identity")
|
||||||
|
source = _object(identity.get("source"), "E38 source")
|
||||||
|
execution = _object(identity.get("execution"), "E38 execution")
|
||||||
|
profile = _object(identity.get("profile"), "E38 profile")
|
||||||
|
metrics = _object(result.report.get("metrics"), "E38 metrics")
|
||||||
|
dimensions = _object(metrics.get("dimensions"), "E38 dimensions")
|
||||||
|
quality_gate = _object(result.report.get("quality_gate"), "E38 gate")
|
||||||
|
return {
|
||||||
|
"result_id": result.result_id,
|
||||||
|
"created_at_utc": result.manifest.get("created_at_utc"),
|
||||||
|
"source_session_id": source.get("session_id"),
|
||||||
|
"source_display_name": source.get("display_name"),
|
||||||
|
"status": result.report.get("status"),
|
||||||
|
"profile_id": profile.get("profile_id"),
|
||||||
|
"worker_node": execution.get("worker_node"),
|
||||||
|
"quality_gate_passed": quality_gate.get("passed"),
|
||||||
|
"metrics": {
|
||||||
|
"development_items": metrics.get("development_items"),
|
||||||
|
"validation_items": metrics.get("validation_items"),
|
||||||
|
"terminal_outcomes": metrics.get("terminal_outcomes"),
|
||||||
|
"accounting_fraction": metrics.get("accounting_fraction"),
|
||||||
|
"false_free_claims": metrics.get("false_free_claims"),
|
||||||
|
"high_severity_failures": metrics.get("high_severity_failures"),
|
||||||
|
"dimensions": copy.deepcopy(dimensions),
|
||||||
|
},
|
||||||
|
"blocking_checks": copy.deepcopy(quality_gate.get("blocking_checks")),
|
||||||
|
"decision": copy.deepcopy(result.report.get("decision")),
|
||||||
|
"limitations": copy.deepcopy(result.report.get("limitations")),
|
||||||
|
"authority": copy.deepcopy(result.report.get("authority")),
|
||||||
|
"access": "read-only",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _empty_catalog(configured: bool) -> dict[str, object]:
|
def _empty_catalog(configured: bool) -> dict[str, object]:
|
||||||
return {
|
return {
|
||||||
"schema_version": LABORATORY_ADVANCED_CATALOG_SCHEMA,
|
"schema_version": LABORATORY_ADVANCED_CATALOG_SCHEMA,
|
||||||
@@ -574,6 +623,7 @@ def build_advanced_laboratory_router(
|
|||||||
e34_root_provider: RootProvider = lambda: None,
|
e34_root_provider: RootProvider = lambda: None,
|
||||||
e35_root_provider: RootProvider = lambda: None,
|
e35_root_provider: RootProvider = lambda: None,
|
||||||
e37_root_provider: RootProvider = lambda: None,
|
e37_root_provider: RootProvider = lambda: None,
|
||||||
|
e38_root_provider: RootProvider = lambda: None,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
router = APIRouter(prefix="/api/v1/laboratory", tags=["laboratory"])
|
router = APIRouter(prefix="/api/v1/laboratory", tags=["laboratory"])
|
||||||
|
|
||||||
@@ -803,4 +853,37 @@ def build_advanced_laboratory_router(
|
|||||||
"invalid_total": invalid_total,
|
"invalid_total": invalid_total,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@router.get("/e38/results")
|
||||||
|
def list_e38_results(
|
||||||
|
limit: int = Query(default=1, ge=1, le=10),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
root = _configured_root(e38_root_provider)
|
||||||
|
if root is None:
|
||||||
|
return _empty_catalog(False)
|
||||||
|
candidates = _candidates(root, _E38_RESULT_ID)
|
||||||
|
items: list[dict[str, object]] = []
|
||||||
|
invalid_total = 0
|
||||||
|
for candidate in candidates:
|
||||||
|
try:
|
||||||
|
result = _read_e38_cached(
|
||||||
|
str(candidate.resolve()),
|
||||||
|
_result_signature(candidate),
|
||||||
|
)
|
||||||
|
if len(items) < limit:
|
||||||
|
items.append(_project_e38(result))
|
||||||
|
except (
|
||||||
|
E38PerceptionBaselineError,
|
||||||
|
KeyError,
|
||||||
|
OSError,
|
||||||
|
TypeError,
|
||||||
|
ValueError,
|
||||||
|
):
|
||||||
|
invalid_total += 1
|
||||||
|
return {
|
||||||
|
**_empty_catalog(True),
|
||||||
|
"items": items,
|
||||||
|
"candidate_total": len(candidates),
|
||||||
|
"invalid_total": invalid_total,
|
||||||
|
}
|
||||||
|
|
||||||
return router
|
return router
|
||||||
|
|||||||
@@ -543,6 +543,13 @@ app.include_router(
|
|||||||
/ "e37"
|
/ "e37"
|
||||||
/ "results"
|
/ "results"
|
||||||
),
|
),
|
||||||
|
e38_root_provider=lambda: (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ ".runtime"
|
||||||
|
/ "compute-experiments"
|
||||||
|
/ "e38"
|
||||||
|
/ "results"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
app.include_router(
|
app.include_router(
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ def _endpoint(router: APIRouter, path: str) -> object:
|
|||||||
def test_advanced_catalogs_are_empty_when_not_configured() -> None:
|
def test_advanced_catalogs_are_empty_when_not_configured() -> None:
|
||||||
router = build_advanced_laboratory_router()
|
router = build_advanced_laboratory_router()
|
||||||
|
|
||||||
for name in ("e31", "e32", "e33", "e34", "e35", "e37"):
|
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38"):
|
||||||
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
|
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
|
||||||
catalog = route(limit=1) # type: ignore[operator]
|
catalog = route(limit=1) # type: ignore[operator]
|
||||||
assert catalog == {
|
assert catalog == {
|
||||||
@@ -48,7 +48,8 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
|
|||||||
e34 = tmp_path / "e34"
|
e34 = tmp_path / "e34"
|
||||||
e35 = tmp_path / "e35"
|
e35 = tmp_path / "e35"
|
||||||
e37 = tmp_path / "e37"
|
e37 = tmp_path / "e37"
|
||||||
for root in (e31, e32, e33, e34, e35, e37):
|
e38 = tmp_path / "e38"
|
||||||
|
for root in (e31, e32, e33, e34, e35, e37, e38):
|
||||||
root.mkdir()
|
root.mkdir()
|
||||||
(e31 / f"e31-source-qualification-{'1' * 64}").mkdir()
|
(e31 / f"e31-source-qualification-{'1' * 64}").mkdir()
|
||||||
(e32 / f"e32-track-geometry-{'2' * 64}").mkdir()
|
(e32 / f"e32-track-geometry-{'2' * 64}").mkdir()
|
||||||
@@ -56,6 +57,7 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
|
|||||||
(e34 / f"e34-temporal-occupied-{'4' * 64}").mkdir()
|
(e34 / f"e34-temporal-occupied-{'4' * 64}").mkdir()
|
||||||
(e35 / f"e35-degradation-recovery-{'5' * 64}").mkdir()
|
(e35 / f"e35-degradation-recovery-{'5' * 64}").mkdir()
|
||||||
(e37 / f"e37-ravnoves-acceptance-{'7' * 64}").mkdir()
|
(e37 / f"e37-ravnoves-acceptance-{'7' * 64}").mkdir()
|
||||||
|
(e38 / f"e38-perception-baseline-{'8' * 64}").mkdir()
|
||||||
router = build_advanced_laboratory_router(
|
router = build_advanced_laboratory_router(
|
||||||
e31_root_provider=lambda: e31,
|
e31_root_provider=lambda: e31,
|
||||||
e32_root_provider=lambda: e32,
|
e32_root_provider=lambda: e32,
|
||||||
@@ -63,9 +65,10 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
|
|||||||
e34_root_provider=lambda: e34,
|
e34_root_provider=lambda: e34,
|
||||||
e35_root_provider=lambda: e35,
|
e35_root_provider=lambda: e35,
|
||||||
e37_root_provider=lambda: e37,
|
e37_root_provider=lambda: e37,
|
||||||
|
e38_root_provider=lambda: e38,
|
||||||
)
|
)
|
||||||
|
|
||||||
for name in ("e31", "e32", "e33", "e34", "e35", "e37"):
|
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38"):
|
||||||
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
|
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
|
||||||
catalog = route(limit=1) # type: ignore[operator]
|
catalog = route(limit=1) # type: ignore[operator]
|
||||||
assert catalog["configured"] is True
|
assert catalog["configured"] is True
|
||||||
@@ -422,3 +425,105 @@ def test_e37_catalog_projects_the_frozen_r0_contract(
|
|||||||
assert item["quality_target_evaluated"] is False
|
assert item["quality_target_evaluated"] is False
|
||||||
assert item["authority"] == authority
|
assert item["authority"] == authority
|
||||||
assert item["access"] == "read-only"
|
assert item["access"] == "read-only"
|
||||||
|
|
||||||
|
|
||||||
|
def test_e38_catalog_projects_failed_dimensions_without_hiding_result(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
result_id = f"e38-perception-baseline-{'8' * 64}"
|
||||||
|
root = tmp_path / "e38"
|
||||||
|
candidate = root / result_id
|
||||||
|
candidate.mkdir(parents=True)
|
||||||
|
(candidate / "manifest.json").write_text("{}", encoding="utf-8")
|
||||||
|
authority = {
|
||||||
|
"commands_enabled": False,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
}
|
||||||
|
dimension = {
|
||||||
|
"correct": 120,
|
||||||
|
"incorrect": 26,
|
||||||
|
"total": 146,
|
||||||
|
"accuracy": 0.821918,
|
||||||
|
"target": 0.9,
|
||||||
|
"passed": False,
|
||||||
|
"confusion": [],
|
||||||
|
"by_stratum": {},
|
||||||
|
}
|
||||||
|
result = SimpleNamespace(
|
||||||
|
result_id=result_id,
|
||||||
|
manifest={
|
||||||
|
"created_at_utc": "2026-07-28T02:50:00Z",
|
||||||
|
"identity": {
|
||||||
|
"source": {
|
||||||
|
"session_id": "20260720T065719Z_viewer_live",
|
||||||
|
"display_name": "RAVNOVES00",
|
||||||
|
},
|
||||||
|
"profile": {
|
||||||
|
"profile_id": "e38-ravnoves00-r1-development-cart/v1",
|
||||||
|
},
|
||||||
|
"execution": {
|
||||||
|
"worker_node": "DESKTOP-OPJ8J04",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
report={
|
||||||
|
"status": "measured-r1-source-scoped-baseline",
|
||||||
|
"metrics": {
|
||||||
|
"development_items": 340,
|
||||||
|
"validation_items": 146,
|
||||||
|
"terminal_outcomes": 146,
|
||||||
|
"accounting_fraction": 1.0,
|
||||||
|
"false_free_claims": 0,
|
||||||
|
"high_severity_failures": 14,
|
||||||
|
"dimensions": {
|
||||||
|
"presence": dimension,
|
||||||
|
"geometry_association": dimension,
|
||||||
|
"freshness": {
|
||||||
|
**dimension,
|
||||||
|
"correct": 139,
|
||||||
|
"incorrect": 7,
|
||||||
|
"accuracy": 0.952055,
|
||||||
|
"passed": True,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"quality_gate": {
|
||||||
|
"passed": False,
|
||||||
|
"blocking_checks": [
|
||||||
|
"presence_target_reached",
|
||||||
|
"geometry_association_target_reached",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"decision": {
|
||||||
|
"r1_baseline_measured": True,
|
||||||
|
"accepted_for_release": False,
|
||||||
|
},
|
||||||
|
"limitations": ["source-scoped"],
|
||||||
|
"authority": authority,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_read(
|
||||||
|
root_text: str,
|
||||||
|
signature: tuple[int, ...],
|
||||||
|
) -> SimpleNamespace:
|
||||||
|
assert root_text == str(candidate.resolve())
|
||||||
|
assert signature
|
||||||
|
return result
|
||||||
|
|
||||||
|
monkeypatch.setattr(advanced_api, "_read_e38_cached", fake_read)
|
||||||
|
router = build_advanced_laboratory_router(
|
||||||
|
e38_root_provider=lambda: root,
|
||||||
|
)
|
||||||
|
route = _endpoint(router, "/api/v1/laboratory/e38/results")
|
||||||
|
catalog = route(limit=1) # type: ignore[operator]
|
||||||
|
|
||||||
|
assert catalog["candidate_total"] == 1
|
||||||
|
assert catalog["invalid_total"] == 0
|
||||||
|
item = catalog["items"][0]
|
||||||
|
assert item["quality_gate_passed"] is False
|
||||||
|
assert item["metrics"]["dimensions"]["freshness"]["passed"] is True
|
||||||
|
assert item["metrics"]["dimensions"]["presence"]["passed"] is False
|
||||||
|
assert item["authority"] == authority
|
||||||
|
assert item["access"] == "read-only"
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from k1link.compute.e38_perception_baseline import (
|
||||||
|
_dimension_metrics,
|
||||||
|
_predict_tree,
|
||||||
|
_train_tree,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_e38_cart_is_deterministic_and_respects_minimum_leaf() -> None:
|
||||||
|
rows = [
|
||||||
|
({"score": 0.1, "support": 0.0}, "background"),
|
||||||
|
({"score": 0.2, "support": 1.0}, "background"),
|
||||||
|
({"score": 0.7, "support": 3.0}, "object"),
|
||||||
|
({"score": 0.8, "support": 4.0}, "object"),
|
||||||
|
({"score": 0.9, "support": 5.0}, "object"),
|
||||||
|
({"score": 1.0, "support": 6.0}, "object"),
|
||||||
|
]
|
||||||
|
first = _train_tree(
|
||||||
|
rows,
|
||||||
|
feature_names=["score", "support"],
|
||||||
|
max_depth=3,
|
||||||
|
min_leaf=2,
|
||||||
|
)
|
||||||
|
second = _train_tree(
|
||||||
|
rows,
|
||||||
|
feature_names=["score", "support"],
|
||||||
|
max_depth=3,
|
||||||
|
min_leaf=2,
|
||||||
|
)
|
||||||
|
assert first == second
|
||||||
|
assert _predict_tree(first, {"score": 0.15, "support": 1.0}) == "background"
|
||||||
|
assert _predict_tree(first, {"score": 0.85, "support": 4.0}) == "object"
|
||||||
|
|
||||||
|
|
||||||
|
def test_e38_dimension_metrics_do_not_blend_strata() -> None:
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
"source_stratum": "agree",
|
||||||
|
"prediction": {"presence": "object-present"},
|
||||||
|
"reference": {"presence": "object-present"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_stratum": "agree",
|
||||||
|
"prediction": {"presence": "object-present"},
|
||||||
|
"reference": {"presence": "background-or-noise"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_stratum": "geometry-only",
|
||||||
|
"prediction": {"presence": "occupied-environment"},
|
||||||
|
"reference": {"presence": "occupied-environment"},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
metrics = _dimension_metrics(rows, dimension="presence", target=0.9)
|
||||||
|
assert metrics["correct"] == 2
|
||||||
|
assert metrics["incorrect"] == 1
|
||||||
|
assert metrics["accuracy"] == 0.666667
|
||||||
|
assert metrics["passed"] is False
|
||||||
|
assert metrics["by_stratum"]["agree"]["accuracy"] == 0.5
|
||||||
|
assert metrics["by_stratum"]["geometry-only"]["accuracy"] == 1.0
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _module() -> object:
|
||||||
|
path = (
|
||||||
|
Path(__file__).resolve().parents[1]
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "prepare_e38_worker_package.py"
|
||||||
|
)
|
||||||
|
spec = importlib.util.spec_from_file_location("e38_worker_package_test", path)
|
||||||
|
assert spec is not None and spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_e38_package_is_minimal_content_addressed_projection(tmp_path: Path) -> None:
|
||||||
|
module = _module()
|
||||||
|
repository = Path(__file__).resolve().parents[1]
|
||||||
|
acceptance = (
|
||||||
|
repository
|
||||||
|
/ ".runtime"
|
||||||
|
/ "compute-experiments"
|
||||||
|
/ "e37"
|
||||||
|
/ "results"
|
||||||
|
/ (
|
||||||
|
"e37-ravnoves-acceptance-"
|
||||||
|
"01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
materialization = (
|
||||||
|
repository
|
||||||
|
/ ".runtime"
|
||||||
|
/ "compute-experiments"
|
||||||
|
/ "e30"
|
||||||
|
/ "materializations"
|
||||||
|
/ (
|
||||||
|
"e30-materialization-"
|
||||||
|
"841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
package = module.build_e38_worker_package(
|
||||||
|
repository_root=repository,
|
||||||
|
acceptance_root=acceptance,
|
||||||
|
materialization_root=materialization,
|
||||||
|
profile_path=(
|
||||||
|
repository
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "e38_ravnoves00_r1_baseline_profile.json"
|
||||||
|
),
|
||||||
|
output_root=tmp_path,
|
||||||
|
)
|
||||||
|
manifest = module.validate_e38_worker_package(package)
|
||||||
|
assert package.name == f"e38-worker-package-{manifest['identity_sha256']}"
|
||||||
|
assert manifest["identity"]["classification"] == (
|
||||||
|
"immutable-ravnoves00-r1-worker-input"
|
||||||
|
)
|
||||||
|
assert len(manifest["artifacts"]) == 13
|
||||||
|
completed = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
str(package / "runtime" / "run_e38_perception_baseline.py"),
|
||||||
|
"--package",
|
||||||
|
str(package),
|
||||||
|
"--output-root",
|
||||||
|
str(tmp_path / "results"),
|
||||||
|
],
|
||||||
|
env={
|
||||||
|
"PYTHONPATH": str(package / "runtime"),
|
||||||
|
"PYTHONDONTWRITEBYTECODE": "1",
|
||||||
|
"E38_WORKER_NODE": "TEST-WORKER-006",
|
||||||
|
},
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert completed.returncode == 0, completed.stderr
|
||||||
|
assert '"quality_gate_passed": false' in completed.stdout
|
||||||
Reference in New Issue
Block a user