feat(lab): measure RAVNOVES00 R1 quality baseline
This commit is contained in:
DCCONSTRUCTIONS
@@ -132,6 +132,53 @@ export interface E37AcceptanceContractResult {
|
||||
access: "read-only";
|
||||
}
|
||||
|
||||
export interface E38DimensionMetric {
|
||||
correct: number;
|
||||
incorrect: number;
|
||||
total: number;
|
||||
accuracy: number;
|
||||
target: number;
|
||||
passed: boolean;
|
||||
byStratum: Readonly<Record<string, {
|
||||
correct: number;
|
||||
incorrect: number;
|
||||
total: number;
|
||||
accuracy: number;
|
||||
}>>;
|
||||
confusion: readonly {
|
||||
reference: string;
|
||||
prediction: string;
|
||||
count: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface E38PerceptionBaselineResult {
|
||||
resultId: string;
|
||||
createdAtUtc: string | null;
|
||||
sourceSessionId: string;
|
||||
sourceDisplayName: string;
|
||||
status: "measured-r1-source-scoped-baseline";
|
||||
profileId: string;
|
||||
workerNode: string;
|
||||
qualityGatePassed: boolean;
|
||||
metrics: {
|
||||
developmentItems: number;
|
||||
validationItems: number;
|
||||
terminalOutcomes: number;
|
||||
accountingFraction: number;
|
||||
falseFreeClaims: number;
|
||||
highSeverityFailures: number;
|
||||
dimensions: {
|
||||
presence: E38DimensionMetric;
|
||||
geometryAssociation: E38DimensionMetric;
|
||||
freshness: E38DimensionMetric;
|
||||
};
|
||||
};
|
||||
blockingChecks: readonly string[];
|
||||
limitations: readonly string[];
|
||||
access: "read-only";
|
||||
}
|
||||
|
||||
export interface AdvancedLaboratoryResults {
|
||||
e31: E31LaboratoryResult | null;
|
||||
e32: E32LaboratoryResult | null;
|
||||
@@ -139,6 +186,7 @@ export interface AdvancedLaboratoryResults {
|
||||
e34: E34TemporalLayerResult | null;
|
||||
e35: E35DegradationRecoveryResult | null;
|
||||
e37: E37AcceptanceContractResult | null;
|
||||
e38: E38PerceptionBaselineResult | null;
|
||||
}
|
||||
|
||||
export class AdvancedLaboratoryContractError extends Error {
|
||||
@@ -539,6 +587,147 @@ function parseE37(value: unknown): E37AcceptanceContractResult {
|
||||
};
|
||||
}
|
||||
|
||||
function parseE38Dimension(
|
||||
value: unknown,
|
||||
label: string,
|
||||
): E38DimensionMetric {
|
||||
const source = record(value, label);
|
||||
const byStratumSource = record(source.by_stratum, `${label}.by_stratum`);
|
||||
if (!Array.isArray(source.confusion)) {
|
||||
throw new AdvancedLaboratoryContractError(
|
||||
`${label}.confusion: ожидался массив.`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
correct: integerValue(source.correct, `${label}.correct`),
|
||||
incorrect: integerValue(source.incorrect, `${label}.incorrect`),
|
||||
total: integerValue(source.total, `${label}.total`),
|
||||
accuracy: numberValue(source.accuracy, `${label}.accuracy`),
|
||||
target: numberValue(source.target, `${label}.target`),
|
||||
passed: booleanValue(source.passed, `${label}.passed`),
|
||||
byStratum: Object.fromEntries(
|
||||
Object.entries(byStratumSource).map(([stratum, value]) => {
|
||||
const metric = record(value, `${label}.by_stratum.${stratum}`);
|
||||
return [
|
||||
stratum,
|
||||
{
|
||||
correct: integerValue(
|
||||
metric.correct,
|
||||
`${label}.by_stratum.${stratum}.correct`,
|
||||
),
|
||||
incorrect: integerValue(
|
||||
metric.incorrect,
|
||||
`${label}.by_stratum.${stratum}.incorrect`,
|
||||
),
|
||||
total: integerValue(
|
||||
metric.total,
|
||||
`${label}.by_stratum.${stratum}.total`,
|
||||
),
|
||||
accuracy: numberValue(
|
||||
metric.accuracy,
|
||||
`${label}.by_stratum.${stratum}.accuracy`,
|
||||
),
|
||||
},
|
||||
];
|
||||
}),
|
||||
),
|
||||
confusion: source.confusion.map((value, index) => {
|
||||
const row = record(value, `${label}.confusion[${index}]`);
|
||||
return {
|
||||
reference: stringValue(
|
||||
row.reference,
|
||||
`${label}.confusion[${index}].reference`,
|
||||
),
|
||||
prediction: stringValue(
|
||||
row.prediction,
|
||||
`${label}.confusion[${index}].prediction`,
|
||||
),
|
||||
count: integerValue(
|
||||
row.count,
|
||||
`${label}.confusion[${index}].count`,
|
||||
),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function parseE38(value: unknown): E38PerceptionBaselineResult {
|
||||
const item = record(value, "E38");
|
||||
const metrics = record(item.metrics, "E38.metrics");
|
||||
const dimensions = record(metrics.dimensions, "E38.metrics.dimensions");
|
||||
diagnosticAuthority(item.authority, "E38.authority");
|
||||
return {
|
||||
resultId: contentId(
|
||||
item.result_id,
|
||||
"e38-perception-baseline",
|
||||
"E38.result_id",
|
||||
),
|
||||
createdAtUtc: optionalString(item.created_at_utc, "E38.created_at_utc"),
|
||||
sourceSessionId: stringValue(
|
||||
item.source_session_id,
|
||||
"E38.source_session_id",
|
||||
),
|
||||
sourceDisplayName: stringValue(
|
||||
item.source_display_name,
|
||||
"E38.source_display_name",
|
||||
),
|
||||
status: exactString(
|
||||
item.status,
|
||||
"measured-r1-source-scoped-baseline",
|
||||
"E38.status",
|
||||
),
|
||||
profileId: stringValue(item.profile_id, "E38.profile_id"),
|
||||
workerNode: stringValue(item.worker_node, "E38.worker_node"),
|
||||
qualityGatePassed: booleanValue(
|
||||
item.quality_gate_passed,
|
||||
"E38.quality_gate_passed",
|
||||
),
|
||||
metrics: {
|
||||
developmentItems: integerValue(
|
||||
metrics.development_items,
|
||||
"E38.metrics.development_items",
|
||||
),
|
||||
validationItems: integerValue(
|
||||
metrics.validation_items,
|
||||
"E38.metrics.validation_items",
|
||||
),
|
||||
terminalOutcomes: integerValue(
|
||||
metrics.terminal_outcomes,
|
||||
"E38.metrics.terminal_outcomes",
|
||||
),
|
||||
accountingFraction: numberValue(
|
||||
metrics.accounting_fraction,
|
||||
"E38.metrics.accounting_fraction",
|
||||
),
|
||||
falseFreeClaims: integerValue(
|
||||
metrics.false_free_claims,
|
||||
"E38.metrics.false_free_claims",
|
||||
),
|
||||
highSeverityFailures: integerValue(
|
||||
metrics.high_severity_failures,
|
||||
"E38.metrics.high_severity_failures",
|
||||
),
|
||||
dimensions: {
|
||||
presence: parseE38Dimension(
|
||||
dimensions.presence,
|
||||
"E38.metrics.dimensions.presence",
|
||||
),
|
||||
geometryAssociation: parseE38Dimension(
|
||||
dimensions.geometry_association,
|
||||
"E38.metrics.dimensions.geometry_association",
|
||||
),
|
||||
freshness: parseE38Dimension(
|
||||
dimensions.freshness,
|
||||
"E38.metrics.dimensions.freshness",
|
||||
),
|
||||
},
|
||||
},
|
||||
blockingChecks: strings(item.blocking_checks, "E38.blocking_checks"),
|
||||
limitations: strings(item.limitations, "E38.limitations"),
|
||||
access: exactString(item.access, "read-only", "E38.access"),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchOne<T>(
|
||||
path: string,
|
||||
parser: (value: unknown) => T,
|
||||
@@ -563,15 +752,16 @@ export async function fetchAdvancedLaboratoryResults({
|
||||
fetcher?: LaboratoryFetch;
|
||||
signal?: AbortSignal;
|
||||
} = {}): Promise<AdvancedLaboratoryResults> {
|
||||
const [e31, e32, e33, e34, e35, e37] = await Promise.all([
|
||||
const [e31, e32, e33, e34, e35, e37, e38] = await Promise.all([
|
||||
fetchOne("/api/v1/laboratory/e31/results?limit=1", parseE31, fetcher, signal),
|
||||
fetchOne("/api/v1/laboratory/e32/results?limit=1", parseE32, fetcher, signal),
|
||||
fetchOne("/api/v1/laboratory/e33/results?limit=1", parseE33, fetcher, signal),
|
||||
fetchE34TemporalLayerResult({ fetcher, signal }),
|
||||
fetchE35DegradationRecoveryResult({ fetcher, signal }),
|
||||
fetchOne("/api/v1/laboratory/e37/results?limit=1", parseE37, fetcher, signal),
|
||||
fetchOne("/api/v1/laboratory/e38/results?limit=1", parseE38, fetcher, signal),
|
||||
]);
|
||||
return { e31, e32, e33, e34, e35, e37 };
|
||||
return { e31, e32, e33, e34, e35, e37, e38 };
|
||||
}
|
||||
import {
|
||||
fetchE34TemporalLayerResult,
|
||||
|
||||
@@ -10,6 +10,7 @@ import { E33Result } from "./E33Result";
|
||||
import { E34Result } from "./E34Result";
|
||||
import { E35Result } from "./E35Result";
|
||||
import { E37Result } from "./E37Result";
|
||||
import { E38Result } from "./E38Result";
|
||||
import { RecordedReplayEvidence } from "./RecordedReplayEvidence";
|
||||
|
||||
export type AdvancedLaboratoryWorkId =
|
||||
@@ -18,7 +19,8 @@ export type AdvancedLaboratoryWorkId =
|
||||
| "e33-worker-shadow"
|
||||
| "e34-temporal-layer"
|
||||
| "e35-degradation-recovery"
|
||||
| "e37-ravnoves-acceptance";
|
||||
| "e37-ravnoves-acceptance"
|
||||
| "e38-perception-baseline";
|
||||
|
||||
type LaboratoryWorkspaceProps = WorkspaceRendererProps & {
|
||||
SpatialView: ComponentType<WorkspaceRendererProps>;
|
||||
@@ -34,6 +36,7 @@ export function isAdvancedLaboratoryWorkId(
|
||||
|| value === "e34-temporal-layer"
|
||||
|| value === "e35-degradation-recovery"
|
||||
|| value === "e37-ravnoves-acceptance"
|
||||
|| value === "e38-perception-baseline"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -78,6 +81,12 @@ export function advancedLaboratoryWorkOptions(
|
||||
label: "LAB E37 · RAVNOVES00 acceptance R0",
|
||||
});
|
||||
}
|
||||
if (results.e38) {
|
||||
options.push({
|
||||
id: "e38-perception-baseline",
|
||||
label: "LAB E38 · perception quality R1",
|
||||
});
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
@@ -115,6 +124,9 @@ export function AdvancedLaboratoryResult({
|
||||
failedSessionId: string | null;
|
||||
replayError: string | null;
|
||||
}) {
|
||||
if (workId === "e38-perception-baseline" && results.e38) {
|
||||
return <E38Result rigLabel={rigLabel} result={results.e38} />;
|
||||
}
|
||||
if (workId === "e37-ravnoves-acceptance" && results.e37) {
|
||||
return <E37Result rigLabel={rigLabel} result={results.e37} />;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import {
|
||||
LaboratoryEvidence,
|
||||
LaboratoryResultSummary,
|
||||
LaboratorySummary,
|
||||
LaboratoryWorkTemplate,
|
||||
} from "../../components/laboratory/LaboratoryPresentation";
|
||||
import type {
|
||||
E38DimensionMetric,
|
||||
E38PerceptionBaselineResult,
|
||||
} from "../../core/laboratory/advancedResults";
|
||||
import { formatNumber } from "../../presentation";
|
||||
|
||||
function percent(value: number): string {
|
||||
return `${(value * 100).toLocaleString("ru-RU", {
|
||||
maximumFractionDigits: 1,
|
||||
})}%`;
|
||||
}
|
||||
|
||||
function worstStratum(metric: E38DimensionMetric): string {
|
||||
const [name, value] = Object.entries(metric.byStratum)
|
||||
.sort((left, right) => left[1].accuracy - right[1].accuracy)[0] ?? [];
|
||||
return name && value
|
||||
? `${name}: ${percent(value.accuracy)}`
|
||||
: "нет данных";
|
||||
}
|
||||
|
||||
function DimensionEvidence({
|
||||
label,
|
||||
metric,
|
||||
}: {
|
||||
label: string;
|
||||
metric: E38DimensionMetric;
|
||||
}) {
|
||||
return (
|
||||
<div>
|
||||
<span>{label}</span>
|
||||
<strong>{percent(metric.accuracy)}</strong>
|
||||
<small>
|
||||
{`${formatNumber(metric.correct, 0)} / ${formatNumber(metric.total, 0)}`}
|
||||
{" · слабее всего: "}
|
||||
{worstStratum(metric)}
|
||||
</small>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function E38Result({
|
||||
rigLabel,
|
||||
result,
|
||||
}: {
|
||||
rigLabel: string;
|
||||
result: E38PerceptionBaselineResult;
|
||||
}) {
|
||||
const metrics = result.metrics;
|
||||
const dimensions = metrics.dimensions;
|
||||
const gateLabel = result.qualityGatePassed
|
||||
? "R1 gate пройден"
|
||||
: "R1 требует улучшения";
|
||||
return (
|
||||
<LaboratoryWorkTemplate
|
||||
summary={(
|
||||
<LaboratorySummary
|
||||
title="LAB E38 · первый baseline качества R1"
|
||||
description="На неизменяемом validation-наборе E37 отдельно измерены распознавание присутствия, принадлежность геометрии и свежесть состояния. Модель обучалась только на development-разделе; validation использован только для итоговой оценки."
|
||||
status={gateLabel}
|
||||
statusTone={result.qualityGatePassed ? "success" : "warning"}
|
||||
facts={[
|
||||
{
|
||||
label: "Конфигурация",
|
||||
value: `${rigLabel} · ${result.sourceDisplayName}`,
|
||||
},
|
||||
{
|
||||
label: "Обучение",
|
||||
value: `${formatNumber(metrics.developmentItems, 0)} development`,
|
||||
},
|
||||
{
|
||||
label: "Проверка",
|
||||
value: `${formatNumber(metrics.validationItems, 0)} validation`,
|
||||
},
|
||||
{
|
||||
label: "Исполнение",
|
||||
value: `Worker 006 · ${result.workerNode}`,
|
||||
},
|
||||
]}
|
||||
brief={{
|
||||
question: "Какое качество уже даёт текущий source-scoped сенсорный контур на замороженном holdout и какие узлы реально не дотягивают до 90%?",
|
||||
approach: "Три неглубоких детерминированных CART-модели обучены на 340 development-кейсах по фактическим признакам E29/E30. Замороженные 146 validation-кейсов не участвовали в подборе деревьев и использованы только для оценки.",
|
||||
principalResult: `Freshness достиг ${percent(dimensions.freshness.accuracy)} и прошёл порог. Presence — ${percent(dimensions.presence.accuracy)}, geometry association — ${percent(dimensions.geometryAssociation.accuracy)}; оба узла пока ниже 90%.`,
|
||||
limitation: "Это инженерно размеченный baseline одного RAVNOVES00, а не независимый ground truth или доказательство переноса на другой маршрут, риг и камеру.",
|
||||
}}
|
||||
method={{
|
||||
completeness: "complete",
|
||||
executionClass: "deterministic",
|
||||
pipelineId: result.profileId,
|
||||
components: [
|
||||
{
|
||||
kind: "source",
|
||||
name: "E37 frozen acceptance contract",
|
||||
version: `${formatNumber(metrics.validationItems, 0)} sealed validation items`,
|
||||
role: "неизменяемый denominator и три раздельные reference-метрики",
|
||||
identitySha256: null,
|
||||
},
|
||||
{
|
||||
kind: "algorithm",
|
||||
name: "Development-only shallow CART",
|
||||
version: "dimension-specific deterministic trees",
|
||||
role: "baseline без обучения на validation и без смешивания метрик",
|
||||
identitySha256: null,
|
||||
},
|
||||
{
|
||||
kind: "runtime",
|
||||
name: "Worker 006",
|
||||
version: result.workerNode,
|
||||
role: "изолированное исполнение pinned package без командных полномочий",
|
||||
identitySha256: null,
|
||||
},
|
||||
],
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
evidence={(
|
||||
<LaboratoryEvidence
|
||||
eyebrow="ЗАМОРОЖЕННЫЙ VALIDATION"
|
||||
title="Три независимые метрики и слабые страты"
|
||||
kind="diagnostic-model"
|
||||
>
|
||||
<div className="laboratory-result-metrics">
|
||||
<DimensionEvidence
|
||||
label="Presence"
|
||||
metric={dimensions.presence}
|
||||
/>
|
||||
<DimensionEvidence
|
||||
label="Geometry association"
|
||||
metric={dimensions.geometryAssociation}
|
||||
/>
|
||||
<DimensionEvidence
|
||||
label="Freshness"
|
||||
metric={dimensions.freshness}
|
||||
/>
|
||||
<div>
|
||||
<span>High severity</span>
|
||||
<strong>{formatNumber(metrics.highSeverityFailures, 0)}</strong>
|
||||
<small>
|
||||
{"блокирующие ошибки независимо от среднего процента"}
|
||||
</small>
|
||||
</div>
|
||||
</div>
|
||||
</LaboratoryEvidence>
|
||||
)}
|
||||
result={(
|
||||
<LaboratoryResultSummary
|
||||
title={result.qualityGatePassed
|
||||
? "Все три метрики достигли source-scoped цели R1"
|
||||
: "Baseline измерен; presence и geometry остаются открыты"}
|
||||
status={gateLabel}
|
||||
statusTone={result.qualityGatePassed ? "success" : "warning"}
|
||||
metrics={[
|
||||
{
|
||||
label: "Presence",
|
||||
value: percent(dimensions.presence.accuracy),
|
||||
hint: `${formatNumber(dimensions.presence.incorrect, 0)} ошибок · цель 90%`,
|
||||
},
|
||||
{
|
||||
label: "Geometry association",
|
||||
value: percent(dimensions.geometryAssociation.accuracy),
|
||||
hint: `${formatNumber(dimensions.geometryAssociation.incorrect, 0)} ошибок · цель 90%`,
|
||||
},
|
||||
{
|
||||
label: "Freshness",
|
||||
value: percent(dimensions.freshness.accuracy),
|
||||
hint: `${formatNumber(dimensions.freshness.incorrect, 0)} ошибок · цель 90%`,
|
||||
},
|
||||
{
|
||||
label: "Учёт / false free",
|
||||
value: `${percent(metrics.accountingFraction)} / ${formatNumber(metrics.falseFreeClaims, 0)}`,
|
||||
hint: `${formatNumber(metrics.terminalOutcomes, 0)} terminal outcomes`,
|
||||
},
|
||||
]}
|
||||
conclusion={{
|
||||
proved: `На sealed validation текущая логика честно закрывает freshness с ${percent(dimensions.freshness.accuracy)}, сохраняет 100% учёта и не публикует false-free claims.`,
|
||||
notProved: `Presence (${percent(dimensions.presence.accuracy)}) и geometry association (${percent(dimensions.geometryAssociation.accuracy)}) не достигли 90%; зафиксировано ${formatNumber(metrics.highSeverityFailures, 0)} high-severity ошибок. R1 и выпуск не приняты.`,
|
||||
decision: "Сохранить E38 как первый измеренный baseline. Следующую итерацию R1 направить на detector/background decision и различение class-bearing объекта от независимой geometry-only среды, не меняя validation.",
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -70,6 +70,7 @@ const EMPTY_ADVANCED_RESULTS: AdvancedLaboratoryResults = {
|
||||
e34: null,
|
||||
e35: null,
|
||||
e37: null,
|
||||
e38: null,
|
||||
};
|
||||
|
||||
function laboratoryWorkOrdinal(value: string): number {
|
||||
|
||||
Reference in New Issue
Block a user