feat(lab): measure RAVNOVES00 R1 quality baseline

This commit is contained in:
DCCONSTRUCTIONS
2026-07-28 03:42:32 +03:00
parent e35f81ac34
commit c018f2b074
20 changed files with 2401 additions and 11 deletions
@@ -132,6 +132,53 @@ export interface E37AcceptanceContractResult {
access: "read-only";
}
export interface E38DimensionMetric {
correct: number;
incorrect: number;
total: number;
accuracy: number;
target: number;
passed: boolean;
byStratum: Readonly<Record<string, {
correct: number;
incorrect: number;
total: number;
accuracy: number;
}>>;
confusion: readonly {
reference: string;
prediction: string;
count: number;
}[];
}
export interface E38PerceptionBaselineResult {
resultId: string;
createdAtUtc: string | null;
sourceSessionId: string;
sourceDisplayName: string;
status: "measured-r1-source-scoped-baseline";
profileId: string;
workerNode: string;
qualityGatePassed: boolean;
metrics: {
developmentItems: number;
validationItems: number;
terminalOutcomes: number;
accountingFraction: number;
falseFreeClaims: number;
highSeverityFailures: number;
dimensions: {
presence: E38DimensionMetric;
geometryAssociation: E38DimensionMetric;
freshness: E38DimensionMetric;
};
};
blockingChecks: readonly string[];
limitations: readonly string[];
access: "read-only";
}
export interface AdvancedLaboratoryResults {
e31: E31LaboratoryResult | null;
e32: E32LaboratoryResult | null;
@@ -139,6 +186,7 @@ export interface AdvancedLaboratoryResults {
e34: E34TemporalLayerResult | null;
e35: E35DegradationRecoveryResult | null;
e37: E37AcceptanceContractResult | null;
e38: E38PerceptionBaselineResult | null;
}
export class AdvancedLaboratoryContractError extends Error {
@@ -539,6 +587,147 @@ function parseE37(value: unknown): E37AcceptanceContractResult {
};
}
function parseE38Dimension(
value: unknown,
label: string,
): E38DimensionMetric {
const source = record(value, label);
const byStratumSource = record(source.by_stratum, `${label}.by_stratum`);
if (!Array.isArray(source.confusion)) {
throw new AdvancedLaboratoryContractError(
`${label}.confusion: ожидался массив.`,
);
}
return {
correct: integerValue(source.correct, `${label}.correct`),
incorrect: integerValue(source.incorrect, `${label}.incorrect`),
total: integerValue(source.total, `${label}.total`),
accuracy: numberValue(source.accuracy, `${label}.accuracy`),
target: numberValue(source.target, `${label}.target`),
passed: booleanValue(source.passed, `${label}.passed`),
byStratum: Object.fromEntries(
Object.entries(byStratumSource).map(([stratum, value]) => {
const metric = record(value, `${label}.by_stratum.${stratum}`);
return [
stratum,
{
correct: integerValue(
metric.correct,
`${label}.by_stratum.${stratum}.correct`,
),
incorrect: integerValue(
metric.incorrect,
`${label}.by_stratum.${stratum}.incorrect`,
),
total: integerValue(
metric.total,
`${label}.by_stratum.${stratum}.total`,
),
accuracy: numberValue(
metric.accuracy,
`${label}.by_stratum.${stratum}.accuracy`,
),
},
];
}),
),
confusion: source.confusion.map((value, index) => {
const row = record(value, `${label}.confusion[${index}]`);
return {
reference: stringValue(
row.reference,
`${label}.confusion[${index}].reference`,
),
prediction: stringValue(
row.prediction,
`${label}.confusion[${index}].prediction`,
),
count: integerValue(
row.count,
`${label}.confusion[${index}].count`,
),
};
}),
};
}
function parseE38(value: unknown): E38PerceptionBaselineResult {
const item = record(value, "E38");
const metrics = record(item.metrics, "E38.metrics");
const dimensions = record(metrics.dimensions, "E38.metrics.dimensions");
diagnosticAuthority(item.authority, "E38.authority");
return {
resultId: contentId(
item.result_id,
"e38-perception-baseline",
"E38.result_id",
),
createdAtUtc: optionalString(item.created_at_utc, "E38.created_at_utc"),
sourceSessionId: stringValue(
item.source_session_id,
"E38.source_session_id",
),
sourceDisplayName: stringValue(
item.source_display_name,
"E38.source_display_name",
),
status: exactString(
item.status,
"measured-r1-source-scoped-baseline",
"E38.status",
),
profileId: stringValue(item.profile_id, "E38.profile_id"),
workerNode: stringValue(item.worker_node, "E38.worker_node"),
qualityGatePassed: booleanValue(
item.quality_gate_passed,
"E38.quality_gate_passed",
),
metrics: {
developmentItems: integerValue(
metrics.development_items,
"E38.metrics.development_items",
),
validationItems: integerValue(
metrics.validation_items,
"E38.metrics.validation_items",
),
terminalOutcomes: integerValue(
metrics.terminal_outcomes,
"E38.metrics.terminal_outcomes",
),
accountingFraction: numberValue(
metrics.accounting_fraction,
"E38.metrics.accounting_fraction",
),
falseFreeClaims: integerValue(
metrics.false_free_claims,
"E38.metrics.false_free_claims",
),
highSeverityFailures: integerValue(
metrics.high_severity_failures,
"E38.metrics.high_severity_failures",
),
dimensions: {
presence: parseE38Dimension(
dimensions.presence,
"E38.metrics.dimensions.presence",
),
geometryAssociation: parseE38Dimension(
dimensions.geometry_association,
"E38.metrics.dimensions.geometry_association",
),
freshness: parseE38Dimension(
dimensions.freshness,
"E38.metrics.dimensions.freshness",
),
},
},
blockingChecks: strings(item.blocking_checks, "E38.blocking_checks"),
limitations: strings(item.limitations, "E38.limitations"),
access: exactString(item.access, "read-only", "E38.access"),
};
}
async function fetchOne<T>(
path: string,
parser: (value: unknown) => T,
@@ -563,15 +752,16 @@ export async function fetchAdvancedLaboratoryResults({
fetcher?: LaboratoryFetch;
signal?: AbortSignal;
} = {}): Promise<AdvancedLaboratoryResults> {
const [e31, e32, e33, e34, e35, e37] = await Promise.all([
const [e31, e32, e33, e34, e35, e37, e38] = await Promise.all([
fetchOne("/api/v1/laboratory/e31/results?limit=1", parseE31, fetcher, signal),
fetchOne("/api/v1/laboratory/e32/results?limit=1", parseE32, fetcher, signal),
fetchOne("/api/v1/laboratory/e33/results?limit=1", parseE33, fetcher, signal),
fetchE34TemporalLayerResult({ fetcher, signal }),
fetchE35DegradationRecoveryResult({ fetcher, signal }),
fetchOne("/api/v1/laboratory/e37/results?limit=1", parseE37, fetcher, signal),
fetchOne("/api/v1/laboratory/e38/results?limit=1", parseE38, fetcher, signal),
]);
return { e31, e32, e33, e34, e35, e37 };
return { e31, e32, e33, e34, e35, e37, e38 };
}
import {
fetchE34TemporalLayerResult,
@@ -10,6 +10,7 @@ import { E33Result } from "./E33Result";
import { E34Result } from "./E34Result";
import { E35Result } from "./E35Result";
import { E37Result } from "./E37Result";
import { E38Result } from "./E38Result";
import { RecordedReplayEvidence } from "./RecordedReplayEvidence";
export type AdvancedLaboratoryWorkId =
@@ -18,7 +19,8 @@ export type AdvancedLaboratoryWorkId =
| "e33-worker-shadow"
| "e34-temporal-layer"
| "e35-degradation-recovery"
| "e37-ravnoves-acceptance";
| "e37-ravnoves-acceptance"
| "e38-perception-baseline";
type LaboratoryWorkspaceProps = WorkspaceRendererProps & {
SpatialView: ComponentType<WorkspaceRendererProps>;
@@ -34,6 +36,7 @@ export function isAdvancedLaboratoryWorkId(
|| value === "e34-temporal-layer"
|| value === "e35-degradation-recovery"
|| value === "e37-ravnoves-acceptance"
|| value === "e38-perception-baseline"
);
}
@@ -78,6 +81,12 @@ export function advancedLaboratoryWorkOptions(
label: "LAB E37 · RAVNOVES00 acceptance R0",
});
}
if (results.e38) {
options.push({
id: "e38-perception-baseline",
label: "LAB E38 · perception quality R1",
});
}
return options;
}
@@ -115,6 +124,9 @@ export function AdvancedLaboratoryResult({
failedSessionId: string | null;
replayError: string | null;
}) {
if (workId === "e38-perception-baseline" && results.e38) {
return <E38Result rigLabel={rigLabel} result={results.e38} />;
}
if (workId === "e37-ravnoves-acceptance" && results.e37) {
return <E37Result rigLabel={rigLabel} result={results.e37} />;
}
@@ -0,0 +1,188 @@
import {
LaboratoryEvidence,
LaboratoryResultSummary,
LaboratorySummary,
LaboratoryWorkTemplate,
} from "../../components/laboratory/LaboratoryPresentation";
import type {
E38DimensionMetric,
E38PerceptionBaselineResult,
} from "../../core/laboratory/advancedResults";
import { formatNumber } from "../../presentation";
function percent(value: number): string {
return `${(value * 100).toLocaleString("ru-RU", {
maximumFractionDigits: 1,
})}%`;
}
function worstStratum(metric: E38DimensionMetric): string {
const [name, value] = Object.entries(metric.byStratum)
.sort((left, right) => left[1].accuracy - right[1].accuracy)[0] ?? [];
return name && value
? `${name}: ${percent(value.accuracy)}`
: "нет данных";
}
function DimensionEvidence({
label,
metric,
}: {
label: string;
metric: E38DimensionMetric;
}) {
return (
<div>
<span>{label}</span>
<strong>{percent(metric.accuracy)}</strong>
<small>
{`${formatNumber(metric.correct, 0)} / ${formatNumber(metric.total, 0)}`}
{" · слабее всего: "}
{worstStratum(metric)}
</small>
</div>
);
}
export function E38Result({
rigLabel,
result,
}: {
rigLabel: string;
result: E38PerceptionBaselineResult;
}) {
const metrics = result.metrics;
const dimensions = metrics.dimensions;
const gateLabel = result.qualityGatePassed
? "R1 gate пройден"
: "R1 требует улучшения";
return (
<LaboratoryWorkTemplate
summary={(
<LaboratorySummary
title="LAB E38 · первый baseline качества R1"
description="На неизменяемом validation-наборе E37 отдельно измерены распознавание присутствия, принадлежность геометрии и свежесть состояния. Модель обучалась только на development-разделе; validation использован только для итоговой оценки."
status={gateLabel}
statusTone={result.qualityGatePassed ? "success" : "warning"}
facts={[
{
label: "Конфигурация",
value: `${rigLabel} · ${result.sourceDisplayName}`,
},
{
label: "Обучение",
value: `${formatNumber(metrics.developmentItems, 0)} development`,
},
{
label: "Проверка",
value: `${formatNumber(metrics.validationItems, 0)} validation`,
},
{
label: "Исполнение",
value: `Worker 006 · ${result.workerNode}`,
},
]}
brief={{
question: "Какое качество уже даёт текущий source-scoped сенсорный контур на замороженном holdout и какие узлы реально не дотягивают до 90%?",
approach: "Три неглубоких детерминированных CART-модели обучены на 340 development-кейсах по фактическим признакам E29/E30. Замороженные 146 validation-кейсов не участвовали в подборе деревьев и использованы только для оценки.",
principalResult: `Freshness достиг ${percent(dimensions.freshness.accuracy)} и прошёл порог. Presence — ${percent(dimensions.presence.accuracy)}, geometry association — ${percent(dimensions.geometryAssociation.accuracy)}; оба узла пока ниже 90%.`,
limitation: "Это инженерно размеченный baseline одного RAVNOVES00, а не независимый ground truth или доказательство переноса на другой маршрут, риг и камеру.",
}}
method={{
completeness: "complete",
executionClass: "deterministic",
pipelineId: result.profileId,
components: [
{
kind: "source",
name: "E37 frozen acceptance contract",
version: `${formatNumber(metrics.validationItems, 0)} sealed validation items`,
role: "неизменяемый denominator и три раздельные reference-метрики",
identitySha256: null,
},
{
kind: "algorithm",
name: "Development-only shallow CART",
version: "dimension-specific deterministic trees",
role: "baseline без обучения на validation и без смешивания метрик",
identitySha256: null,
},
{
kind: "runtime",
name: "Worker 006",
version: result.workerNode,
role: "изолированное исполнение pinned package без командных полномочий",
identitySha256: null,
},
],
}}
/>
)}
evidence={(
<LaboratoryEvidence
eyebrow="ЗАМОРОЖЕННЫЙ VALIDATION"
title="Три независимые метрики и слабые страты"
kind="diagnostic-model"
>
<div className="laboratory-result-metrics">
<DimensionEvidence
label="Presence"
metric={dimensions.presence}
/>
<DimensionEvidence
label="Geometry association"
metric={dimensions.geometryAssociation}
/>
<DimensionEvidence
label="Freshness"
metric={dimensions.freshness}
/>
<div>
<span>High severity</span>
<strong>{formatNumber(metrics.highSeverityFailures, 0)}</strong>
<small>
{"блокирующие ошибки независимо от среднего процента"}
</small>
</div>
</div>
</LaboratoryEvidence>
)}
result={(
<LaboratoryResultSummary
title={result.qualityGatePassed
? "Все три метрики достигли source-scoped цели R1"
: "Baseline измерен; presence и geometry остаются открыты"}
status={gateLabel}
statusTone={result.qualityGatePassed ? "success" : "warning"}
metrics={[
{
label: "Presence",
value: percent(dimensions.presence.accuracy),
hint: `${formatNumber(dimensions.presence.incorrect, 0)} ошибок · цель 90%`,
},
{
label: "Geometry association",
value: percent(dimensions.geometryAssociation.accuracy),
hint: `${formatNumber(dimensions.geometryAssociation.incorrect, 0)} ошибок · цель 90%`,
},
{
label: "Freshness",
value: percent(dimensions.freshness.accuracy),
hint: `${formatNumber(dimensions.freshness.incorrect, 0)} ошибок · цель 90%`,
},
{
label: "Учёт / false free",
value: `${percent(metrics.accountingFraction)} / ${formatNumber(metrics.falseFreeClaims, 0)}`,
hint: `${formatNumber(metrics.terminalOutcomes, 0)} terminal outcomes`,
},
]}
conclusion={{
proved: `На sealed validation текущая логика честно закрывает freshness с ${percent(dimensions.freshness.accuracy)}, сохраняет 100% учёта и не публикует false-free claims.`,
notProved: `Presence (${percent(dimensions.presence.accuracy)}) и geometry association (${percent(dimensions.geometryAssociation.accuracy)}) не достигли 90%; зафиксировано ${formatNumber(metrics.highSeverityFailures, 0)} high-severity ошибок. R1 и выпуск не приняты.`,
decision: "Сохранить E38 как первый измеренный baseline. Следующую итерацию R1 направить на detector/background decision и различение class-bearing объекта от независимой geometry-only среды, не меняя validation.",
}}
/>
)}
/>
);
}
@@ -70,6 +70,7 @@ const EMPTY_ADVANCED_RESULTS: AdvancedLaboratoryResults = {
e34: null,
e35: null,
e37: null,
e38: null,
};
function laboratoryWorkOrdinal(value: string): number {
@@ -355,6 +355,83 @@ function e37() {
};
}
function e38Dimension({
accuracy,
correct,
incorrect,
passed,
}) {
return {
correct,
incorrect,
total: 146,
accuracy,
target: 0.9,
passed,
by_stratum: {
agree: {
correct: 25,
incorrect: 4,
total: 29,
accuracy: 25 / 29,
},
},
confusion: [{
reference: "object-present",
prediction: "object-present",
count: correct,
}],
};
}
function e38() {
return {
result_id: `e38-perception-baseline-${"8".repeat(64)}`,
created_at_utc: "2026-07-28T02:50:00Z",
source_session_id: "20260720T065719Z_viewer_live",
source_display_name: "RAVNOVES00",
status: "measured-r1-source-scoped-baseline",
profile_id: "e38-ravnoves00-r1-development-cart/v1",
worker_node: "DESKTOP-OPJ8J04",
quality_gate_passed: false,
metrics: {
development_items: 340,
validation_items: 146,
terminal_outcomes: 146,
accounting_fraction: 1,
false_free_claims: 0,
high_severity_failures: 14,
dimensions: {
presence: e38Dimension({
accuracy: 0.821918,
correct: 120,
incorrect: 26,
passed: false,
}),
geometry_association: e38Dimension({
accuracy: 0.815068,
correct: 119,
incorrect: 27,
passed: false,
}),
freshness: e38Dimension({
accuracy: 0.952055,
correct: 139,
incorrect: 7,
passed: true,
}),
},
},
blocking_checks: [
"presence_target_reached",
"geometry_association_target_reached",
],
limitations: ["source-scoped"],
authority,
access: "read-only",
};
}
before(async () => {
server = await createServer({
appType: "custom",
@@ -371,9 +448,9 @@ after(async () => {
await server?.close();
});
test("decodes E31E37 from separate read-only catalogs", async () => {
test("decodes E31E38 from separate read-only catalogs", async () => {
const requests = [];
const items = [e31(), e32(), e33(), e34(), e35(), e37()];
const items = [e31(), e32(), e33(), e34(), e35(), e37(), e38()];
const decoded = await fetchAdvancedLaboratoryResults({
fetcher: async (input, init) => {
requests.push({ input: String(input), method: init?.method });
@@ -396,6 +473,9 @@ test("decodes E31E37 from separate read-only catalogs", async () => {
assert.equal(decoded.e37.metrics.reviewedItems, 486);
assert.equal(decoded.e37.metrics.validationItems, 146);
assert.equal(decoded.e37.qualityTargetEvaluated, false);
assert.equal(decoded.e38.metrics.dimensions.presence.accuracy, 0.821918);
assert.equal(decoded.e38.metrics.dimensions.freshness.passed, true);
assert.equal(decoded.e38.qualityGatePassed, false);
assert.deepEqual(requests, [
{ input: "/api/v1/laboratory/e31/results?limit=1", method: "GET" },
{ input: "/api/v1/laboratory/e32/results?limit=1", method: "GET" },
@@ -403,6 +483,7 @@ test("decodes E31E37 from separate read-only catalogs", async () => {
{ input: "/api/v1/laboratory/e34/results?limit=1", method: "GET" },
{ input: "/api/v1/laboratory/e35/results?limit=1", method: "GET" },
{ input: "/api/v1/laboratory/e37/results?limit=1", method: "GET" },
{ input: "/api/v1/laboratory/e38/results?limit=1", method: "GET" },
]);
});
@@ -418,7 +499,8 @@ test("rejects authority escalation in an accepted-looking result", async () => {
: String(input).includes("/e32/") ? e32()
: String(input).includes("/e33/") ? e33()
: String(input).includes("/e34/") ? e34()
: String(input).includes("/e35/") ? e35() : e37(),
: String(input).includes("/e35/") ? e35()
: String(input).includes("/e37/") ? e37() : e38(),
)), { status: 200 }),
}),
AdvancedLaboratoryContractError,
@@ -30,6 +30,10 @@ const e35ResultUrl = new URL(
"../src/workspaces/laboratory/E35Result.tsx",
import.meta.url,
);
const e38ResultUrl = new URL(
"../src/workspaces/laboratory/E38Result.tsx",
import.meta.url,
);
const e35StylesUrl = new URL(
"../src/styles/e35-degradation-recovery.css",
import.meta.url,
@@ -217,6 +221,24 @@ test("E35 extends the canonical LAB with fault and recovery evidence", async ()
);
});
test("E38 reports the frozen R1 baseline through the canonical LAB anatomy", async () => {
const [e38Source, advancedSource] = await Promise.all([
readFile(e38ResultUrl, "utf8"),
readFile(advancedLaboratoryResultUrl, "utf8"),
]);
assert.match(e38Source, /<LaboratoryWorkTemplate/);
assert.match(e38Source, /<LaboratoryEvidence/);
assert.match(e38Source, /<LaboratoryResultSummary/);
assert.match(e38Source, /validation использован только для итоговой оценки/);
assert.match(e38Source, /Presence —/);
assert.match(e38Source, /R1 и выпуск не приняты/);
assert.match(e38Source, /не меняя validation/);
assert.doesNotMatch(e38Source, /className="laboratory-(?:summary|result-summary)"/);
assert.match(advancedSource, /id: "e38-perception-baseline"/);
assert.match(advancedSource, /<E38Result/);
});
test("the primary point-cloud viewer restores from fullscreen on Escape", async () => {
const workspacesSource = await readFile(workspacesUrl, "utf8");