feat: publish full RELLIS qualification
This commit is contained in:
DCCONSTRUCTIONS
parent
21c7a5507c
commit
d8b7ed4526
|
|
@ -131,6 +131,12 @@ K1 `lio_pcl` remains a post-LIO vendor-map increment and is not promoted to a
|
|||
raw scan. Large dataset bytes are admitted only on the Windows/WSL worker under
|
||||
`D:\NDC_MISSIONCORE\datasets`; no archive is downloaded automatically.
|
||||
|
||||
The complete RELLIS-3D v1.1 release is now admitted there and its full
|
||||
`2,413`-frame validation split is available in **Полигон → Датасеты**. The
|
||||
sealed Current/Patchwork++ comparison rejected Patchwork++ for navigation:
|
||||
despite much lower latency and a small Ground-IoU gain, obstacle non-ground
|
||||
recall regressed from `80.46%` to `69.70%`.
|
||||
|
||||
See [the Dataset Gateway plan](docs/14_LIDAR_DATASET_GATEWAY.md) and
|
||||
[ADR 0021](docs/adr/0021-dataset-gateway-representation-boundary.md).
|
||||
|
||||
|
|
|
|||
|
|
@ -1665,6 +1665,36 @@
|
|||
font-size: 0.82rem;
|
||||
}
|
||||
|
||||
.rellis-review__decision {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
border-radius: 0.9rem;
|
||||
background: rgb(var(--nodedc-danger-rgb) / 0.08);
|
||||
padding: 0.78rem 0.9rem;
|
||||
}
|
||||
|
||||
.rellis-review__decision strong {
|
||||
color: rgb(var(--nodedc-danger-rgb));
|
||||
font-size: 0.68rem;
|
||||
}
|
||||
|
||||
.rellis-review__decision span {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.58rem;
|
||||
line-height: 1.45;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.rellis-review__decision[data-passed="true"] {
|
||||
background: rgb(255 255 255 / 0.045);
|
||||
}
|
||||
|
||||
.rellis-review__decision[data-passed="true"] strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
}
|
||||
|
||||
.polygon-review-summary,
|
||||
.polygon-review-unavailable,
|
||||
.polygon-review-analysis,
|
||||
|
|
|
|||
|
|
@ -10,8 +10,10 @@ import {
|
|||
type LidarGroundViewMode,
|
||||
} from "./LidarGroundPointCloud";
|
||||
import {
|
||||
fetchGroundQualification,
|
||||
fetchGroundReview,
|
||||
fetchGroundReviewFrame,
|
||||
type GroundQualification,
|
||||
type GroundReview,
|
||||
type GroundReviewFrame,
|
||||
type GroundReviewFrameSummary,
|
||||
|
|
@ -324,6 +326,7 @@ function qualifiedModeExplanation(mode: LidarGroundViewMode) {
|
|||
}
|
||||
|
||||
function RellisQualifiedReview({ runId }: { runId: string }) {
|
||||
const [qualification, setQualification] = useState<GroundQualification | null>(null);
|
||||
const [review, setReview] = useState<GroundReview | null>(null);
|
||||
const [frame, setFrame] = useState<GroundReviewFrame | null>(null);
|
||||
const [selectedSequence, setSelectedSequence] = useState<string | null>(null);
|
||||
|
|
@ -336,15 +339,20 @@ function RellisQualifiedReview({ runId }: { runId: string }) {
|
|||
useEffect(() => {
|
||||
const controller = new AbortController();
|
||||
setError(null);
|
||||
setQualification(null);
|
||||
setReview(null);
|
||||
setFrame(null);
|
||||
cache.current.clear();
|
||||
void fetchGroundReview(runId, { signal: controller.signal })
|
||||
.then((value) => {
|
||||
if (controller.signal.aborted || !value) return;
|
||||
setReview(value);
|
||||
setSelectedSequence(value.frames[0]?.datasetSequenceId ?? null);
|
||||
setSelectedFrame(value.frames[0]?.frameId ?? null);
|
||||
void Promise.all([
|
||||
fetchGroundQualification(runId, { signal: controller.signal }),
|
||||
fetchGroundReview(runId, { signal: controller.signal }),
|
||||
])
|
||||
.then(([qualificationValue, reviewValue]) => {
|
||||
if (controller.signal.aborted || !qualificationValue || !reviewValue) return;
|
||||
setQualification(qualificationValue);
|
||||
setReview(reviewValue);
|
||||
setSelectedSequence(reviewValue.frames[0]?.datasetSequenceId ?? null);
|
||||
setSelectedFrame(reviewValue.frames[0]?.frameId ?? null);
|
||||
})
|
||||
.catch((loadError: unknown) => {
|
||||
if (!controller.signal.aborted) setError(errorMessage(loadError));
|
||||
|
|
@ -440,7 +448,7 @@ function RellisQualifiedReview({ runId }: { runId: string }) {
|
|||
<header className="dataset-sequence-review__truth">
|
||||
<div>
|
||||
<span className="section-eyebrow">RELLIS · FULL VALIDATION</span>
|
||||
<h2>Пять последовательностей Ouster OS1 с покадровым сравнением</h2>
|
||||
<h2>Две validation-последовательности Ouster OS1 с покадровым сравнением</h2>
|
||||
<p>
|
||||
Все {review.frameCount.toLocaleString("ru-RU")} validation-сканов
|
||||
выровнены с labels и pose index. Высота Patchwork++ определена только
|
||||
|
|
@ -577,6 +585,26 @@ function RellisQualifiedReview({ runId }: { runId: string }) {
|
|||
</article>
|
||||
</div>
|
||||
|
||||
{qualification ? (
|
||||
<div
|
||||
className="rellis-review__decision"
|
||||
data-passed={qualification.decision.passed ? "true" : "false"}
|
||||
>
|
||||
<strong>
|
||||
{qualification.decision.passed
|
||||
? "Кандидат прошёл независимую проверку"
|
||||
: "Patchwork++ не прошёл независимую проверку"}
|
||||
</strong>
|
||||
<span>
|
||||
Полный validation: obstacle recall{" "}
|
||||
{formatPercent(qualification.current.micro.obstacleNonGroundRecall)}
|
||||
{" → "}
|
||||
{formatPercent(qualification.patchwork.micro.obstacleNonGroundRecall)}.
|
||||
Алгоритм не принят для navigation/safety.
|
||||
</span>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<p className="dataset-sequence-review__raw-note">
|
||||
Это независимая исследовательская проверка переносимости. Она не
|
||||
включает управление машиной и не принимает алгоритм для navigation/safety.
|
||||
|
|
|
|||
|
|
@ -879,10 +879,11 @@ workflow:
|
|||
explicitly grouped into eight independent fragments; Play is bounded to one
|
||||
fragment and described as accelerated review of sparse annotated scans.
|
||||
- The catalog is multi-source. RELLIS-3D v1.1 appears as an independent Ouster
|
||||
domain with its own license and admission state. Its current `smoke-ready`
|
||||
view shows official frame `000104`, semantic colors, FLU axes and the
|
||||
versioned `ground / non-ground / ignore` mapping; it does not show invented
|
||||
algorithm metrics.
|
||||
domain with its own license and admission state. Its complete admitted
|
||||
release exposes the two real validation sequences and all `2,413` scans with
|
||||
public ground truth, Current, Patchwork++ and error modes. Metrics come only
|
||||
from the sealed qualification report and are scoped to the selected
|
||||
sequence.
|
||||
- The dataset viewer exposes source frame number, nanosecond timestamp gaps,
|
||||
ground truth and Current/Patchwork++ overlays.
|
||||
- The fixed sensor-centric coordinate frame and operator camera are preserved
|
||||
|
|
@ -899,11 +900,15 @@ The accepted `3.3 GB` annotated validation ZIP is not a continuous recording.
|
|||
Continuous ego-motion requires a separately admitted raw ROS bag and
|
||||
localization source.
|
||||
|
||||
RELLIS S0 uses only a bounded derivative of the pinned official example for
|
||||
browser inspection. The full `14 GB` Ouster SemanticKITTI scans, labels and
|
||||
poses remain a separate D-only admission gate. Its `CC-BY-NC-SA-3.0` license
|
||||
means cross-dataset research evidence cannot silently become an unrestricted
|
||||
commercial runtime or training dependency.
|
||||
RELLIS has passed the full D-only release-admission gate: `13,556` aligned
|
||||
scan/label pairs, official splits, five source sequences and complete LiDAR
|
||||
pose coverage. The R1 validation run processed all `2,413` frames after a
|
||||
train-only sensor-height calibration. Its Patchwork++ candidate was explicitly
|
||||
rejected: Ground IoU improved by only `1.41` points while obstacle non-ground
|
||||
recall fell from `80.46%` to `69.70%`. The result remains research evidence,
|
||||
not navigation or safety authority. The `CC-BY-NC-SA-3.0` license also prevents
|
||||
the source from silently becoming an unrestricted commercial runtime or
|
||||
training dependency.
|
||||
|
||||
The backend reads the configured repository from
|
||||
`MISSIONCORE_POLYGON_RUNS_ROOT` using
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Implemented now:
|
|||
- path-free `missioncore.dataset-admission/v1` worker evidence instead of a
|
||||
static React status;
|
||||
- bounded `missioncore.dataset-native-scan-preview/v1` (GOOSE) and `/v2`
|
||||
(RELLIS smoke) artifacts, selected by source at
|
||||
(RELLIS) artifacts, selected by source at
|
||||
`GET /api/v1/lidar/dataset-gateway/preview?source_id=...`;
|
||||
- explicit `native-scan`, `normalized-scan` and `rolling-local-map`
|
||||
representations;
|
||||
|
|
@ -40,10 +40,13 @@ Implemented now:
|
|||
`Открыть` action;
|
||||
- one admitted GOOSE validation frame with source colors, normalized remission
|
||||
and an independent ground-truth view;
|
||||
- a pinned RELLIS-3D v1.1 compatibility smoke using the official Ouster example
|
||||
frame `000104`, its native `131,072` XYZI points, point-aligned labels,
|
||||
official ontology colors, explicit FLU sensor axes and a versioned
|
||||
`ground / non-ground / ignore` evaluation policy;
|
||||
- a fail-closed admission of the complete official RELLIS-3D v1.1 release:
|
||||
`13,556` aligned Ouster scans and labels, all five sequences, official
|
||||
train/validation/test splits and complete LiDAR pose coverage;
|
||||
- a full RELLIS validation viewer with its two real source sequences,
|
||||
`2,413` source scans, ground truth, Current, Patchwork++ and error modes;
|
||||
- a train-only, label-backed Ouster height calibration and a sealed R1
|
||||
Current/Patchwork++ qualification over every RELLIS validation frame;
|
||||
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
|
||||
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
|
||||
one-revolution scan and explicit absence of deskew/full vehicle TF claims;
|
||||
|
|
@ -59,8 +62,8 @@ Not implemented:
|
|||
- no implicit coordinate conversion;
|
||||
- no fake ring/timestamp reconstruction for K1 MQTT evidence;
|
||||
- no model training or production promotion;
|
||||
- no full RELLIS Ouster archive, pose set, algorithm comparison or ROS bag
|
||||
admission yet;
|
||||
- no RELLIS ROS bag admission, continuous synchronized playback or production
|
||||
promotion;
|
||||
- no rolling-map implementation yet.
|
||||
|
||||
## Product surface boundary
|
||||
|
|
@ -82,9 +85,10 @@ The gateway is not part of device quality diagnostics.
|
|||
|
||||
Before compatible source evidence exists, the catalog explains the blocked
|
||||
storage gate and keeps `Открыть датасет` disabled. GOOSE becomes available only
|
||||
after the worker manifest says `frame-ready`; RELLIS becomes available at
|
||||
`smoke-ready` only for the pinned official example. Both bounded previews must
|
||||
pass their own identity, point-alignment and safety checks.
|
||||
after the worker manifest says `frame-ready`; RELLIS becomes `dataset-ready`
|
||||
only after all four pinned archives, every split pair and pose coverage pass
|
||||
admission. Both bounded previews must pass their own identity, point-alignment
|
||||
and safety checks.
|
||||
|
||||
## Why public recordings look different
|
||||
|
||||
|
|
@ -203,8 +207,10 @@ time; TTL/dynamic filtering prevents stale ghosts.
|
|||
- [x] Official RELLIS Ouster example passes the shared SemanticKITTI reader.
|
||||
- [x] RELLIS axes, ontology colors and versioned ground-target/ignore mapping
|
||||
are visible in the same Dataset workflow.
|
||||
- [ ] Full RELLIS Ouster SemanticKITTI scans, labels and poses admitted on
|
||||
- [x] Full RELLIS Ouster SemanticKITTI scans, labels and poses admitted on
|
||||
worker D.
|
||||
- [x] Current/Patchwork++ comparison executed and sealed over all `2,413`
|
||||
RELLIS validation frames.
|
||||
- [ ] Current/Patchwork++ RELLIS comparison qualified without obstacle loss.
|
||||
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
|
||||
|
||||
|
|
@ -239,12 +245,60 @@ non-ground; and `void`, `water`, `sky`, generic `object` and `puddle` as
|
|||
silently counted as either free ground or obstacle. This mapping is a Mission
|
||||
Core evaluation decision, not an upstream RELLIS claim.
|
||||
|
||||
The smoke closes reader, alignment, axes, palette and mapping compatibility
|
||||
only. The next gate admits the `14 GB` Ouster SemanticKITTI scans plus the
|
||||
`174 MB` labels and `174 MB` poses to D-only worker storage. Full ROS bags are
|
||||
not needed for the ground cross-check. Patchwork++ may be compared only after a
|
||||
physical Ouster mounting-height profile is evidenced; no height is inferred
|
||||
from the example cloud.
|
||||
The smoke closed reader, alignment, axes, palette and mapping compatibility
|
||||
only. The complete release was then admitted from four pinned archives kept
|
||||
exclusively on worker D:
|
||||
|
||||
- scans: `15,037,868,425` bytes, SHA-256
|
||||
`4e2bb5654bda5b9d08b9ac1ddbd39bd6dd231163a77ff811f2f282f3bbdf8ee7`;
|
||||
- labels: `182,265,710` bytes, SHA-256
|
||||
`03297c30b9f2182c74be93f8564d6ce1349237fcc4f89fc96479fe71bb40d905`;
|
||||
- poses: `609,344` bytes, SHA-256
|
||||
`6deaf38a9cac3a480cfdb70fa9a1d9278f25c36f5efd2487a3c4a6a400250f67`;
|
||||
- splits: `76,774` bytes, SHA-256
|
||||
`de639728e0058d7d477188028d9b811313a601884ef00d267ce13736be7cf54f`.
|
||||
|
||||
Admission identity
|
||||
`a3f3f161a5a7edccdf66cea75ecf9004b8f6a895282faabe661c075c15a82774`
|
||||
proves `13,556` scan/label pairs, the official `7,800 / 2,413 / 3,343`
|
||||
train/validation/test split counts, all five sequences and matching pose
|
||||
coverage. No ROS bag is needed for this single-scan ground cross-check.
|
||||
|
||||
## Full RELLIS qualification result
|
||||
|
||||
The immutable R1 replay-shadow run
|
||||
`rellis-ground-5b49d241a42b41d9b427` processed all `2,413` validation frames.
|
||||
Its qualification identity is
|
||||
`5b49d241a42b41d9b4279062c7517c99e16880e025cd2b70d3966d49f3ff6b7f`.
|
||||
The physical-height input was calibrated from `64` deterministic
|
||||
sequence-stratified **train** frames only: `1.0937 m` median height,
|
||||
`0.0538 m` median absolute deviation and no validation labels used.
|
||||
|
||||
| Validation micro metric | Current | Patchwork++ |
|
||||
| --- | ---: | ---: |
|
||||
| Ground IoU | `40.5450%` | `41.9599%` |
|
||||
| Natural-ground recall | `72.2992%` | `93.2555%` |
|
||||
| Obstacle non-ground recall | `80.4612%` | `69.7040%` |
|
||||
| Assigned fraction | `100%` | `100%` |
|
||||
| Latency p50 | `900.22 ms` | `8.58 ms` |
|
||||
| Latency p95 | `1284.29 ms` | `13.98 ms` |
|
||||
| Maximum latency | `1706.78 ms` | `20.50 ms` |
|
||||
|
||||
The sealed decision is `qualification-rejected`. Patchwork++ improved Ground
|
||||
IoU by only `1.41` percentage points against the required `2.00`, preserved
|
||||
only `69.70%` of obstacle points against the required `90%`, and regressed
|
||||
obstacle recall by `10.76` points against the allowed `1%`. Coverage, latency
|
||||
and catastrophic per-frame regression gates passed, but none of those can
|
||||
override obstacle loss. The provider therefore remains visualization and
|
||||
research evidence only:
|
||||
`promoted_to_navigation_or_safety = false`.
|
||||
|
||||
The bound review-pack identity is
|
||||
`1ee454fedd37bd56c3c9d07e17a3b5f59cdb29f7c1ae2086f8bc746149f3eb1b`.
|
||||
It contains bounded `12,000`-point previews for every validation scan and keeps
|
||||
the two real validation sequences separate. This lets an operator inspect raw
|
||||
geometry, public ground truth, both predictions and both error maps before
|
||||
changing the profile or acceptance thresholds.
|
||||
|
||||
## First GOOSE admission evidence
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue