feat(perception): integrate calibrated operator pipeline

Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
@@ -2,20 +2,21 @@ from __future__ import annotations

 import inspect
 import re
-from collections.abc import Awaitable, Callable, Mapping
+from collections.abc import Awaitable, Callable, Iterator, Mapping
+from itertools import chain
+from pathlib import Path
 from threading import Lock
 from typing import Annotated, Any, Literal, Protocol
 from urllib.parse import quote

-from fastapi import APIRouter, Body, Header, HTTPException, Query, Response
-from fastapi.responses import FileResponse, JSONResponse
+from fastapi import APIRouter, Body, Header, HTTPException, Query, Request, Response
+from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
 from pydantic import BaseModel, ConfigDict, Field, StrictBool, field_validator, model_validator
 from starlette.concurrency import run_in_threadpool
 from starlette.types import Receive, Scope, Send

-from k1link.compute import RecordedPerceptionOverlayError
+from k1link.compute import RecordedPerceptionOverlayError, RecordedPerceptionVideo
 from k1link.sessions import (
-    RECORDED_MEDIA_MANIFEST_SCHEMA,
    LayoutConflictError,
    MaterializedRecording,
    RecordedMediaFile,
@@ -42,6 +43,7 @@ from k1link.viewer.recorded import (
 from k1link.viewer.rerun_bridge import RerunSceneSettings

 DEFAULT_REPLAY_ACTION_ID = "stream.start-replay"
+RECORDED_MEDIA_STREAM_MANIFEST_SCHEMA = "missioncore.observation-recorded-media/v3"
 SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
 SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
 MAX_SAFE_INTEGER = 9_007_199_254_740_991
@@ -103,7 +105,12 @@ class RecordedBlueprintRequest(StrictApiModel):
    point_size: float = Field(default=2.5, strict=True, ge=0.1, le=32.0)
    palette: Literal["turbo", "viridis", "plasma", "grayscale", "custom"] = "turbo"
    custom_color: str = Field(default="#f7f8f4", pattern=r"^#[0-9A-Fa-f]{6}$")
-    active_view: Literal["spatial", "perception", "metrics"] = "spatial"
+    active_view: Literal["spatial", "perception", "perception3d", "metrics"] = "spatial"
+    view_reset_generation: Literal[0, 1] = 0
+    unified_perception: StrictBool = False
+    show_detections_2d: StrictBool = False
+    show_segmentation: StrictBool = False
+    show_cuboids_3d: StrictBool = False


 class RecordedPerceptionRequest(StrictApiModel):
@@ -254,6 +261,14 @@ class RecordedPerceptionOverlayProvider(Protocol):
    ) -> bytes | None: ...


+class RecordedPerceptionMediaProvider(Protocol):
+    def video(
+        self,
+        session_id: str,
+        result_id: str | None = None,
+    ) -> RecordedPerceptionVideo | None: ...
+
+
 def build_session_router(
    store: SessionStore,
    *,
@@ -263,6 +278,7 @@ def build_session_router(
    recording_preparation_manager: SessionRecordingPreparationManager | None = None,
    media_inspector: RecordedMediaInspector | None = None,
    perception_overlay_provider: RecordedPerceptionOverlayProvider | None = None,
+    perception_media_provider: RecordedPerceptionMediaProvider | None = None,
    allow_synchronous_recording_fallback: bool = False,
    replay_action_id: str = DEFAULT_REPLAY_ACTION_ID,
 ) -> APIRouter:
@@ -463,6 +479,7 @@ def build_session_router(
                reserved.recording,
                command,
                reserved.recorded_media or (),
+                perception_media_provider=perception_media_provider,
            )

        if recording_materializer is not None and allow_synchronous_recording_fallback:
@@ -477,7 +494,12 @@ def build_session_router(
                )
            except SessionIntegrityError as exc:
                raise HTTPException(status_code=409, detail=str(exc)) from exc
-            return _recording_launch_document(recording, command, recorded_media)
+            return _recording_launch_document(
+                recording,
+                command,
+                recorded_media,
+                perception_media_provider=perception_media_provider,
+            )
        if recording_materializer is not None:
            raise HTTPException(
                status_code=503,
@@ -544,6 +566,7 @@ def build_session_router(
                    reserved.recording,
                    snapshot.command,
                    snapshot.recorded_media or (),
+                    perception_media_provider=perception_media_provider,
                ),
                headers={
                    "Cache-Control": "no-store",
@@ -778,6 +801,11 @@ def build_session_router(
                application_id=RECORDED_APPLICATION_ID,
                recording_id=request.recording_id,
                active_view=request.active_view,
+                view_reset_generation=request.view_reset_generation,
+                unified_perception=request.unified_perception,
+                show_detections_2d=request.show_detections_2d,
+                show_segmentation=request.show_segmentation,
+                show_cuboids_3d=request.show_cuboids_3d,
            )
        except SessionNotFoundError as exc:
            raise HTTPException(status_code=404, detail=str(exc)) from exc
@@ -853,6 +881,63 @@ def build_session_router(
            },
        )

+    @router.get(
+        "/api/v1/observation-sessions/{session_id}/perception-media/"
+        "{result_id}/manifest"
+    )
+    def get_recorded_perception_media_manifest(
+        session_id: str,
+        result_id: str,
+        if_match: Annotated[str | None, Header(alias="If-Match")] = None,
+    ) -> JSONResponse:
+        video = _resolve_recorded_perception_video(
+            store,
+            catalog_refresher,
+            perception_media_provider,
+            session_id,
+            result_id,
+        )
+        etag = f'"sha256:{video.sha256}"'
+        _require_matching_digest(etag, if_match)
+        return JSONResponse(
+            content=_recorded_perception_manifest_document(video),
+            headers={
+                "Cache-Control": "private, no-cache",
+                "ETag": etag,
+                "X-Content-Type-Options": "nosniff",
+            },
+        )
+
+    @router.api_route(
+        "/api/v1/observation-sessions/{session_id}/perception-media/"
+        "{result_id}/recording.mp4",
+        methods=["GET", "HEAD"],
+    )
+    def stream_recorded_perception_media(
+        request: Request,
+        session_id: str,
+        result_id: str,
+        generation: Annotated[str, Query(min_length=64, max_length=64)],
+        range_header: Annotated[str | None, Header(alias="Range")] = None,
+    ) -> Response:
+        video = _resolve_recorded_perception_video(
+            store,
+            catalog_refresher,
+            perception_media_provider,
+            session_id,
+            result_id,
+        )
+        if SAFE_SHA256.fullmatch(generation) is None or generation != video.sha256:
+            raise HTTPException(
+                status_code=412,
+                detail="Поколение видео сегментации было заменено.",
+            )
+        return _recorded_perception_video_response(
+            video,
+            range_header,
+            head_only=request.method == "HEAD",
+        )
+
    @router.get("/api/v1/observation-sessions/{session_id}/media/{artifact_id}/manifest")
    def get_recorded_media_manifest(
        session_id: str,
@@ -944,6 +1029,41 @@ def build_session_router(
            raise HTTPException(status_code=409, detail=str(exc)) from exc
        return _recorded_media_file_response(media_file, range_header)

+    @router.api_route(
+        "/api/v1/observation-sessions/{session_id}/media/{artifact_id}/"
+        "epochs/{epoch_ordinal}/recording.mp4",
+        methods=["GET", "HEAD"],
+    )
+    def stream_recorded_media_epoch(
+        request: Request,
+        session_id: str,
+        artifact_id: str,
+        epoch_ordinal: int,
+        generation: Annotated[str, Query(min_length=64, max_length=64)],
+        range_header: Annotated[str | None, Header(alias="Range")] = None,
+    ) -> Response:
+        manifest = _resolve_recorded_media_manifest(
+            store,
+            recorded_media_inspector,
+            recording_preparation_manager,
+            allow_synchronous_recording_fallback,
+            catalog_refresher,
+            session_id,
+            artifact_id,
+        )
+        if SAFE_SHA256.fullmatch(generation) is None or generation != manifest.generation_sha256:
+            raise HTTPException(
+                status_code=412,
+                detail="Поколение записанного видео было заменено.",
+            )
+        return _recorded_media_epoch_response(
+            recorded_media_inspector,
+            manifest,
+            epoch_ordinal,
+            range_header,
+            head_only=request.method == "HEAD",
+        )
+
    @router.get("/api/v1/workspace-layouts/{workspace_id}")
    def get_workspace_layout(workspace_id: str, response: Response) -> dict[str, Any]:
        try:
@@ -1259,9 +1379,16 @@ def _recording_launch_document(
    recording: MaterializedRecording,
    command: ReplayCommand,
    recorded_media: tuple[RecordedMediaManifest, ...] = (),
+    *,
+    perception_media_provider: RecordedPerceptionMediaProvider | None = None,
 ) -> dict[str, Any]:
+    del perception_media_provider
    encoded_session_id = quote(recording.session_id, safe="")
    source_url = f"/api/v1/observation-sessions/{encoded_session_id}/recording.rrd"
+    media_sources = [
+        _recorded_media_launch_source(manifest, index=index)
+        for index, manifest in enumerate(recorded_media, start=1)
+    ]
    return {
        "schema_version": "missioncore.observation-session-replay/v2",
        "launch": {
@@ -1280,13 +1407,7 @@ def _recording_launch_document(
                "speed": command.speed,
                "loop": command.loop,
            },
-            "media_sources": [
-                _recorded_media_launch_source(
-                    manifest,
-                    index=index,
-                )
-                for index, manifest in enumerate(recorded_media, start=1)
-            ],
+            "media_sources": media_sources,
        },
    }

@@ -1334,6 +1455,137 @@ def _recorded_media_launch_source(
    }


+def _recorded_perception_launch_source(video: RecordedPerceptionVideo) -> dict[str, Any]:
+    encoded_session_id = quote(video.session_id, safe="")
+    encoded_result_id = quote(video.result_id, safe="")
+    return {
+        "id": video.public_source_id,
+        "label": video.label,
+        "modality": "video",
+        "manifest_url": (
+            f"/api/v1/observation-sessions/{encoded_session_id}/perception-media/"
+            f"{encoded_result_id}/manifest"
+        ),
+        "media_type": "video/mp4",
+        "manifest_generation_sha256": video.sha256,
+        "byte_length": video.byte_length,
+        "timeline_start_seconds": video.timeline_start_seconds,
+        "timeline_end_seconds": video.timeline_end_seconds,
+        "seekable": True,
+        "synchronization": "host-arrival-best-effort",
+    }
+
+
+def _resolve_recorded_perception_video(
+    store: SessionStore,
+    catalog_refresher: CatalogRefresher | None,
+    provider: RecordedPerceptionMediaProvider | None,
+    session_id: str,
+    result_id: str,
+) -> RecordedPerceptionVideo:
+    if provider is None:
+        raise HTTPException(status_code=404, detail="Видео сегментации не найдено.")
+    try:
+        _prepare_replay(
+            store,
+            catalog_refresher,
+            session_id,
+            1.0,
+            False,
+            False,
+        )
+        video = provider.video(session_id, result_id)
+    except SessionNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    except (SessionNotReplayableError, SessionIntegrityError) as exc:
+        raise HTTPException(status_code=409, detail=str(exc)) from exc
+    except ValueError as exc:
+        raise HTTPException(
+            status_code=422,
+            detail="Некорректный идентификатор результата сегментации.",
+        ) from exc
+    if video is None:
+        raise HTTPException(status_code=404, detail="Видео сегментации не найдено.")
+    return video
+
+
+def _recorded_perception_manifest_document(
+    video: RecordedPerceptionVideo,
+) -> dict[str, Any]:
+    encoded_session_id = quote(video.session_id, safe="")
+    encoded_result_id = quote(video.result_id, safe="")
+    base = (
+        f"/api/v1/observation-sessions/{encoded_session_id}/perception-media/"
+        f"{encoded_result_id}"
+    )
+    return {
+        "schema_version": RECORDED_MEDIA_STREAM_MANIFEST_SCHEMA,
+        "source_id": video.public_source_id,
+        "generation_sha256": video.sha256,
+        "byte_length": video.byte_length,
+        "timeline_start_seconds": video.timeline_start_seconds,
+        "timeline_end_seconds": video.timeline_end_seconds,
+        "synchronization": "host-arrival-best-effort",
+        "epochs": [
+            {
+                "ordinal": 1,
+                "timeline_start_seconds": video.timeline_start_seconds,
+                "timeline_end_seconds": video.timeline_end_seconds,
+                "media_type": video.media_type,
+                "byte_length": video.byte_length,
+                "stream_url": f"{base}/recording.mp4?generation={video.sha256}",
+            }
+        ],
+    }
+
+
+def _recorded_perception_video_response(
+    video: RecordedPerceptionVideo,
+    range_header: str | None,
+    *,
+    head_only: bool,
+) -> Response:
+    start, end = 0, video.byte_length - 1
+    status_code = 200
+    if range_header is not None:
+        start, end = _parse_byte_range(range_header, video.byte_length)
+        status_code = 206
+    headers = {
+        "Accept-Ranges": "bytes",
+        "Cache-Control": "private, max-age=31536000, immutable, no-transform",
+        "ETag": f'"sha256:{video.sha256}"',
+        "Content-Length": str(end - start + 1),
+        "X-Content-Type-Options": "nosniff",
+        "Content-Disposition": 'inline; filename="recorded-perception.mp4"',
+    }
+    if status_code == 206:
+        headers["Content-Range"] = f"bytes {start}-{end}/{video.byte_length}"
+    if head_only:
+        return Response(
+            status_code=status_code,
+            media_type=video.media_type,
+            headers=headers,
+        )
+    return StreamingResponse(
+        _iter_regular_file_range(video.path, start, end),
+        status_code=status_code,
+        media_type=video.media_type,
+        headers=headers,
+    )
+
+
+def _iter_regular_file_range(path: Path, start: int, end: int) -> Iterator[bytes]:
+    remaining = end - start + 1
+    with path.open("rb") as stream:
+        stream.seek(start)
+        while remaining:
+            chunk = stream.read(min(1024 * 1024, remaining))
+            if not chunk:
+                raise SessionIntegrityError("recorded perception video ended unexpectedly")
+            remaining -= len(chunk)
+            yield chunk
+
+
 def _recorded_media_manifest_document(
    manifest: RecordedMediaManifest,
 ) -> dict[str, Any]:
@@ -1341,7 +1593,7 @@ def _recorded_media_manifest_document(
    encoded_artifact_id = quote(manifest.artifact_id, safe="")
    base = f"/api/v1/observation-sessions/{encoded_session_id}/media/{encoded_artifact_id}"
    return {
-        "schema_version": RECORDED_MEDIA_MANIFEST_SCHEMA,
+        "schema_version": RECORDED_MEDIA_STREAM_MANIFEST_SCHEMA,
        "source_id": manifest.public_source_id,
        "generation_sha256": manifest.generation_sha256,
        "byte_length": manifest.byte_length,
@@ -1354,20 +1606,12 @@ def _recorded_media_manifest_document(
                "timeline_start_seconds": epoch.timeline_start_seconds,
                "timeline_end_seconds": epoch.timeline_end_seconds,
                "media_type": epoch.media_type,
-                "init_url": f"{base}/epochs/{epoch.ordinal}/init.mp4",
-                "init_byte_length": epoch.init_byte_length,
-                "init_sha256": epoch.init_sha256,
-                "segment_count": len(epoch.segments),
-                "segment_url_prefix": f"{base}/epochs/{epoch.ordinal}/segments/",
-                "segments": [
-                    {
-                        "sequence": segment.sequence,
-                        "url": (f"{base}/epochs/{epoch.ordinal}/segments/{segment.sequence}.m4s"),
-                        "byte_length": segment.byte_length,
-                        "sha256": segment.sha256,
-                    }
-                    for segment in epoch.segments
-                ],
+                "byte_length": epoch.init_byte_length
+                + sum(segment.byte_length for segment in epoch.segments),
+                "stream_url": (
+                    f"{base}/epochs/{epoch.ordinal}/recording.mp4"
+                    f"?generation={manifest.generation_sha256}"
+                ),
            }
            for epoch in manifest.epochs
        ],
@@ -1500,6 +1744,94 @@ def _recorded_media_file_response(
    )


+def _recorded_media_epoch_response(
+    inspector: RecordedMediaInspector,
+    manifest: RecordedMediaManifest,
+    epoch_ordinal: int,
+    range_header: str | None,
+    *,
+    head_only: bool,
+) -> Response:
+    matches = tuple(epoch for epoch in manifest.epochs if epoch.ordinal == epoch_ordinal)
+    if len(matches) != 1:
+        raise HTTPException(status_code=404, detail="Эпоха записанного медиаканала не найдена.")
+    epoch = matches[0]
+    byte_length = epoch.init_byte_length + sum(
+        segment.byte_length for segment in epoch.segments
+    )
+    start, end = (0, byte_length - 1)
+    status_code = 200
+    if range_header is not None:
+        start, end = _parse_byte_range(range_header, byte_length)
+        status_code = 206
+
+    etag = f'"generation:{manifest.generation_sha256}:epoch:{epoch.ordinal}"'
+    headers = {
+        "Accept-Ranges": "bytes",
+        "Cache-Control": "private, max-age=31536000, immutable, no-transform",
+        "ETag": etag,
+        "Content-Length": str(end - start + 1),
+        "X-Content-Type-Options": "nosniff",
+        "Content-Disposition": (
+            f'inline; filename="recorded-camera-{epoch.ordinal}.mp4"'
+        ),
+    }
+    if status_code == 206:
+        headers["Content-Range"] = f"bytes {start}-{end}/{byte_length}"
+    if head_only:
+        return Response(
+            status_code=status_code,
+            media_type=epoch.media_type,
+            headers=headers,
+        )
+    return StreamingResponse(
+        _iter_recorded_media_epoch_range(
+            inspector,
+            manifest,
+            epoch.ordinal,
+            start,
+            end,
+        ),
+        status_code=status_code,
+        media_type=epoch.media_type,
+        headers=headers,
+    )
+
+
+def _iter_recorded_media_epoch_range(
+    inspector: RecordedMediaInspector,
+    manifest: RecordedMediaManifest,
+    epoch_ordinal: int,
+    start: int,
+    end: int,
+) -> Iterator[bytes]:
+    matches = tuple(epoch for epoch in manifest.epochs if epoch.ordinal == epoch_ordinal)
+    if len(matches) != 1:
+        raise SessionIntegrityError("recorded media epoch is unavailable")
+    epoch = matches[0]
+    offset = 0
+    parts = chain(
+        ((epoch.init_byte_length, None),),
+        ((segment.byte_length, segment.sequence) for segment in epoch.segments),
+    )
+    for part_length, sequence in parts:
+        part_end = offset + part_length - 1
+        if part_end < start:
+            offset += part_length
+            continue
+        if offset > end:
+            break
+        media_file = (
+            inspector.get_init(manifest, epoch_ordinal)
+            if sequence is None
+            else inspector.get_segment(manifest, epoch_ordinal, sequence)
+        )
+        local_start = max(0, start - offset)
+        local_end = min(part_length - 1, end - offset)
+        yield media_file.payload[local_start : local_end + 1]
+        offset += part_length
+
+
 def _parse_byte_range(value: str, byte_length: int) -> tuple[int, int]:
    match = re.fullmatch(r"bytes=(\d*)-(\d*)", value.strip())
    if match is None or byte_length < 1: