fix(lab): make recorded replay seek-safe

2026-08-24 02:15:30 +03:00
parent 7400fe2fd7
commit 992c5a8b74
12 changed files with 1096 additions and 287 deletions
@@ -100,7 +100,7 @@ class RecordedCameraFrameService:
        target = self._inspector.get_segment(manifest, epoch.ordinal, sequence)
        fragments: list[bytes] = [target.payload]
        key_sequence = sequence
-        while not _fragment_is_sync(fragments[0]):
+        while not epoch.segments[key_sequence - 1].random_access:
            key_sequence -= 1
            if key_sequence < 1 or sequence - key_sequence > _MAX_KEYFRAME_DISTANCE:
                raise SessionIntegrityError("camera frame has no bounded sync fragment")
@@ -171,92 +171,6 @@ def _frame_location(
    raise SessionIntegrityError("camera frame is outside the recorded media manifest")


-def _fragment_is_sync(payload: bytes) -> bool:
-    tfhd_default_flags: int | None = None
-    sample_flags: int | None = None
-    sample_count: int | None = None
-    for box_type, body in _walk_boxes(payload):
-        if box_type == b"tfhd":
-            if len(body) < 8:
-                raise SessionIntegrityError("camera fragment tfhd is truncated")
-            flags = int.from_bytes(body[1:4], "big")
-            offset = 8
-            for mask, size in ((0x000001, 8), (0x000002, 4), (0x000008, 4), (0x000010, 4)):
-                if flags & mask:
-                    offset += size
-            if flags & 0x000020:
-                if offset + 4 > len(body):
-                    raise SessionIntegrityError("camera fragment default flags are truncated")
-                tfhd_default_flags = struct.unpack_from(">I", body, offset)[0]
-        elif box_type == b"trun":
-            if len(body) < 8:
-                raise SessionIntegrityError("camera fragment trun is truncated")
-            flags = int.from_bytes(body[1:4], "big")
-            sample_count = struct.unpack_from(">I", body, 4)[0]
-            if sample_count != 1:
-                raise SessionIntegrityError("camera fragment must contain exactly one sample")
-            offset = 8
-            if flags & 0x000001:
-                offset += 4
-            if flags & 0x000004:
-                if offset + 4 > len(body):
-                    raise SessionIntegrityError("camera fragment first flags are truncated")
-                sample_flags = struct.unpack_from(">I", body, offset)[0]
-                offset += 4
-            per_sample_sizes = (
-                (0x000100, 4),
-                (0x000200, 4),
-                (0x000400, 4),
-                (0x000800, 4),
-            )
-            for mask, size in per_sample_sizes:
-                if flags & mask:
-                    if offset + size > len(body):
-                        raise SessionIntegrityError("camera fragment sample data is truncated")
-                    if mask == 0x000400:
-                        sample_flags = struct.unpack_from(">I", body, offset)[0]
-                    offset += size
-    if sample_count != 1:
-        raise SessionIntegrityError("camera fragment has no unique video sample")
-    effective_flags = sample_flags if sample_flags is not None else tfhd_default_flags
-    if effective_flags is None:
-        raise SessionIntegrityError("camera fragment sample flags are unavailable")
-    return (effective_flags & 0x00010000) == 0
-
-
-def _walk_boxes(payload: bytes):
-    containers = {b"moof", b"traf"}
-    pending = [(0, len(payload))]
-    boxes = 0
-    while pending:
-        start, end = pending.pop()
-        offset = start
-        while offset + 8 <= end:
-            boxes += 1
-            if boxes > 64:
-                raise SessionIntegrityError("camera fragment box budget was exceeded")
-            size = struct.unpack_from(">I", payload, offset)[0]
-            box_type = payload[offset + 4 : offset + 8]
-            header = 8
-            if size == 1:
-                if offset + 16 > end:
-                    raise SessionIntegrityError("camera fragment extended box is truncated")
-                size = struct.unpack_from(">Q", payload, offset + 8)[0]
-                header = 16
-            elif size == 0:
-                size = end - offset
-            if size < header or offset + size > end:
-                raise SessionIntegrityError("camera fragment box size is invalid")
-            body_start = offset + header
-            body_end = offset + size
-            yield box_type, payload[body_start:body_end]
-            if box_type in containers:
-                pending.append((body_start, body_end))
-            offset = body_end
-        if offset != end:
-            raise SessionIntegrityError("camera fragment box boundary is invalid")
-
-
 def _jpeg_dimensions(payload: bytes) -> tuple[int, int]:
    if len(payload) < 4 or payload[:2] != b"\xff\xd8" or payload[-2:] != b"\xff\xd9":
        raise SessionIntegrityError("camera frame decoder returned an invalid JPEG")
@@ -18,8 +18,8 @@ from .models import RecordedMediaArtifact, ReplayCommand, SessionIntegrityError

 CAMERA_ARCHIVE_SCHEMA = "missioncore.camera-recording/v1"
 CAMERA_INDEX_SCHEMA = "missioncore.camera-recording-index/v1"
-RECORDED_MEDIA_MANIFEST_SCHEMA = "missioncore.observation-recorded-media/v2"
-RECORDED_MEDIA_PREPARATION_SCHEMA = "missioncore.recorded-media-preparation/v1"
+RECORDED_MEDIA_MANIFEST_SCHEMA = "missioncore.observation-recorded-media/v4"
+RECORDED_MEDIA_PREPARATION_SCHEMA = "missioncore.recorded-media-preparation/v3"
 MAX_MEDIA_SUMMARY_BYTES = 2 * 1024 * 1024
 MAX_MEDIA_INDEX_LINE_BYTES = 64 * 1024
 MAX_INIT_BYTES = 8 * 1024 * 1024
@@ -43,6 +43,8 @@ class RecordedMediaSegment:
    path: Path
    byte_length: int
    sha256: str
+    random_access: bool
+    end_time_seconds: float


@dataclass(frozen=True, slots=True)
@@ -91,12 +93,14 @@ class _Mp4VideoTiming:
    track_id: int
    timescale: int
    default_sample_duration: int | None
+    default_sample_flags: int | None


@dataclass(frozen=True, slots=True)
 class _Mp4VideoFragmentTiming:
    base_decode_time: int
    duration_units: int
+    random_access: bool

    @property
    def end_decode_time(self) -> int:
@@ -194,9 +198,7 @@ class RecordedMediaInspector:
                    "recorded media preparation could not be inspected"
                ) from exc
            if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
-                raise SessionIntegrityError(
-                    "recorded media preparation is not a regular file"
-                )
+                raise SessionIntegrityError("recorded media preparation is not a regular file")
            try:
                sidecar.unlink()
            except OSError as exc:
@@ -472,6 +474,8 @@ def _sidecar_manifest_document(manifest: RecordedMediaManifest) -> dict[str, Any
                        "sequence": segment.sequence,
                        "byte_length": segment.byte_length,
                        "sha256": segment.sha256,
+                        "random_access": segment.random_access,
+                        "end_time_seconds": segment.end_time_seconds,
                    }
                    for segment in epoch.segments
                ],
@@ -581,33 +585,46 @@ def _epoch_from_sidecar(
    ):
        raise SessionIntegrityError("recorded media prepared epoch descriptor is invalid")
    segment_documents = value.get("segments")
-    if (
-        not isinstance(segment_documents, list)
-        or not segment_documents
-    ):
+    if not isinstance(segment_documents, list) or not segment_documents:
        raise SessionIntegrityError("recorded media prepared segments are invalid")
    segments_root = epoch_path / "segments"
    segments: list[RecordedMediaSegment] = []
+    previous_end_time_seconds = 0.0
    for sequence, segment in enumerate(segment_documents, start=1):
        if not isinstance(segment, dict) or segment.get("sequence") != sequence:
            raise SessionIntegrityError("recorded media prepared segment is inconsistent")
        byte_length = segment.get("byte_length")
        sha256 = segment.get("sha256")
+        random_access = segment.get("random_access")
+        end_time_seconds = segment.get("end_time_seconds")
        if (
            not _positive_int(byte_length)
            or int(byte_length) > MAX_MEDIA_SEGMENT_BYTES
            or not isinstance(sha256, str)
            or _SHA256_PATTERN.fullmatch(sha256) is None
+            or not isinstance(random_access, bool)
+            or not _finite_non_negative_number(end_time_seconds)
+            or float(end_time_seconds) <= previous_end_time_seconds
        ):
            raise SessionIntegrityError("recorded media prepared segment is invalid")
+        previous_end_time_seconds = float(end_time_seconds)
        segments.append(
            RecordedMediaSegment(
                sequence=sequence,
                path=segments_root / f"{sequence}.m4s",
                byte_length=int(byte_length),
                sha256=sha256,
+                random_access=random_access,
+                end_time_seconds=previous_end_time_seconds,
            )
        )
+    if not math.isclose(
+        previous_end_time_seconds,
+        float(end) - float(start),
+        rel_tol=0.0,
+        abs_tol=1e-9,
+    ):
+        raise SessionIntegrityError("recorded media prepared segment timeline is inconsistent")
    return RecordedMediaEpoch(
        ordinal=ordinal,
        path=epoch_path,
@@ -863,6 +880,8 @@ def _manifest_generation_sha256(
                        "sequence": segment.sequence,
                        "byte_length": segment.byte_length,
                        "sha256": segment.sha256,
+                        "random_access": segment.random_access,
+                        "end_time_seconds": segment.end_time_seconds,
                    }
                    for segment in epoch.segments
                ],
@@ -895,10 +914,7 @@ def _read_epoch(
    ):
        raise SessionIntegrityError("recorded media summary is incompatible")
    segment_count = summary.get("segment_count")
-    if (
-        not _non_negative_int(segment_count)
-        or not 1 <= int(segment_count) <= MAX_SAFE_INTEGER
-    ):
+    if not _non_negative_int(segment_count) or not 1 <= int(segment_count) <= MAX_SAFE_INTEGER:
        raise SessionIntegrityError("recorded media segment count is invalid")
    match = _EPOCH_PATTERN.fullmatch(epoch.name)
    if (
@@ -922,10 +938,7 @@ def _read_epoch(
        expected_count=int(segment_count),
    )
    expected_index_sha256 = summary.get("index_sha256")
-    if (
-        not isinstance(expected_index_sha256, str)
-        or index_sha256 != expected_index_sha256
-    ):
+    if not isinstance(expected_index_sha256, str) or index_sha256 != expected_index_sha256:
        raise SessionIntegrityError("recorded media index digest changed")

    use_monotonic_clock = all(
@@ -957,7 +970,7 @@ def _read_epoch(
    segments_root = (epoch / "segments").resolve(strict=True)
    if not segments_root.is_dir() or segments_root.parent != epoch:
        raise SessionIntegrityError("recorded media segment directory is invalid")
-    segments: list[RecordedMediaSegment] = []
+    segment_descriptors: list[tuple[int, Path, int, str]] = []
    for sequence, entry in enumerate(entries, start=1):
        byte_length = entry.get("length")
        digest = entry.get("sha256")
@@ -977,14 +990,7 @@ def _read_epoch(
            raise SessionIntegrityError("recorded media segment length is outside bounds")
        if metadata.st_size != int(byte_length):
            raise SessionIntegrityError("recorded media segment length changed")
-        segments.append(
-            RecordedMediaSegment(
-                sequence=sequence,
-                path=path,
-                byte_length=int(byte_length),
-                sha256=digest,
-            )
-        )
+        segment_descriptors.append((sequence, path, int(byte_length), digest))

    init_path = (epoch / "init.mp4").resolve()
    init_payload = _read_confined_file(init_path, epoch, MAX_INIT_BYTES)
@@ -997,28 +1003,31 @@ def _read_epoch(
    fragment_timings: list[_Mp4VideoFragmentTiming] = []
    stream_sha256 = hashlib.sha256(init_payload)
    valid_bytes = len(init_payload)
-    for segment in segments:
+    for _sequence, path, byte_length, digest in segment_descriptors:
        payload = _read_confined_file(
-            segment.path,
+            path,
            segments_root,
-            segment.byte_length,
+            byte_length,
        )
-        if hashlib.sha256(payload).hexdigest() != segment.sha256:
+        if hashlib.sha256(payload).hexdigest() != digest:
            raise SessionIntegrityError("recorded media segment digest changed")
        stream_sha256.update(payload)
        valid_bytes += len(payload)
-        fragment_timings.append(
-            _mp4_video_fragment_timing(
-                payload,
-                timing,
-                _Mp4ParseBudget(),
-            )
+        fragment_timing = _mp4_video_fragment_timing(
+            payload,
+            timing,
+            _Mp4ParseBudget(),
        )
+        fragment_timings.append(fragment_timing)
    if (
        summary.get("valid_bytes") != valid_bytes
        or summary.get("stream_sha256") != stream_sha256.hexdigest()
    ):
        raise SessionIntegrityError("recorded media stream aggregate changed")
+    if not fragment_timings[0].random_access:
+        raise SessionIntegrityError(
+            "recorded media codec epoch does not begin with a random-access fragment"
+        )
    for previous, current in zip(
        fragment_timings,
        fragment_timings[1:],
@@ -1039,6 +1048,32 @@ def _read_epoch(
    timeline_end_seconds = timeline_start_seconds + total_duration_seconds
    if not math.isfinite(timeline_end_seconds) or timeline_end_seconds < timeline_start_seconds:
        raise SessionIntegrityError("recorded media epoch timeline is invalid")
+    epoch_duration_seconds = timeline_end_seconds - timeline_start_seconds
+    segments: list[RecordedMediaSegment] = []
+    cumulative_duration_units = 0
+    previous_end_time_seconds = 0.0
+    for index, (descriptor, fragment_timing) in enumerate(
+        zip(segment_descriptors, fragment_timings, strict=True),
+        start=1,
+    ):
+        sequence, path, byte_length, digest = descriptor
+        cumulative_duration_units += fragment_timing.duration_units
+        end_time_seconds = cumulative_duration_units / timing.timescale
+        if index == len(fragment_timings):
+            end_time_seconds = epoch_duration_seconds
+        if not math.isfinite(end_time_seconds) or end_time_seconds <= previous_end_time_seconds:
+            raise SessionIntegrityError("recorded media segment timeline is invalid")
+        previous_end_time_seconds = end_time_seconds
+        segments.append(
+            RecordedMediaSegment(
+                sequence=sequence,
+                path=path,
+                byte_length=byte_length,
+                sha256=digest,
+                random_access=fragment_timing.random_access,
+                end_time_seconds=end_time_seconds,
+            )
+        )
    return RecordedMediaEpoch(
        ordinal=ordinal,
        path=epoch,
@@ -1129,17 +1164,17 @@ def _mp4_video_timing(payload: bytes, budget: _Mp4ParseBudget) -> _Mp4VideoTimin
        raise SessionIntegrityError("recorded media init has no unique moov box")
    moov = moov_payloads[0]
    moov_boxes = tuple(_iter_mp4_boxes(moov, budget))
-    defaults: dict[int, int] = {}
+    defaults: dict[int, tuple[int, int]] = {}
    for box_type, box_payload in moov_boxes:
        if box_type != b"mvex":
            continue
        for child_type, child_payload in _iter_mp4_boxes(box_payload, budget):
            if child_type != b"trex":
                continue
-            track_id, trex_default_duration = _parse_trex(child_payload)
+            track_id, trex_default_duration, trex_default_flags = _parse_trex(child_payload)
            if track_id in defaults:
                raise SessionIntegrityError("recorded media init repeats a trex track")
-            defaults[track_id] = trex_default_duration
+            defaults[track_id] = (trex_default_duration, trex_default_flags)

    video_tracks: list[tuple[int, int]] = []
    for box_type, trak_payload in moov_boxes:
@@ -1152,13 +1187,15 @@ def _mp4_video_timing(payload: bytes, budget: _Mp4ParseBudget) -> _Mp4VideoTimin
    if len(video_tracks) != 1:
        raise SessionIntegrityError("recorded media init has no unique video track")
    track_id, timescale = video_tracks[0]
-    default_duration = defaults.get(track_id)
+    default_sample = defaults.get(track_id)
+    default_duration = None if default_sample is None else default_sample[0]
    return _Mp4VideoTiming(
        track_id=track_id,
        timescale=timescale,
        default_sample_duration=(
            default_duration if default_duration is not None and default_duration > 0 else None
        ),
+        default_sample_flags=None if default_sample is None else default_sample[1],
    )


@@ -1219,7 +1256,7 @@ def _mp4_video_fragment_timing(
        tfhd_payloads = [box for kind, box in boxes if kind == b"tfhd"]
        if len(tfhd_payloads) != 1:
            raise SessionIntegrityError("recorded media fragment tfhd is ambiguous")
-        track_id, fragment_default_duration = _parse_tfhd(tfhd_payloads[0])
+        track_id, fragment_default_duration, fragment_default_flags = _parse_tfhd(tfhd_payloads[0])
        if track_id != timing.track_id:
            continue
        tfdt_payloads = [box for kind, box in boxes if kind == b"tfdt"]
@@ -1230,15 +1267,28 @@ def _mp4_video_fragment_timing(
        if not trun_payloads:
            raise SessionIntegrityError("recorded media video fragment has no trun box")
        default_duration = fragment_default_duration or timing.default_sample_duration
-        duration = sum(
-            _parse_trun_duration_units(trun, default_duration, budget) for trun in trun_payloads
+        default_flags = (
+            fragment_default_flags
+            if fragment_default_flags is not None
+            else timing.default_sample_flags
        )
+        trun_descriptors = tuple(
+            _parse_trun_descriptor(trun, default_duration, default_flags, budget)
+            for trun in trun_payloads
+        )
+        fragment_sample_count = sum(item[2] for item in trun_descriptors)
+        if fragment_sample_count != 1:
+            raise SessionIntegrityError(
+                "recorded media video fragment must contain exactly one sample"
+            )
+        duration = sum(item[0] for item in trun_descriptors)
        if base_decode_time > MAX_SAFE_INTEGER - duration:
            raise SessionIntegrityError("recorded media fragment decode time is outside bounds")
        matching_timings.append(
            _Mp4VideoFragmentTiming(
                base_decode_time=base_decode_time,
                duration_units=duration,
+                random_access=(trun_descriptors[0][1] & 0x00010000) == 0,
            )
        )
    if len(matching_timings) != 1:
@@ -1292,15 +1342,16 @@ def _parse_hdlr_type(payload: bytes) -> bytes:
    return payload[8:12]


-def _parse_trex(payload: bytes) -> tuple[int, int]:
+def _parse_trex(payload: bytes) -> tuple[int, int, int]:
    _full_box_version(payload)
    return (
        _read_u32(payload, 4, "trex track id"),
        _read_u32(payload, 12, "trex default sample duration"),
+        _read_u32(payload, 20, "trex default sample flags"),
    )


-def _parse_tfhd(payload: bytes) -> tuple[int, int | None]:
+def _parse_tfhd(payload: bytes) -> tuple[int, int | None, int | None]:
    flags = _full_box_flags(payload)
    track_id = _read_u32(payload, 4, "tfhd track id")
    cursor = 8
@@ -1311,10 +1362,16 @@ def _parse_tfhd(payload: bytes) -> tuple[int, int | None]:
    if flags & 0x000008:
        default_duration = _read_u32(payload, cursor, "tfhd default sample duration")
        cursor += 4
-    for flag in (0x000010, 0x000020):
-        if flags & flag:
-            cursor = _advance_box_cursor(payload, cursor, 4, "tfhd optional field")
-    return track_id, default_duration if default_duration and default_duration > 0 else None
+    if flags & 0x000010:
+        cursor = _advance_box_cursor(payload, cursor, 4, "tfhd default sample size")
+    default_flags: int | None = None
+    if flags & 0x000020:
+        default_flags = _read_u32(payload, cursor, "tfhd default sample flags")
+    return (
+        track_id,
+        default_duration if default_duration and default_duration > 0 else None,
+        default_flags,
+    )


 def _parse_tfdt(payload: bytes) -> int:
@@ -1328,11 +1385,12 @@ def _parse_tfdt(payload: bytes) -> int:
    raise SessionIntegrityError("recorded media tfdt version is unsupported")


-def _parse_trun_duration_units(
+def _parse_trun_descriptor(
    payload: bytes,
    default_duration: int | None,
+    default_sample_flags: int | None,
    budget: _Mp4ParseBudget,
-) -> int:
+) -> tuple[int, int, int]:
    flags = _full_box_flags(payload)
    sample_count = _read_u32(payload, 4, "trun sample count")
    if sample_count < 1:
@@ -1341,30 +1399,51 @@ def _parse_trun_duration_units(
    cursor = 8
    if flags & 0x000001:
        cursor = _advance_box_cursor(payload, cursor, 4, "trun data offset")
+    first_sample_flags: int | None = None
    if flags & 0x000004:
-        cursor = _advance_box_cursor(payload, cursor, 4, "trun first sample flags")
-    per_sample_width = sum(4 for flag in (0x000100, 0x000200, 0x000400, 0x000800) if flags & flag)
-    if per_sample_width and sample_count > (len(payload) - cursor) // per_sample_width:
-        raise SessionIntegrityError("recorded media trun samples are truncated")
-    if flags & 0x000100:
-        duration = 0
-        for _ in range(sample_count):
+        first_sample_flags = _read_u32(payload, cursor, "trun first sample flags")
+        cursor += 4
+    if first_sample_flags is not None and flags & 0x000400:
+        raise SessionIntegrityError("recorded media trun sample flags are ambiguous")
+
+    duration = 0
+    first_per_sample_flags: int | None = None
+    for sample_index in range(sample_count):
+        if flags & 0x000100:
            sample_duration = _read_u32(payload, cursor, "trun sample duration")
            if sample_duration <= 0:
                raise SessionIntegrityError("recorded media sample duration is invalid")
            duration += sample_duration
-            cursor += per_sample_width
-        return duration
-    if default_duration is None or default_duration <= 0:
-        raise SessionIntegrityError("recorded media sample duration is unavailable")
-    if per_sample_width:
-        _advance_box_cursor(
-            payload,
-            cursor,
-            sample_count * per_sample_width,
-            "trun sample table",
-        )
-    return sample_count * default_duration
+            cursor += 4
+        elif default_duration is None or default_duration <= 0:
+            raise SessionIntegrityError("recorded media sample duration is unavailable")
+        else:
+            duration += default_duration
+        if flags & 0x000200:
+            cursor = _advance_box_cursor(payload, cursor, 4, "trun sample size")
+        if flags & 0x000400:
+            sample_flags = _read_u32(payload, cursor, "trun sample flags")
+            if sample_index == 0:
+                first_per_sample_flags = sample_flags
+            cursor += 4
+        if flags & 0x000800:
+            cursor = _advance_box_cursor(
+                payload,
+                cursor,
+                4,
+                "trun sample composition time offset",
+            )
+
+    effective_flags = (
+        first_per_sample_flags
+        if first_per_sample_flags is not None
+        else first_sample_flags
+        if first_sample_flags is not None
+        else default_sample_flags
+    )
+    if effective_flags is None:
+        raise SessionIntegrityError("recorded media sample flags are unavailable")
+    return duration, effective_flags, sample_count


 def _full_box_version(payload: bytes) -> int:
@@ -1614,9 +1693,11 @@ def _read_media_index(
                    "recorded media index length does not match its summary"
                )
            after = os.fstat(stream.fileno())
-        if (
-            (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns)
-            != (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns)
+        if (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) != (
+            before.st_dev,
+            before.st_ino,
+            before.st_size,
+            before.st_mtime_ns,
        ):
            raise SessionIntegrityError("recorded media index changed during validation")
        return entries, digest.hexdigest()
@@ -48,7 +48,8 @@ from k1link.viewer.recorded import (
 from k1link.viewer.rerun_bridge import RerunSceneSettings

 DEFAULT_REPLAY_ACTION_ID = "stream.start-replay"
-RECORDED_MEDIA_STREAM_MANIFEST_SCHEMA = "missioncore.observation-recorded-media/v3"
+RECORDED_MEDIA_STREAM_MANIFEST_SCHEMA = "missioncore.observation-recorded-media/v4"
+RECORDED_PERCEPTION_STREAM_MANIFEST_SCHEMA = "missioncore.observation-recorded-media/v3"
 SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
 SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
 MAX_SAFE_INTEGER = 9_007_199_254_740_991
@@ -1706,7 +1707,7 @@ def _recorded_perception_manifest_document(
    encoded_result_id = quote(video.result_id, safe="")
    base = f"/api/v1/observation-sessions/{encoded_session_id}/perception-media/{encoded_result_id}"
    return {
-        "schema_version": RECORDED_MEDIA_STREAM_MANIFEST_SCHEMA,
+        "schema_version": RECORDED_PERCEPTION_STREAM_MANIFEST_SCHEMA,
        "source_id": video.public_source_id,
        "generation_sha256": video.sha256,
        "byte_length": video.byte_length,
@@ -1795,6 +1796,13 @@ def _recorded_media_manifest_document(
                "media_type": epoch.media_type,
                "byte_length": epoch.init_byte_length
                + sum(segment.byte_length for segment in epoch.segments),
+                "segment_count": len(epoch.segments),
+                "random_access_sequences": sorted(
+                    segment.sequence for segment in epoch.segments if segment.random_access
+                ),
+                "segment_end_times_seconds": [
+                    segment.end_time_seconds for segment in epoch.segments
+                ],
                "stream_url": (
                    f"{base}/epochs/{epoch.ordinal}/recording.mp4"
                    f"?generation={manifest.generation_sha256}"