diff --git a/apps/control-station/src/components/ObservationSessionSelect.tsx b/apps/control-station/src/components/ObservationSessionSelect.tsx index 307638a..440e1e4 100644 --- a/apps/control-station/src/components/ObservationSessionSelect.tsx +++ b/apps/control-station/src/components/ObservationSessionSelect.tsx @@ -11,7 +11,7 @@ import type { } from "../core/observation/useObservationSessions"; import "../styles/observation-sessions.css"; -type SessionVisualState = "ready" | "processing" | "error"; +type SessionVisualState = "ready" | "processing" | "cold" | "error"; export function observationSessionVisualState( session: ObservationSessionSummary, @@ -26,12 +26,14 @@ export function observationSessionVisualState( if (session.preparation.state === "ready" && session.replayable) return "ready"; return "error"; } - return session.status === "recording" ? "processing" : "error"; + if (session.status === "recording") return "processing"; + return session.replayable ? "cold" : "error"; } export function observationSessionVisualLabel(state: SessionVisualState): string { if (state === "ready") return "Готово"; if (state === "processing") return "Обработка"; + if (state === "cold") return "Подготовить"; return "Ошибка"; } diff --git a/apps/control-station/src/styles/observation-sessions.css b/apps/control-station/src/styles/observation-sessions.css index c1c4b08..239c889 100644 --- a/apps/control-station/src/styles/observation-sessions.css +++ b/apps/control-station/src/styles/observation-sessions.css @@ -137,8 +137,12 @@ animation: observation-session-processing 1.1s ease-in-out infinite; } +.observation-session-option i[data-session-visual-state="cold"], .observation-session-option i[data-session-visual-state="error"] { background: var(--nodedc-text-muted); +} + +.observation-session-option i[data-session-visual-state="error"] { opacity: 0.48; } diff --git a/apps/control-station/test/observationSessions.test.mjs b/apps/control-station/test/observationSessions.test.mjs index edd4b9a..4039ef3 100644 --- a/apps/control-station/test/observationSessions.test.mjs +++ b/apps/control-station/test/observationSessions.test.mjs @@ -160,7 +160,7 @@ test("session catalog exposes authoritative background preparation state", () => }); }); -test("saved-session rows use only authoritative ready, processing and error states", () => { +test("saved-session rows distinguish durable, active, cold and failed states", () => { const makeDecoded = (state, overrides = {}) => decodeObservationSessionCatalog({ items: [session({ status: "interrupted", @@ -187,7 +187,7 @@ test("saved-session rows use only authoritative ready, processing and error stat observationSessionVisualState(makeDecoded("failed", { status: "recording" })), "error", ); - assert.equal(observationSessionVisualState(makeDecoded(null)), "error"); + assert.equal(observationSessionVisualState(makeDecoded(null)), "cold"); assert.equal( observationSessionVisualState(makeDecoded(null, { status: "recording" })), "processing", @@ -206,6 +206,7 @@ test("saved-session rows use only authoritative ready, processing and error stat ); assert.equal(observationSessionVisualLabel("ready"), "Готово"); assert.equal(observationSessionVisualLabel("processing"), "Обработка"); + assert.equal(observationSessionVisualLabel("cold"), "Подготовить"); assert.equal(observationSessionVisualLabel("error"), "Ошибка"); }); diff --git a/docs/09_OBSERVATION_SESSIONS.md b/docs/09_OBSERVATION_SESSIONS.md index 4ca3a95..932c3e2 100644 --- a/docs/09_OBSERVATION_SESSIONS.md +++ b/docs/09_OBSERVATION_SESSIONS.md @@ -57,22 +57,26 @@ not mutate playback state, so changing accumulation, size, color or visibility does not pause the active replay or replace its camera view. Completed and recovered sessions are discovered on startup and by the catalog -reconciler, then deduplicated into one bounded, single-worker preparation queue. -The worker validates the native source, exports point/pose data, verifies source -stability and both SHA-256 digests, finalizes once, and atomically publishes the +reconciler. The first scan is only a historical baseline: it never enqueues the +existing archive. A session that becomes finalized after that baseline is +deduplicated into the bounded, single-worker preparation queue. The worker +validates the native source, exports point/pose data, verifies source stability +and both SHA-256 digests, finalizes once, and atomically publishes the digest-bound derived RRD plus its cache sidecar. Before the same job becomes -`ready`, it also parses and validates every archived camera index and retains -the immutable manifest generation in memory. The catalog exposes preparation -state and progress while this runs; a recording URL and camera descriptors are -returned only after the complete launch generation has passed validation. +`ready`, it also parses and validates every archived camera index and publishes +the immutable media-manifest sidecar. The catalog exposes preparation state and +progress while this runs; a recording URL and camera descriptors are returned +only after the complete launch generation has passed validation. -Preparation is a backend lifecycle, not a viewer action: sealing or recovering -a native session makes it eligible for the reconciler, which builds the derived -RRD once in the background. Reopening, seeking or changing the workspace reads -that prepared artifact and never reruns conversion. A cold process may validate -an existing cache and rebuild its in-memory camera manifest in the worker, but -no replay, status, RRD or manifest request hashes native evidence or parses a -camera index synchronously. +Preparation is a backend lifecycle, not a viewer computation. Reopening, +seeking, refreshing the catalog or changing the workspace never reruns a +published conversion. After a process restart, a catalog/replay read restores a +complete compatible RRD and camera package from its durable sidecars using +bounded schema and stat-identity checks; it does not hash native evidence, parse +camera indexes or invoke the exporter. A historical session whose package is +missing, stale or from an incompatible cache schema remains cold. Only an +explicit replay/RRD request schedules that one session for preparation; listing +the catalog and finalizing another session do not schedule it. The session menu uses three operator indicators: @@ -80,6 +84,7 @@ The session menu uses three operator indicators: | --- | --- | --- | | Solid green + `Готово` | `ready` | Verified RRD is available. | | Pulsing green + `Обработка` | `queued`, `validating`, `exporting`, `finalizing` | Background preparation is active. | +| Solid gray + `Подготовить` | replayable session with no compatible published package | No work is running; choosing the row schedules this session only. | | Dim gray + `Ошибка` | failed, cancelled, non-replayable or invalid session | No launchable recording; inspect the message or retry. | Saved-session status never uses yellow or red. Switching sessions aborts only @@ -93,7 +98,8 @@ Later openings reuse a verified, digest-bound cache. Derived RRD cache v9 is intentionally incompatible with v8 and older generations. In addition to plugin and ordered-artifact identity, it binds the durable clock origin and active envelope and materializes real `session_time = 0` origin and sealed completion -rows. Older generations are rebuilt once in the background. +rows. Older generations remain cold after startup and are rebuilt only when an +operator explicitly opens that session. The browser may use the strict `source_url` with `If-Match`, while the embedded Rerun loader uses the canonical `viewer_source_url` whose lowercase SHA-256 `generation` query is bound to the same launch descriptor. A missing or stale @@ -194,11 +200,13 @@ Native `.k1mqtt` bytes with aligned metadata, the capture-clock artifacts and canonical camera fMP4 archives are the evidence source of truth. The derived RRD contains every decodable point and pose frame plus every valid `ModelingReport` distance/speed/scan-time sample from that source, but remains a rebuildable view -rather than an evidence master. A cache entry is rebuilt only when native source -identity/digests change or an incompatible derived-data export revision is -introduced. A UI blueprint or workspace-layout revision never invalidates or -rewrites the data RRD. Cache v8 and older payloads are not reusable as v9 because -they do not bind both real capture-envelope endpoints. +rather than an evidence master. A cache entry is eligible for rebuild only when +native source identity/digests change or an incompatible derived-data export +revision is introduced. Historical entries are not rebuilt during startup or +because another session is finalized; an explicit open schedules the affected +entry. A UI blueprint or workspace-layout revision never invalidates or rewrites +the data RRD. Cache v8 and older payloads are not reusable as v9 because they do +not bind both real capture-envelope endpoints. `ModelingReport` time is the device's `ScanTime` counter at two ticks per second; distance and speed are the reported `MoveDistance`/`MoveSpeed` values. Live state @@ -318,9 +326,10 @@ written under the private derived cache with a schema, generation and checksum. Publication uses a private temporary file, file and directory `fsync`, and atomic rename; startup scavenges crash-left temporary files. A restart reuses this sidecar after confined O(n) stat validation, without rereading, hashing or parsing -media. A missing, corrupt or stale sidecar is rebuilt only by the background -worker. No replay, status, manifest or payload request performs conversion or -recalculates these intervals. +media. A missing, corrupt or stale sidecar remains cold until an explicit open +schedules the complete session package in the background worker. No catalog, +status, manifest or payload request itself performs conversion or recalculates +these intervals. Every derived RRD contains a real `session_time = 0` row at the internal `/__mission_core/session_origin` entity. The anchor is deliberately outside diff --git a/docs/adr/0008-durable-observation-sessions-and-workspace-layout.md b/docs/adr/0008-durable-observation-sessions-and-workspace-layout.md index ac09220..886ef48 100644 --- a/docs/adr/0008-durable-observation-sessions-and-workspace-layout.md +++ b/docs/adr/0008-durable-observation-sessions-and-workspace-layout.md @@ -73,11 +73,15 @@ committed metadata; replay stops at the last validated aligned boundary. ### Recorded spatial playback -When a native capture is normally sealed or recovery-sealed, a bounded -single-worker reconciler automatically prepares its private derived `.rrd`. -Preparation is a backend lifecycle and is never executed by an HTTP replay -request. Reopening, seeking, switching tabs or changing display settings reads -the already prepared artifact and does not rerun conversion. Export is lossless +When a native capture is normally sealed or recovery-sealed during the current +process lifetime, a bounded single-worker reconciler automatically prepares its +private derived `.rrd`. The reconciler's first scan establishes a historical +baseline and does not enqueue old sessions. Preparation is a backend lifecycle +and is never executed inside an HTTP replay request: an explicit request may +enqueue one cold historical package and returns `202` while the worker owns the +conversion. Reopening, seeking, switching tabs, listing the catalog or changing +display settings reads the already published artifact and does not rerun +conversion. Export is lossless with respect to every decodable point and pose message in the native capture; it does not pass through the bounded live-preview queue. The RRD uses the recording-local `session_time` duration timeline, while the receive wall clock @@ -123,21 +127,23 @@ future replay v3 will need distinct spatial and session-union ranges. The path-free descriptor is atomically persisted in the private derived cache with its checksum and a stat identity covering the native timing origin and every camera summary, index, init and segment file. Restart performs confined -stat validation and reuses an unchanged descriptor without reading/hashing media; -a missing, corrupt or stale sidecar is rebuilt in the background. An unparseable -or ambiguous fragment fails readiness rather than exposing a fake seekable -camera. Replay requests consume the prepared descriptor and never repeat media -conversion or timing analysis. Launch and manifest aggregate byte counts must -agree before the browser admits camera payload downloads. +stat validation and reuses an unchanged descriptor without reading/hashing media. +A missing, corrupt or stale historical sidecar stays cold until that session is +explicitly opened; startup, catalog reads and completion of another session do +not rebuild it. An unparseable or ambiguous fragment fails readiness rather than +exposing a fake seekable camera. Replay requests consume the prepared descriptor +and never repeat media conversion or timing analysis. Launch and manifest +aggregate byte counts must agree before the browser admits camera payload +downloads. The derived RRD writes an actual zero-time anchor at the internal `/__mission_core/session_origin` entity. Keeping it outside `/world` prevents a synthetic visualization layer while ensuring that the decoded RRD timeline, not merely its summary document, begins at session time zero. This changes the -derived payload contract. Cache v7 keeps that anchor and additionally binds the -derived recording to the owning plugin ID, primary artifact and ordered source -artifact set; it rejects v6 and older sidecars and performs one background -rebuild. +derived payload contract. Cache v9 keeps that anchor, binds the owning plugin, +ordered source artifacts and capture-envelope endpoints, and materializes the +sealed completion rows. Older sidecars are incompatible and remain cold until +that session is explicitly opened. Replay v2 exposes two paths to the same pinned generation. `source_url` retains the strict `If-Match: "sha256:…"` contract for Mission Core clients; diff --git a/src/k1link/sessions/media.py b/src/k1link/sessions/media.py index aa1735d..780a029 100644 --- a/src/k1link/sessions/media.py +++ b/src/k1link/sessions/media.py @@ -175,6 +175,42 @@ class RecordedMediaInspector: self._cache[key] = _CachedManifest(identity=identity, manifest=manifest) return manifest + def restore_prepared( + self, + artifact: RecordedMediaArtifact, + replay: ReplayCommand, + ) -> RecordedMediaManifest | None: + """Restore one previously prepared manifest without rebuilding it. + + A catalog read or process restart must never turn an old camera archive + into fresh preparation work. The durable sidecar is accepted only when + its complete source stat identity still matches the sealed archive. + Missing or stale sidecars remain cold until the operator explicitly + requests that session. + """ + + if artifact.session_id != replay.session_id: + raise SessionIntegrityError("recorded media does not belong to replay session") + epoch_paths = _epoch_paths(artifact.source_path) + key = (artifact.session_id, artifact.artifact_id) + with self._lock: + cached = self._cache.get(key) + if cached is not None: + identity = _prepared_source_identity( + replay, + epoch_paths, + cached.manifest.epochs, + ) + if cached.identity == identity: + return cached.manifest + prepared = self._load_prepared_sidecar(artifact, replay, epoch_paths) + if prepared is None: + return None + manifest, identity = prepared + with self._lock: + self._cache[key] = _CachedManifest(identity=identity, manifest=manifest) + return manifest + def _load_prepared_sidecar( self, artifact: RecordedMediaArtifact, diff --git a/src/k1link/sessions/preparation.py b/src/k1link/sessions/preparation.py index 6bb4298..990b738 100644 --- a/src/k1link/sessions/preparation.py +++ b/src/k1link/sessions/preparation.py @@ -111,6 +111,11 @@ class SessionRecordingPreparationManager: tuple[RecordedMediaManifest, ...], ] | None = None, + ready_restorer: Callable[ + [ReplayCommand, MaterializedRecording], + tuple[RecordedMediaManifest, ...] | None, + ] + | None = None, ) -> None: if queue_capacity < 1: raise ValueError("recording preparation queue capacity must be positive") @@ -123,6 +128,7 @@ class SessionRecordingPreparationManager: # hung exporter must become observable to the browser stall detector. self._heartbeat_interval_seconds = heartbeat_interval_seconds self._ready_preparer = ready_preparer + self._ready_restorer = ready_restorer self._guard = threading.RLock() self._current_by_session: dict[str, _PreparationJob] = {} self._closed = True @@ -237,6 +243,74 @@ class SessionRecordingPreparationManager: self._current_by_session[command.session_id] = ready return self._snapshot_locked(ready) + def restore_published( + self, + command: ReplayCommand, + ) -> RecordingPreparationSnapshot | None: + """Restore a complete durable package without enqueueing preparation. + + The RRD and every recorded-media descriptor must already have been + published by an earlier successful job. Missing/stale pieces return a + cold miss; this method never invokes an exporter or media parser. + """ + + source_command = _source_command(command) + identity = _source_identity(source_command) + with self._guard: + current = self._current_by_session.get(command.session_id) + if ( + current is not None + and current.source_identity == identity + and current.state in ACTIVE_PREPARATION_STATES | {"ready"} + ): + return self._snapshot_locked(current) + + recording = self.materializer.restore_published(source_command) + if recording is None: + with self._guard: + current = self._current_by_session.get(command.session_id) + if current is not None and current.state == "ready": + self._current_by_session.pop(command.session_id, None) + return None + + if self._ready_preparer is None: + recorded_media: tuple[RecordedMediaManifest, ...] = () + else: + if self._ready_restorer is None: + return None + restored_media = self._ready_restorer(source_command, recording) + if restored_media is None: + return None + recorded_media = restored_media + validate_recorded_media_timeline( + recorded_media, + recording_start_seconds=recording.timeline_start_ns / 1_000_000_000, + recording_end_seconds=recording.timeline_end_ns / 1_000_000_000, + ) + + with self._guard: + current = self._current_by_session.get(command.session_id) + if ( + current is not None + and current.source_identity == identity + and current.state in ACTIVE_PREPARATION_STATES | {"ready"} + ): + if current.state == "ready": + current.recording = recording + current.recorded_media = recorded_media + return self._snapshot_locked(current) + ready = _PreparationJob( + preparation_id=uuid4().hex, + source_identity=identity, + command=source_command, + state="ready", + progress=1.0, + recording=recording, + recorded_media=recorded_media, + ) + self._current_by_session[command.session_id] = ready + return self._snapshot_locked(ready) + def resolve_cached_pinned( self, command: ReplayCommand, @@ -325,10 +399,7 @@ class SessionRecordingPreparationManager: job is None or job.state != "ready" or job.recording is None - or ( - preparation_id is not None - and job.preparation_id != preparation_id - ) + or (preparation_id is not None and job.preparation_id != preparation_id) ): return None recording = job.recording @@ -456,9 +527,7 @@ class SessionRecordingPreparationManager: recording_start_seconds=( recording.timeline_start_ns / 1_000_000_000 ), - recording_end_seconds=( - recording.timeline_end_ns / 1_000_000_000 - ), + recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000), ) except Exception: with self._guard: diff --git a/src/k1link/sessions/recording.py b/src/k1link/sessions/recording.py index 3bb4f78..6c04eb5 100644 --- a/src/k1link/sessions/recording.py +++ b/src/k1link/sessions/recording.py @@ -232,6 +232,26 @@ class SessionRecordingMaterializer: with self._lock_for(session_id): return self._load_cached_recording(session_id, _validate_source(command)) + def restore_published(self, command: ReplayCommand) -> MaterializedRecording | None: + """Restore an immutable published cache using bounded metadata checks. + + Full source/output digests are proved before the cache is published. + On a later process start the private cache is restored only when its + schema, source stat identities and derived-file stat identity still + match the durable sidecar. This keeps a prepared recording durable + across restarts without rereading gigabytes or starting an exporter. + """ + + session_id = _validate_command_shape(command) + if not self._has_compatible_cache_candidate(session_id): + return None + with self._lock_for(session_id): + return self._load_cached_recording( + session_id, + _validate_source(command), + verify_digests=False, + ) + def get_cached_pinned( self, command: ReplayCommand, @@ -430,6 +450,7 @@ class SessionRecordingMaterializer: source: _ValidatedSource, *, pin: bool = False, + verify_digests: bool = True, ) -> MaterializedRecording | None: # Cache validation and the optional lease are one transaction with # eviction. This makes it safe for FileResponse to open the path after @@ -451,6 +472,7 @@ class SessionRecordingMaterializer: source=source, recording_path=recording_path, sidecar_path=sidecar_path, + verify_digests=verify_digests, ) if cached is None: return None @@ -661,6 +683,7 @@ class SessionRecordingMaterializer: source: _ValidatedSource, recording_path: Path, sidecar_path: Path, + verify_digests: bool = True, ) -> MaterializedRecording | None: if recording_path.is_symlink() or sidecar_path.is_symlink(): return None @@ -698,12 +721,19 @@ class SessionRecordingMaterializer: source_sha256: str | None = None for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True): - digest = _sha256_prefix_stable( - artifact.path, - artifact.file_stat, - artifact.replay_byte_length, + digest = ( + _sha256_prefix_stable( + artifact.path, + artifact.file_stat, + artifact.replay_byte_length, + ) + if verify_digests + else cached["sha256"] ) - if artifact.expected_sha256 is not None and digest != artifact.expected_sha256: + if ( + artifact.expected_sha256 is not None + and cached["sha256"] != artifact.expected_sha256 + ): raise RecordingMaterializationError( "recording artifact digest no longer matches catalog" ) @@ -713,9 +743,11 @@ class SessionRecordingMaterializer: source_sha256 = digest if source_sha256 is None or source_sha256 != document["source_sha256"]: return None - recording_sha256 = _sha256_stable(recording_path, recording_stat) - if recording_sha256 != document["recording_sha256"]: - return None + recording_sha256 = document["recording_sha256"] + if verify_digests: + recording_sha256 = _sha256_stable(recording_path, recording_stat) + if recording_sha256 != document["recording_sha256"]: + return None _chmod_best_effort(recording_path, 0o600) _chmod_best_effort(sidecar_path, 0o600) if document["schema_version"] != CACHE_SCHEMA: diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index a9c9b08..e7db064 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -1,7 +1,7 @@ from __future__ import annotations import asyncio -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager, suppress from pathlib import Path from typing import Any @@ -65,9 +65,25 @@ def _prepare_recorded_media_for_launch( ) +def _restore_recorded_media_for_launch( + command: ReplayCommand, + _: MaterializedRecording, +) -> tuple[RecordedMediaManifest, ...] | None: + """Restore only previously published media descriptors.""" + + restored: list[RecordedMediaManifest] = [] + for artifact in session_store.list_recorded_media(command.session_id): + manifest = session_recorded_media_inspector.restore_prepared(artifact, command) + if manifest is None: + return None + restored.append(manifest) + return tuple(restored) + + session_recording_preparation_manager = SessionRecordingPreparationManager( session_recording_materializer, ready_preparer=_prepare_recorded_media_for_launch, + ready_restorer=_restore_recorded_media_for_launch, ) @@ -82,47 +98,73 @@ def refresh_observation_catalog() -> tuple[str, ...]: return tuple(dict.fromkeys(imported)) -def enqueue_replayable_recordings() -> tuple[str, ...]: - """Reconcile finalized catalog sessions into the durable RRD work queue.""" +def finalized_replayable_recording_ids() -> tuple[str, ...]: + """List finalized replayable catalog identities without scheduling work.""" - enqueued: list[str] = [] + finalized: list[str] = [] cursor: str | None = None while True: page = session_store.list_recent(limit=100, cursor=cursor) - for summary in page.items: - if not summary.replayable or summary.status not in { - "ready", - "interrupted", - "failed", - }: - continue - try: - command = session_store.prepare_replay(summary.session_id) - session_recording_preparation_manager.enqueue( - command, - retry_interrupted=True, - ) - except RecordingPreparationQueueFull: - return tuple(enqueued) - except Exception: - # One stale/corrupt catalog row must not starve every valid - # finalized session that follows it in the reconciliation - # page. The row remains visible as failed/interrupted evidence - # and can be repaired independently. - continue - enqueued.append(summary.session_id) + finalized.extend( + summary.session_id + for summary in page.items + if summary.replayable and summary.status in {"ready", "interrupted", "failed"} + ) cursor = page.next_cursor if cursor is None: + return tuple(finalized) + + +def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]: + """Schedule only explicitly selected newly finalized sessions.""" + + enqueued: list[str] = [] + for session_id in dict.fromkeys(session_ids): + try: + command = session_store.prepare_replay(session_id) + session_recording_preparation_manager.enqueue( + command, + retry_interrupted=True, + ) + except RecordingPreparationQueueFull: return tuple(enqueued) + except Exception: + # One stale/corrupt row must not starve a later newly completed + # session. Historical cold caches remain operator-triggered. + continue + enqueued.append(session_id) + return tuple(enqueued) + + +def newly_finalized_recording_ids( + known_finalized: set[str] | None, + current_finalized: Iterable[str], +) -> tuple[str, ...]: + """Return only post-startup completions; the first scan is a baseline.""" + + if known_finalized is None: + return () + return tuple(sorted(set(current_finalized) - known_finalized)) async def _recording_preparation_reconciler() -> None: - """Discover newly completed/recovered captures without blocking requests.""" + """Prepare sessions finalized during this process, never historical rows.""" + known_finalized: set[str] | None = None while True: try: await asyncio.to_thread(refresh_observation_catalog) - await asyncio.to_thread(enqueue_replayable_recordings) + finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids)) + newly_finalized = newly_finalized_recording_ids( + known_finalized, + finalized, + ) + if newly_finalized: + await asyncio.to_thread( + enqueue_replayable_recordings, + newly_finalized, + ) + known_finalized = finalized except Exception: # A transient filesystem/catalog failure must not permanently # disable preparation of sessions completed later in the run. diff --git a/src/k1link/web/session_api.py b/src/k1link/web/session_api.py index f755820..0ae893c 100644 --- a/src/k1link/web/session_api.py +++ b/src/k1link/web/session_api.py @@ -45,9 +45,7 @@ SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$") SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$") MAX_SAFE_INTEGER = 9_007_199_254_740_991 REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform" -IMMUTABLE_RECORDING_CACHE_CONTROL = ( - "private, max-age=31536000, immutable, no-transform" -) +IMMUTABLE_RECORDING_CACHE_CONTROL = "private, max-age=31536000, immutable, no-transform" class _ReleasingFileResponse(FileResponse): @@ -321,13 +319,17 @@ def build_session_router( if recording_preparation_manager is not None: try: - # Enqueue performs only catalog identity/lstat checks. A cold - # RRD cache and every camera index are validated exclusively - # by the process-owned worker. - snapshot = recording_preparation_manager.enqueue( - command, - retry_failed=True, - ) + snapshot = recording_preparation_manager.status(command.session_id) + if snapshot is None: + snapshot = await run_in_threadpool( + recording_preparation_manager.restore_published, + command, + ) + if snapshot is None: + snapshot = recording_preparation_manager.enqueue( + command, + retry_failed=True, + ) except RecordingPreparationQueueFull as exc: raise HTTPException( status_code=503, @@ -504,7 +506,12 @@ def build_session_router( False, False, ) - snapshot = recording_preparation_manager.enqueue(command) + snapshot = await run_in_threadpool( + recording_preparation_manager.restore_published, + command, + ) + if snapshot is None: + snapshot = recording_preparation_manager.enqueue(command) except SessionNotFoundError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc except (SessionNotReplayableError, SessionIntegrityError) as exc: @@ -1036,15 +1043,16 @@ def _catalog_preparation_document( snapshot = manager.status(session_id) if snapshot is None: try: - snapshot = manager.enqueue(store.prepare_replay(session_id)) + snapshot = manager.restore_published(store.prepare_replay(session_id)) except ( - RecordingPreparationQueueFull, SessionNotFoundError, SessionNotReplayableError, SessionIntegrityError, ValueError, ): return None + if snapshot is None: + return None document: dict[str, Any] = { "preparation_id": snapshot.preparation_id, "state": snapshot.state, @@ -1165,10 +1173,7 @@ def _recorded_media_manifest_document( "segments": [ { "sequence": segment.sequence, - "url": ( - f"{base}/epochs/{epoch.ordinal}/segments/" - f"{segment.sequence}.m4s" - ), + "url": (f"{base}/epochs/{epoch.ordinal}/segments/{segment.sequence}.m4s"), "byte_length": segment.byte_length, "sha256": segment.sha256, } @@ -1198,19 +1203,13 @@ def _resolve_recorded_media_manifest( detail="Некорректный идентификатор записанного медиаканала.", ) from exc snapshot = manager.status(session_id) - if ( - snapshot is None - or snapshot.state != "ready" - or snapshot.recorded_media is None - ): + if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None: raise HTTPException( status_code=409, detail="Медиаканал ещё не подготовлен.", ) matches = tuple( - manifest - for manifest in snapshot.recorded_media - if manifest.artifact_id == artifact_id + manifest for manifest in snapshot.recorded_media if manifest.artifact_id == artifact_id ) if len(matches) != 1: raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.") diff --git a/tests/test_session_api.py b/tests/test_session_api.py index 8b0f885..ebaf049 100644 --- a/tests/test_session_api.py +++ b/tests/test_session_api.py @@ -574,6 +574,39 @@ def test_cold_replay_returns_quick_202_then_status_returns_ready_launch( manager.close() +def test_catalog_read_never_prepares_a_cold_historical_session(tmp_path: Path) -> None: + repository = tmp_path / "repo" + sessions = repository / "sessions" + session = make_legacy_session(sessions, "20260716T205632Z_viewer_live") + store = SessionStore(repository, data_dir=tmp_path / "data") + store.reconcile_archive(xgrids_k1_archive_source(sessions)) + calls = 0 + + def exporter(_source: Path, _destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + raise AssertionError("catalog reads must not invoke the exporter") + + materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter) + manager = SessionRecordingPreparationManager(materializer) + router = build_session_router( + store, + recording_materializer=materializer, + recording_preparation_manager=manager, + ) + list_route = endpoint(router, "/api/v1/observation-sessions", "GET") + try: + item = list_route(limit=20, cursor=None)["items"][0] + time.sleep(0.02) + + assert item["id"] == session.name + assert item["preparation"] is None + assert manager.status(session.name) is None + assert calls == 0 + finally: + manager.close() + + def test_recording_response_releases_pin_once_when_asgi_send_fails( tmp_path: Path, ) -> None: diff --git a/tests/test_session_catalog_lifecycle.py b/tests/test_session_catalog_lifecycle.py index b5a9432..6ca3811 100644 --- a/tests/test_session_catalog_lifecycle.py +++ b/tests/test_session_catalog_lifecycle.py @@ -160,7 +160,7 @@ def test_catalog_refresh_failure_is_a_stable_service_error(tmp_path: Path) -> No assert "private local path" not in str(error.value.detail) -def test_startup_warmup_enqueues_every_finalized_replayable_session( +def test_startup_scan_baselines_historical_sessions_without_enqueuing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -188,17 +188,14 @@ def test_startup_warmup_enqueues_every_finalized_replayable_session( monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) try: - enqueued = app_module.enqueue_replayable_recordings() - deadline = time.monotonic() + 2 - snapshot = manager.status(session.name) - while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline: - time.sleep(0.005) - snapshot = manager.status(session.name) + historical = set(app_module.finalized_replayable_recording_ids()) + newly_finalized = app_module.newly_finalized_recording_ids(None, historical) + enqueued = app_module.enqueue_replayable_recordings(newly_finalized) - assert enqueued == (session.name,) - assert snapshot is not None - assert snapshot.state == "ready" - assert snapshot.recording is not None + assert historical == {session.name} + assert newly_finalized == () + assert enqueued == () + assert manager.status(session.name) is None finally: manager.close() @@ -233,7 +230,7 @@ def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) try: - enqueued = app_module.enqueue_replayable_recordings() + enqueued = app_module.enqueue_replayable_recordings((stale.name, good.name)) deadline = time.monotonic() + 2 snapshot = manager.status(good.name) while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline: @@ -283,7 +280,7 @@ def test_reconciler_requeues_only_a_restart_interrupted_job( monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) try: - app_module.enqueue_replayable_recordings() + app_module.enqueue_replayable_recordings((session.name,)) assert started.wait(timeout=1) manager.close(timeout=0.001) manager.start() @@ -291,15 +288,13 @@ def test_reconciler_requeues_only_a_restart_interrupted_job( deadline = time.monotonic() + 2 snapshot = manager.status(session.name) - while ( - snapshot is None or snapshot.state != "cancelled" - ) and time.monotonic() < deadline: + while (snapshot is None or snapshot.state != "cancelled") and time.monotonic() < deadline: time.sleep(0.005) snapshot = manager.status(session.name) assert snapshot is not None and snapshot.state == "cancelled" interrupted_id = snapshot.preparation_id - app_module.enqueue_replayable_recordings() + app_module.enqueue_replayable_recordings((session.name,)) deadline = time.monotonic() + 2 while time.monotonic() < deadline: snapshot = manager.status(session.name) @@ -336,18 +331,16 @@ def test_reconciler_does_not_loop_retry_a_genuine_failed_job( monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) try: - app_module.enqueue_replayable_recordings() + app_module.enqueue_replayable_recordings((session.name,)) deadline = time.monotonic() + 2 snapshot = manager.status(session.name) - while ( - snapshot is None or snapshot.state != "failed" - ) and time.monotonic() < deadline: + while (snapshot is None or snapshot.state != "failed") and time.monotonic() < deadline: time.sleep(0.005) snapshot = manager.status(session.name) assert snapshot is not None and snapshot.state == "failed" failed_id = snapshot.preparation_id - app_module.enqueue_replayable_recordings() + app_module.enqueue_replayable_recordings((session.name,)) time.sleep(0.05) unchanged = manager.status(session.name) assert unchanged is not None diff --git a/tests/test_session_preparation.py b/tests/test_session_preparation.py index 80480bb..f96d45c 100644 --- a/tests/test_session_preparation.py +++ b/tests/test_session_preparation.py @@ -231,6 +231,46 @@ def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Pat manager.close() +def test_published_package_restores_after_process_restart_without_export( + tmp_path: Path, +) -> None: + command = _command(tmp_path / "session") + private_root = tmp_path / "private" + calls = 0 + + def exporter(source: Path, destination: Path) -> dict[str, object]: + nonlocal calls + calls += 1 + return _summary(source, destination, b"durable-recording") + + first = SessionRecordingPreparationManager( + SessionRecordingMaterializer(private_root, exporter=exporter) + ) + try: + first.enqueue(command) + ready = _wait_for_state(first, command.session_id, {"ready"}) + assert ready.recording is not None + finally: + first.close() + + def forbidden_exporter(_source: Path, _destination: Path) -> dict[str, object]: + raise AssertionError("a published package must not be exported again") + + restarted = SessionRecordingPreparationManager( + SessionRecordingMaterializer(private_root, exporter=forbidden_exporter) + ) + try: + restored = restarted.restore_published(command) + + assert restored is not None + assert restored.state == "ready" + assert restored.recording is not None + assert restored.recording.path.read_bytes() == b"durable-recording" + assert calls == 1 + finally: + restarted.close() + + def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer( tmp_path: Path, ) -> None: