fix(archive): preserve prepared sessions across restarts

This commit is contained in:
DCCONSTRUCTIONS 2026-07-18 11:08:27 +03:00
parent aa3680948f
commit 574a494759
13 changed files with 398 additions and 132 deletions

View File

@ -11,7 +11,7 @@ import type {
} from "../core/observation/useObservationSessions"; } from "../core/observation/useObservationSessions";
import "../styles/observation-sessions.css"; import "../styles/observation-sessions.css";
type SessionVisualState = "ready" | "processing" | "error"; type SessionVisualState = "ready" | "processing" | "cold" | "error";
export function observationSessionVisualState( export function observationSessionVisualState(
session: ObservationSessionSummary, session: ObservationSessionSummary,
@ -26,12 +26,14 @@ export function observationSessionVisualState(
if (session.preparation.state === "ready" && session.replayable) return "ready"; if (session.preparation.state === "ready" && session.replayable) return "ready";
return "error"; return "error";
} }
return session.status === "recording" ? "processing" : "error"; if (session.status === "recording") return "processing";
return session.replayable ? "cold" : "error";
} }
export function observationSessionVisualLabel(state: SessionVisualState): string { export function observationSessionVisualLabel(state: SessionVisualState): string {
if (state === "ready") return "Готово"; if (state === "ready") return "Готово";
if (state === "processing") return "Обработка"; if (state === "processing") return "Обработка";
if (state === "cold") return "Подготовить";
return "Ошибка"; return "Ошибка";
} }

View File

@ -137,8 +137,12 @@
animation: observation-session-processing 1.1s ease-in-out infinite; animation: observation-session-processing 1.1s ease-in-out infinite;
} }
.observation-session-option i[data-session-visual-state="cold"],
.observation-session-option i[data-session-visual-state="error"] { .observation-session-option i[data-session-visual-state="error"] {
background: var(--nodedc-text-muted); background: var(--nodedc-text-muted);
}
.observation-session-option i[data-session-visual-state="error"] {
opacity: 0.48; opacity: 0.48;
} }

View File

@ -160,7 +160,7 @@ test("session catalog exposes authoritative background preparation state", () =>
}); });
}); });
test("saved-session rows use only authoritative ready, processing and error states", () => { test("saved-session rows distinguish durable, active, cold and failed states", () => {
const makeDecoded = (state, overrides = {}) => decodeObservationSessionCatalog({ const makeDecoded = (state, overrides = {}) => decodeObservationSessionCatalog({
items: [session({ items: [session({
status: "interrupted", status: "interrupted",
@ -187,7 +187,7 @@ test("saved-session rows use only authoritative ready, processing and error stat
observationSessionVisualState(makeDecoded("failed", { status: "recording" })), observationSessionVisualState(makeDecoded("failed", { status: "recording" })),
"error", "error",
); );
assert.equal(observationSessionVisualState(makeDecoded(null)), "error"); assert.equal(observationSessionVisualState(makeDecoded(null)), "cold");
assert.equal( assert.equal(
observationSessionVisualState(makeDecoded(null, { status: "recording" })), observationSessionVisualState(makeDecoded(null, { status: "recording" })),
"processing", "processing",
@ -206,6 +206,7 @@ test("saved-session rows use only authoritative ready, processing and error stat
); );
assert.equal(observationSessionVisualLabel("ready"), "Готово"); assert.equal(observationSessionVisualLabel("ready"), "Готово");
assert.equal(observationSessionVisualLabel("processing"), "Обработка"); assert.equal(observationSessionVisualLabel("processing"), "Обработка");
assert.equal(observationSessionVisualLabel("cold"), "Подготовить");
assert.equal(observationSessionVisualLabel("error"), "Ошибка"); assert.equal(observationSessionVisualLabel("error"), "Ошибка");
}); });

View File

@ -57,22 +57,26 @@ not mutate playback state, so changing accumulation, size, color or visibility
does not pause the active replay or replace its camera view. does not pause the active replay or replace its camera view.
Completed and recovered sessions are discovered on startup and by the catalog Completed and recovered sessions are discovered on startup and by the catalog
reconciler, then deduplicated into one bounded, single-worker preparation queue. reconciler. The first scan is only a historical baseline: it never enqueues the
The worker validates the native source, exports point/pose data, verifies source existing archive. A session that becomes finalized after that baseline is
stability and both SHA-256 digests, finalizes once, and atomically publishes the deduplicated into the bounded, single-worker preparation queue. The worker
validates the native source, exports point/pose data, verifies source stability
and both SHA-256 digests, finalizes once, and atomically publishes the
digest-bound derived RRD plus its cache sidecar. Before the same job becomes digest-bound derived RRD plus its cache sidecar. Before the same job becomes
`ready`, it also parses and validates every archived camera index and retains `ready`, it also parses and validates every archived camera index and publishes
the immutable manifest generation in memory. The catalog exposes preparation the immutable media-manifest sidecar. The catalog exposes preparation state and
state and progress while this runs; a recording URL and camera descriptors are progress while this runs; a recording URL and camera descriptors are returned
returned only after the complete launch generation has passed validation. only after the complete launch generation has passed validation.
Preparation is a backend lifecycle, not a viewer action: sealing or recovering Preparation is a backend lifecycle, not a viewer computation. Reopening,
a native session makes it eligible for the reconciler, which builds the derived seeking, refreshing the catalog or changing the workspace never reruns a
RRD once in the background. Reopening, seeking or changing the workspace reads published conversion. After a process restart, a catalog/replay read restores a
that prepared artifact and never reruns conversion. A cold process may validate complete compatible RRD and camera package from its durable sidecars using
an existing cache and rebuild its in-memory camera manifest in the worker, but bounded schema and stat-identity checks; it does not hash native evidence, parse
no replay, status, RRD or manifest request hashes native evidence or parses a camera indexes or invoke the exporter. A historical session whose package is
camera index synchronously. missing, stale or from an incompatible cache schema remains cold. Only an
explicit replay/RRD request schedules that one session for preparation; listing
the catalog and finalizing another session do not schedule it.
The session menu uses three operator indicators: The session menu uses three operator indicators:
@ -80,6 +84,7 @@ The session menu uses three operator indicators:
| --- | --- | --- | | --- | --- | --- |
| Solid green + `Готово` | `ready` | Verified RRD is available. | | Solid green + `Готово` | `ready` | Verified RRD is available. |
| Pulsing green + `Обработка` | `queued`, `validating`, `exporting`, `finalizing` | Background preparation is active. | | Pulsing green + `Обработка` | `queued`, `validating`, `exporting`, `finalizing` | Background preparation is active. |
| Solid gray + `Подготовить` | replayable session with no compatible published package | No work is running; choosing the row schedules this session only. |
| Dim gray + `Ошибка` | failed, cancelled, non-replayable or invalid session | No launchable recording; inspect the message or retry. | | Dim gray + `Ошибка` | failed, cancelled, non-replayable or invalid session | No launchable recording; inspect the message or retry. |
Saved-session status never uses yellow or red. Switching sessions aborts only Saved-session status never uses yellow or red. Switching sessions aborts only
@ -93,7 +98,8 @@ Later openings reuse a verified, digest-bound cache. Derived RRD cache v9 is
intentionally incompatible with v8 and older generations. In addition to plugin intentionally incompatible with v8 and older generations. In addition to plugin
and ordered-artifact identity, it binds the durable clock origin and active and ordered-artifact identity, it binds the durable clock origin and active
envelope and materializes real `session_time = 0` origin and sealed completion envelope and materializes real `session_time = 0` origin and sealed completion
rows. Older generations are rebuilt once in the background. rows. Older generations remain cold after startup and are rebuilt only when an
operator explicitly opens that session.
The browser may use the strict `source_url` with `If-Match`, while the embedded The browser may use the strict `source_url` with `If-Match`, while the embedded
Rerun loader uses the canonical `viewer_source_url` whose lowercase SHA-256 Rerun loader uses the canonical `viewer_source_url` whose lowercase SHA-256
`generation` query is bound to the same launch descriptor. A missing or stale `generation` query is bound to the same launch descriptor. A missing or stale
@ -194,11 +200,13 @@ Native `.k1mqtt` bytes with aligned metadata, the capture-clock artifacts and
canonical camera fMP4 archives are the evidence source of truth. The derived RRD canonical camera fMP4 archives are the evidence source of truth. The derived RRD
contains every decodable point and pose frame plus every valid `ModelingReport` contains every decodable point and pose frame plus every valid `ModelingReport`
distance/speed/scan-time sample from that source, but remains a rebuildable view distance/speed/scan-time sample from that source, but remains a rebuildable view
rather than an evidence master. A cache entry is rebuilt only when native source rather than an evidence master. A cache entry is eligible for rebuild only when
identity/digests change or an incompatible derived-data export revision is native source identity/digests change or an incompatible derived-data export
introduced. A UI blueprint or workspace-layout revision never invalidates or revision is introduced. Historical entries are not rebuilt during startup or
rewrites the data RRD. Cache v8 and older payloads are not reusable as v9 because because another session is finalized; an explicit open schedules the affected
they do not bind both real capture-envelope endpoints. entry. A UI blueprint or workspace-layout revision never invalidates or rewrites
the data RRD. Cache v8 and older payloads are not reusable as v9 because they do
not bind both real capture-envelope endpoints.
`ModelingReport` time is the device's `ScanTime` counter at two ticks per second; `ModelingReport` time is the device's `ScanTime` counter at two ticks per second;
distance and speed are the reported `MoveDistance`/`MoveSpeed` values. Live state distance and speed are the reported `MoveDistance`/`MoveSpeed` values. Live state
@ -318,9 +326,10 @@ written under the private derived cache with a schema, generation and checksum.
Publication uses a private temporary file, file and directory `fsync`, and atomic Publication uses a private temporary file, file and directory `fsync`, and atomic
rename; startup scavenges crash-left temporary files. A restart reuses this rename; startup scavenges crash-left temporary files. A restart reuses this
sidecar after confined O(n) stat validation, without rereading, hashing or parsing sidecar after confined O(n) stat validation, without rereading, hashing or parsing
media. A missing, corrupt or stale sidecar is rebuilt only by the background media. A missing, corrupt or stale sidecar remains cold until an explicit open
worker. No replay, status, manifest or payload request performs conversion or schedules the complete session package in the background worker. No catalog,
recalculates these intervals. status, manifest or payload request itself performs conversion or recalculates
these intervals.
Every derived RRD contains a real `session_time = 0` row at the internal Every derived RRD contains a real `session_time = 0` row at the internal
`/__mission_core/session_origin` entity. The anchor is deliberately outside `/__mission_core/session_origin` entity. The anchor is deliberately outside

View File

@ -73,11 +73,15 @@ committed metadata; replay stops at the last validated aligned boundary.
### Recorded spatial playback ### Recorded spatial playback
When a native capture is normally sealed or recovery-sealed, a bounded When a native capture is normally sealed or recovery-sealed during the current
single-worker reconciler automatically prepares its private derived `.rrd`. process lifetime, a bounded single-worker reconciler automatically prepares its
Preparation is a backend lifecycle and is never executed by an HTTP replay private derived `.rrd`. The reconciler's first scan establishes a historical
request. Reopening, seeking, switching tabs or changing display settings reads baseline and does not enqueue old sessions. Preparation is a backend lifecycle
the already prepared artifact and does not rerun conversion. Export is lossless and is never executed inside an HTTP replay request: an explicit request may
enqueue one cold historical package and returns `202` while the worker owns the
conversion. Reopening, seeking, switching tabs, listing the catalog or changing
display settings reads the already published artifact and does not rerun
conversion. Export is lossless
with respect to every decodable point and pose message in the native capture; with respect to every decodable point and pose message in the native capture;
it does not pass through the bounded live-preview queue. The RRD uses the it does not pass through the bounded live-preview queue. The RRD uses the
recording-local `session_time` duration timeline, while the receive wall clock recording-local `session_time` duration timeline, while the receive wall clock
@ -123,21 +127,23 @@ future replay v3 will need distinct spatial and session-union ranges.
The path-free descriptor is atomically persisted in the private derived cache The path-free descriptor is atomically persisted in the private derived cache
with its checksum and a stat identity covering the native timing origin and with its checksum and a stat identity covering the native timing origin and
every camera summary, index, init and segment file. Restart performs confined every camera summary, index, init and segment file. Restart performs confined
stat validation and reuses an unchanged descriptor without reading/hashing media; stat validation and reuses an unchanged descriptor without reading/hashing media.
a missing, corrupt or stale sidecar is rebuilt in the background. An unparseable A missing, corrupt or stale historical sidecar stays cold until that session is
or ambiguous fragment fails readiness rather than exposing a fake seekable explicitly opened; startup, catalog reads and completion of another session do
camera. Replay requests consume the prepared descriptor and never repeat media not rebuild it. An unparseable or ambiguous fragment fails readiness rather than
conversion or timing analysis. Launch and manifest aggregate byte counts must exposing a fake seekable camera. Replay requests consume the prepared descriptor
agree before the browser admits camera payload downloads. and never repeat media conversion or timing analysis. Launch and manifest
aggregate byte counts must agree before the browser admits camera payload
downloads.
The derived RRD writes an actual zero-time anchor at the internal The derived RRD writes an actual zero-time anchor at the internal
`/__mission_core/session_origin` entity. Keeping it outside `/world` prevents a `/__mission_core/session_origin` entity. Keeping it outside `/world` prevents a
synthetic visualization layer while ensuring that the decoded RRD timeline, synthetic visualization layer while ensuring that the decoded RRD timeline,
not merely its summary document, begins at session time zero. This changes the not merely its summary document, begins at session time zero. This changes the
derived payload contract. Cache v7 keeps that anchor and additionally binds the derived payload contract. Cache v9 keeps that anchor, binds the owning plugin,
derived recording to the owning plugin ID, primary artifact and ordered source ordered source artifacts and capture-envelope endpoints, and materializes the
artifact set; it rejects v6 and older sidecars and performs one background sealed completion rows. Older sidecars are incompatible and remain cold until
rebuild. that session is explicitly opened.
Replay v2 exposes two paths to the same pinned generation. `source_url` retains Replay v2 exposes two paths to the same pinned generation. `source_url` retains
the strict `If-Match: "sha256:…"` contract for Mission Core clients; the strict `If-Match: "sha256:…"` contract for Mission Core clients;

View File

@ -175,6 +175,42 @@ class RecordedMediaInspector:
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest) self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
return manifest return manifest
def restore_prepared(
self,
artifact: RecordedMediaArtifact,
replay: ReplayCommand,
) -> RecordedMediaManifest | None:
"""Restore one previously prepared manifest without rebuilding it.
A catalog read or process restart must never turn an old camera archive
into fresh preparation work. The durable sidecar is accepted only when
its complete source stat identity still matches the sealed archive.
Missing or stale sidecars remain cold until the operator explicitly
requests that session.
"""
if artifact.session_id != replay.session_id:
raise SessionIntegrityError("recorded media does not belong to replay session")
epoch_paths = _epoch_paths(artifact.source_path)
key = (artifact.session_id, artifact.artifact_id)
with self._lock:
cached = self._cache.get(key)
if cached is not None:
identity = _prepared_source_identity(
replay,
epoch_paths,
cached.manifest.epochs,
)
if cached.identity == identity:
return cached.manifest
prepared = self._load_prepared_sidecar(artifact, replay, epoch_paths)
if prepared is None:
return None
manifest, identity = prepared
with self._lock:
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
return manifest
def _load_prepared_sidecar( def _load_prepared_sidecar(
self, self,
artifact: RecordedMediaArtifact, artifact: RecordedMediaArtifact,

View File

@ -111,6 +111,11 @@ class SessionRecordingPreparationManager:
tuple[RecordedMediaManifest, ...], tuple[RecordedMediaManifest, ...],
] ]
| None = None, | None = None,
ready_restorer: Callable[
[ReplayCommand, MaterializedRecording],
tuple[RecordedMediaManifest, ...] | None,
]
| None = None,
) -> None: ) -> None:
if queue_capacity < 1: if queue_capacity < 1:
raise ValueError("recording preparation queue capacity must be positive") raise ValueError("recording preparation queue capacity must be positive")
@ -123,6 +128,7 @@ class SessionRecordingPreparationManager:
# hung exporter must become observable to the browser stall detector. # hung exporter must become observable to the browser stall detector.
self._heartbeat_interval_seconds = heartbeat_interval_seconds self._heartbeat_interval_seconds = heartbeat_interval_seconds
self._ready_preparer = ready_preparer self._ready_preparer = ready_preparer
self._ready_restorer = ready_restorer
self._guard = threading.RLock() self._guard = threading.RLock()
self._current_by_session: dict[str, _PreparationJob] = {} self._current_by_session: dict[str, _PreparationJob] = {}
self._closed = True self._closed = True
@ -237,6 +243,74 @@ class SessionRecordingPreparationManager:
self._current_by_session[command.session_id] = ready self._current_by_session[command.session_id] = ready
return self._snapshot_locked(ready) return self._snapshot_locked(ready)
def restore_published(
self,
command: ReplayCommand,
) -> RecordingPreparationSnapshot | None:
"""Restore a complete durable package without enqueueing preparation.
The RRD and every recorded-media descriptor must already have been
published by an earlier successful job. Missing/stale pieces return a
cold miss; this method never invokes an exporter or media parser.
"""
source_command = _source_command(command)
identity = _source_identity(source_command)
with self._guard:
current = self._current_by_session.get(command.session_id)
if (
current is not None
and current.source_identity == identity
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
):
return self._snapshot_locked(current)
recording = self.materializer.restore_published(source_command)
if recording is None:
with self._guard:
current = self._current_by_session.get(command.session_id)
if current is not None and current.state == "ready":
self._current_by_session.pop(command.session_id, None)
return None
if self._ready_preparer is None:
recorded_media: tuple[RecordedMediaManifest, ...] = ()
else:
if self._ready_restorer is None:
return None
restored_media = self._ready_restorer(source_command, recording)
if restored_media is None:
return None
recorded_media = restored_media
validate_recorded_media_timeline(
recorded_media,
recording_start_seconds=recording.timeline_start_ns / 1_000_000_000,
recording_end_seconds=recording.timeline_end_ns / 1_000_000_000,
)
with self._guard:
current = self._current_by_session.get(command.session_id)
if (
current is not None
and current.source_identity == identity
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
):
if current.state == "ready":
current.recording = recording
current.recorded_media = recorded_media
return self._snapshot_locked(current)
ready = _PreparationJob(
preparation_id=uuid4().hex,
source_identity=identity,
command=source_command,
state="ready",
progress=1.0,
recording=recording,
recorded_media=recorded_media,
)
self._current_by_session[command.session_id] = ready
return self._snapshot_locked(ready)
def resolve_cached_pinned( def resolve_cached_pinned(
self, self,
command: ReplayCommand, command: ReplayCommand,
@ -325,10 +399,7 @@ class SessionRecordingPreparationManager:
job is None job is None
or job.state != "ready" or job.state != "ready"
or job.recording is None or job.recording is None
or ( or (preparation_id is not None and job.preparation_id != preparation_id)
preparation_id is not None
and job.preparation_id != preparation_id
)
): ):
return None return None
recording = job.recording recording = job.recording
@ -456,9 +527,7 @@ class SessionRecordingPreparationManager:
recording_start_seconds=( recording_start_seconds=(
recording.timeline_start_ns / 1_000_000_000 recording.timeline_start_ns / 1_000_000_000
), ),
recording_end_seconds=( recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000),
recording.timeline_end_ns / 1_000_000_000
),
) )
except Exception: except Exception:
with self._guard: with self._guard:

View File

@ -232,6 +232,26 @@ class SessionRecordingMaterializer:
with self._lock_for(session_id): with self._lock_for(session_id):
return self._load_cached_recording(session_id, _validate_source(command)) return self._load_cached_recording(session_id, _validate_source(command))
def restore_published(self, command: ReplayCommand) -> MaterializedRecording | None:
"""Restore an immutable published cache using bounded metadata checks.
Full source/output digests are proved before the cache is published.
On a later process start the private cache is restored only when its
schema, source stat identities and derived-file stat identity still
match the durable sidecar. This keeps a prepared recording durable
across restarts without rereading gigabytes or starting an exporter.
"""
session_id = _validate_command_shape(command)
if not self._has_compatible_cache_candidate(session_id):
return None
with self._lock_for(session_id):
return self._load_cached_recording(
session_id,
_validate_source(command),
verify_digests=False,
)
def get_cached_pinned( def get_cached_pinned(
self, self,
command: ReplayCommand, command: ReplayCommand,
@ -430,6 +450,7 @@ class SessionRecordingMaterializer:
source: _ValidatedSource, source: _ValidatedSource,
*, *,
pin: bool = False, pin: bool = False,
verify_digests: bool = True,
) -> MaterializedRecording | None: ) -> MaterializedRecording | None:
# Cache validation and the optional lease are one transaction with # Cache validation and the optional lease are one transaction with
# eviction. This makes it safe for FileResponse to open the path after # eviction. This makes it safe for FileResponse to open the path after
@ -451,6 +472,7 @@ class SessionRecordingMaterializer:
source=source, source=source,
recording_path=recording_path, recording_path=recording_path,
sidecar_path=sidecar_path, sidecar_path=sidecar_path,
verify_digests=verify_digests,
) )
if cached is None: if cached is None:
return None return None
@ -661,6 +683,7 @@ class SessionRecordingMaterializer:
source: _ValidatedSource, source: _ValidatedSource,
recording_path: Path, recording_path: Path,
sidecar_path: Path, sidecar_path: Path,
verify_digests: bool = True,
) -> MaterializedRecording | None: ) -> MaterializedRecording | None:
if recording_path.is_symlink() or sidecar_path.is_symlink(): if recording_path.is_symlink() or sidecar_path.is_symlink():
return None return None
@ -698,12 +721,19 @@ class SessionRecordingMaterializer:
source_sha256: str | None = None source_sha256: str | None = None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True): for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
digest = _sha256_prefix_stable( digest = (
artifact.path, _sha256_prefix_stable(
artifact.file_stat, artifact.path,
artifact.replay_byte_length, artifact.file_stat,
artifact.replay_byte_length,
)
if verify_digests
else cached["sha256"]
) )
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256: if (
artifact.expected_sha256 is not None
and cached["sha256"] != artifact.expected_sha256
):
raise RecordingMaterializationError( raise RecordingMaterializationError(
"recording artifact digest no longer matches catalog" "recording artifact digest no longer matches catalog"
) )
@ -713,9 +743,11 @@ class SessionRecordingMaterializer:
source_sha256 = digest source_sha256 = digest
if source_sha256 is None or source_sha256 != document["source_sha256"]: if source_sha256 is None or source_sha256 != document["source_sha256"]:
return None return None
recording_sha256 = _sha256_stable(recording_path, recording_stat) recording_sha256 = document["recording_sha256"]
if recording_sha256 != document["recording_sha256"]: if verify_digests:
return None recording_sha256 = _sha256_stable(recording_path, recording_stat)
if recording_sha256 != document["recording_sha256"]:
return None
_chmod_best_effort(recording_path, 0o600) _chmod_best_effort(recording_path, 0o600)
_chmod_best_effort(sidecar_path, 0o600) _chmod_best_effort(sidecar_path, 0o600)
if document["schema_version"] != CACHE_SCHEMA: if document["schema_version"] != CACHE_SCHEMA:

View File

@ -1,7 +1,7 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
from collections.abc import AsyncIterator from collections.abc import AsyncIterator, Iterable
from contextlib import asynccontextmanager, suppress from contextlib import asynccontextmanager, suppress
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@ -65,9 +65,25 @@ def _prepare_recorded_media_for_launch(
) )
def _restore_recorded_media_for_launch(
command: ReplayCommand,
_: MaterializedRecording,
) -> tuple[RecordedMediaManifest, ...] | None:
"""Restore only previously published media descriptors."""
restored: list[RecordedMediaManifest] = []
for artifact in session_store.list_recorded_media(command.session_id):
manifest = session_recorded_media_inspector.restore_prepared(artifact, command)
if manifest is None:
return None
restored.append(manifest)
return tuple(restored)
session_recording_preparation_manager = SessionRecordingPreparationManager( session_recording_preparation_manager = SessionRecordingPreparationManager(
session_recording_materializer, session_recording_materializer,
ready_preparer=_prepare_recorded_media_for_launch, ready_preparer=_prepare_recorded_media_for_launch,
ready_restorer=_restore_recorded_media_for_launch,
) )
@ -82,47 +98,73 @@ def refresh_observation_catalog() -> tuple[str, ...]:
return tuple(dict.fromkeys(imported)) return tuple(dict.fromkeys(imported))
def enqueue_replayable_recordings() -> tuple[str, ...]: def finalized_replayable_recording_ids() -> tuple[str, ...]:
"""Reconcile finalized catalog sessions into the durable RRD work queue.""" """List finalized replayable catalog identities without scheduling work."""
enqueued: list[str] = [] finalized: list[str] = []
cursor: str | None = None cursor: str | None = None
while True: while True:
page = session_store.list_recent(limit=100, cursor=cursor) page = session_store.list_recent(limit=100, cursor=cursor)
for summary in page.items: finalized.extend(
if not summary.replayable or summary.status not in { summary.session_id
"ready", for summary in page.items
"interrupted", if summary.replayable and summary.status in {"ready", "interrupted", "failed"}
"failed", )
}:
continue
try:
command = session_store.prepare_replay(summary.session_id)
session_recording_preparation_manager.enqueue(
command,
retry_interrupted=True,
)
except RecordingPreparationQueueFull:
return tuple(enqueued)
except Exception:
# One stale/corrupt catalog row must not starve every valid
# finalized session that follows it in the reconciliation
# page. The row remains visible as failed/interrupted evidence
# and can be repaired independently.
continue
enqueued.append(summary.session_id)
cursor = page.next_cursor cursor = page.next_cursor
if cursor is None: if cursor is None:
return tuple(finalized)
def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]:
"""Schedule only explicitly selected newly finalized sessions."""
enqueued: list[str] = []
for session_id in dict.fromkeys(session_ids):
try:
command = session_store.prepare_replay(session_id)
session_recording_preparation_manager.enqueue(
command,
retry_interrupted=True,
)
except RecordingPreparationQueueFull:
return tuple(enqueued) return tuple(enqueued)
except Exception:
# One stale/corrupt row must not starve a later newly completed
# session. Historical cold caches remain operator-triggered.
continue
enqueued.append(session_id)
return tuple(enqueued)
def newly_finalized_recording_ids(
known_finalized: set[str] | None,
current_finalized: Iterable[str],
) -> tuple[str, ...]:
"""Return only post-startup completions; the first scan is a baseline."""
if known_finalized is None:
return ()
return tuple(sorted(set(current_finalized) - known_finalized))
async def _recording_preparation_reconciler() -> None: async def _recording_preparation_reconciler() -> None:
"""Discover newly completed/recovered captures without blocking requests.""" """Prepare sessions finalized during this process, never historical rows."""
known_finalized: set[str] | None = None
while True: while True:
try: try:
await asyncio.to_thread(refresh_observation_catalog) await asyncio.to_thread(refresh_observation_catalog)
await asyncio.to_thread(enqueue_replayable_recordings) finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
newly_finalized = newly_finalized_recording_ids(
known_finalized,
finalized,
)
if newly_finalized:
await asyncio.to_thread(
enqueue_replayable_recordings,
newly_finalized,
)
known_finalized = finalized
except Exception: except Exception:
# A transient filesystem/catalog failure must not permanently # A transient filesystem/catalog failure must not permanently
# disable preparation of sessions completed later in the run. # disable preparation of sessions completed later in the run.

View File

@ -45,9 +45,7 @@ SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$") SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
MAX_SAFE_INTEGER = 9_007_199_254_740_991 MAX_SAFE_INTEGER = 9_007_199_254_740_991
REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform" REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform"
IMMUTABLE_RECORDING_CACHE_CONTROL = ( IMMUTABLE_RECORDING_CACHE_CONTROL = "private, max-age=31536000, immutable, no-transform"
"private, max-age=31536000, immutable, no-transform"
)
class _ReleasingFileResponse(FileResponse): class _ReleasingFileResponse(FileResponse):
@ -321,13 +319,17 @@ def build_session_router(
if recording_preparation_manager is not None: if recording_preparation_manager is not None:
try: try:
# Enqueue performs only catalog identity/lstat checks. A cold snapshot = recording_preparation_manager.status(command.session_id)
# RRD cache and every camera index are validated exclusively if snapshot is None:
# by the process-owned worker. snapshot = await run_in_threadpool(
snapshot = recording_preparation_manager.enqueue( recording_preparation_manager.restore_published,
command, command,
retry_failed=True, )
) if snapshot is None:
snapshot = recording_preparation_manager.enqueue(
command,
retry_failed=True,
)
except RecordingPreparationQueueFull as exc: except RecordingPreparationQueueFull as exc:
raise HTTPException( raise HTTPException(
status_code=503, status_code=503,
@ -504,7 +506,12 @@ def build_session_router(
False, False,
False, False,
) )
snapshot = recording_preparation_manager.enqueue(command) snapshot = await run_in_threadpool(
recording_preparation_manager.restore_published,
command,
)
if snapshot is None:
snapshot = recording_preparation_manager.enqueue(command)
except SessionNotFoundError as exc: except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc raise HTTPException(status_code=404, detail=str(exc)) from exc
except (SessionNotReplayableError, SessionIntegrityError) as exc: except (SessionNotReplayableError, SessionIntegrityError) as exc:
@ -1036,15 +1043,16 @@ def _catalog_preparation_document(
snapshot = manager.status(session_id) snapshot = manager.status(session_id)
if snapshot is None: if snapshot is None:
try: try:
snapshot = manager.enqueue(store.prepare_replay(session_id)) snapshot = manager.restore_published(store.prepare_replay(session_id))
except ( except (
RecordingPreparationQueueFull,
SessionNotFoundError, SessionNotFoundError,
SessionNotReplayableError, SessionNotReplayableError,
SessionIntegrityError, SessionIntegrityError,
ValueError, ValueError,
): ):
return None return None
if snapshot is None:
return None
document: dict[str, Any] = { document: dict[str, Any] = {
"preparation_id": snapshot.preparation_id, "preparation_id": snapshot.preparation_id,
"state": snapshot.state, "state": snapshot.state,
@ -1165,10 +1173,7 @@ def _recorded_media_manifest_document(
"segments": [ "segments": [
{ {
"sequence": segment.sequence, "sequence": segment.sequence,
"url": ( "url": (f"{base}/epochs/{epoch.ordinal}/segments/{segment.sequence}.m4s"),
f"{base}/epochs/{epoch.ordinal}/segments/"
f"{segment.sequence}.m4s"
),
"byte_length": segment.byte_length, "byte_length": segment.byte_length,
"sha256": segment.sha256, "sha256": segment.sha256,
} }
@ -1198,19 +1203,13 @@ def _resolve_recorded_media_manifest(
detail="Некорректный идентификатор записанного медиаканала.", detail="Некорректный идентификатор записанного медиаканала.",
) from exc ) from exc
snapshot = manager.status(session_id) snapshot = manager.status(session_id)
if ( if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None:
snapshot is None
or snapshot.state != "ready"
or snapshot.recorded_media is None
):
raise HTTPException( raise HTTPException(
status_code=409, status_code=409,
detail="Медиаканал ещё не подготовлен.", detail="Медиаканал ещё не подготовлен.",
) )
matches = tuple( matches = tuple(
manifest manifest for manifest in snapshot.recorded_media if manifest.artifact_id == artifact_id
for manifest in snapshot.recorded_media
if manifest.artifact_id == artifact_id
) )
if len(matches) != 1: if len(matches) != 1:
raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.") raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.")

View File

@ -574,6 +574,39 @@ def test_cold_replay_returns_quick_202_then_status_returns_ready_launch(
manager.close() manager.close()
def test_catalog_read_never_prepares_a_cold_historical_session(tmp_path: Path) -> None:
repository = tmp_path / "repo"
sessions = repository / "sessions"
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
store = SessionStore(repository, data_dir=tmp_path / "data")
store.reconcile_archive(xgrids_k1_archive_source(sessions))
calls = 0
def exporter(_source: Path, _destination: Path) -> dict[str, object]:
nonlocal calls
calls += 1
raise AssertionError("catalog reads must not invoke the exporter")
materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter)
manager = SessionRecordingPreparationManager(materializer)
router = build_session_router(
store,
recording_materializer=materializer,
recording_preparation_manager=manager,
)
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
try:
item = list_route(limit=20, cursor=None)["items"][0]
time.sleep(0.02)
assert item["id"] == session.name
assert item["preparation"] is None
assert manager.status(session.name) is None
assert calls == 0
finally:
manager.close()
def test_recording_response_releases_pin_once_when_asgi_send_fails( def test_recording_response_releases_pin_once_when_asgi_send_fails(
tmp_path: Path, tmp_path: Path,
) -> None: ) -> None:

View File

@ -160,7 +160,7 @@ def test_catalog_refresh_failure_is_a_stable_service_error(tmp_path: Path) -> No
assert "private local path" not in str(error.value.detail) assert "private local path" not in str(error.value.detail)
def test_startup_warmup_enqueues_every_finalized_replayable_session( def test_startup_scan_baselines_historical_sessions_without_enqueuing(
tmp_path: Path, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
@ -188,17 +188,14 @@ def test_startup_warmup_enqueues_every_finalized_replayable_session(
monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try: try:
enqueued = app_module.enqueue_replayable_recordings() historical = set(app_module.finalized_replayable_recording_ids())
deadline = time.monotonic() + 2 newly_finalized = app_module.newly_finalized_recording_ids(None, historical)
snapshot = manager.status(session.name) enqueued = app_module.enqueue_replayable_recordings(newly_finalized)
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
time.sleep(0.005)
snapshot = manager.status(session.name)
assert enqueued == (session.name,) assert historical == {session.name}
assert snapshot is not None assert newly_finalized == ()
assert snapshot.state == "ready" assert enqueued == ()
assert snapshot.recording is not None assert manager.status(session.name) is None
finally: finally:
manager.close() manager.close()
@ -233,7 +230,7 @@ def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session
monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try: try:
enqueued = app_module.enqueue_replayable_recordings() enqueued = app_module.enqueue_replayable_recordings((stale.name, good.name))
deadline = time.monotonic() + 2 deadline = time.monotonic() + 2
snapshot = manager.status(good.name) snapshot = manager.status(good.name)
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline: while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
@ -283,7 +280,7 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try: try:
app_module.enqueue_replayable_recordings() app_module.enqueue_replayable_recordings((session.name,))
assert started.wait(timeout=1) assert started.wait(timeout=1)
manager.close(timeout=0.001) manager.close(timeout=0.001)
manager.start() manager.start()
@ -291,15 +288,13 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
deadline = time.monotonic() + 2 deadline = time.monotonic() + 2
snapshot = manager.status(session.name) snapshot = manager.status(session.name)
while ( while (snapshot is None or snapshot.state != "cancelled") and time.monotonic() < deadline:
snapshot is None or snapshot.state != "cancelled"
) and time.monotonic() < deadline:
time.sleep(0.005) time.sleep(0.005)
snapshot = manager.status(session.name) snapshot = manager.status(session.name)
assert snapshot is not None and snapshot.state == "cancelled" assert snapshot is not None and snapshot.state == "cancelled"
interrupted_id = snapshot.preparation_id interrupted_id = snapshot.preparation_id
app_module.enqueue_replayable_recordings() app_module.enqueue_replayable_recordings((session.name,))
deadline = time.monotonic() + 2 deadline = time.monotonic() + 2
while time.monotonic() < deadline: while time.monotonic() < deadline:
snapshot = manager.status(session.name) snapshot = manager.status(session.name)
@ -336,18 +331,16 @@ def test_reconciler_does_not_loop_retry_a_genuine_failed_job(
monkeypatch.setattr(app_module, "session_store", store) monkeypatch.setattr(app_module, "session_store", store)
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager) monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
try: try:
app_module.enqueue_replayable_recordings() app_module.enqueue_replayable_recordings((session.name,))
deadline = time.monotonic() + 2 deadline = time.monotonic() + 2
snapshot = manager.status(session.name) snapshot = manager.status(session.name)
while ( while (snapshot is None or snapshot.state != "failed") and time.monotonic() < deadline:
snapshot is None or snapshot.state != "failed"
) and time.monotonic() < deadline:
time.sleep(0.005) time.sleep(0.005)
snapshot = manager.status(session.name) snapshot = manager.status(session.name)
assert snapshot is not None and snapshot.state == "failed" assert snapshot is not None and snapshot.state == "failed"
failed_id = snapshot.preparation_id failed_id = snapshot.preparation_id
app_module.enqueue_replayable_recordings() app_module.enqueue_replayable_recordings((session.name,))
time.sleep(0.05) time.sleep(0.05)
unchanged = manager.status(session.name) unchanged = manager.status(session.name)
assert unchanged is not None assert unchanged is not None

View File

@ -231,6 +231,46 @@ def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Pat
manager.close() manager.close()
def test_published_package_restores_after_process_restart_without_export(
tmp_path: Path,
) -> None:
command = _command(tmp_path / "session")
private_root = tmp_path / "private"
calls = 0
def exporter(source: Path, destination: Path) -> dict[str, object]:
nonlocal calls
calls += 1
return _summary(source, destination, b"durable-recording")
first = SessionRecordingPreparationManager(
SessionRecordingMaterializer(private_root, exporter=exporter)
)
try:
first.enqueue(command)
ready = _wait_for_state(first, command.session_id, {"ready"})
assert ready.recording is not None
finally:
first.close()
def forbidden_exporter(_source: Path, _destination: Path) -> dict[str, object]:
raise AssertionError("a published package must not be exported again")
restarted = SessionRecordingPreparationManager(
SessionRecordingMaterializer(private_root, exporter=forbidden_exporter)
)
try:
restored = restarted.restore_published(command)
assert restored is not None
assert restored.state == "ready"
assert restored.recording is not None
assert restored.recording.path.read_bytes() == b"durable-recording"
assert calls == 1
finally:
restarted.close()
def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer( def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer(
tmp_path: Path, tmp_path: Path,
) -> None: ) -> None: