fix(archive): preserve prepared sessions across restarts
This commit is contained in:
parent
aa3680948f
commit
574a494759
|
|
@ -11,7 +11,7 @@ import type {
|
|||
} from "../core/observation/useObservationSessions";
|
||||
import "../styles/observation-sessions.css";
|
||||
|
||||
type SessionVisualState = "ready" | "processing" | "error";
|
||||
type SessionVisualState = "ready" | "processing" | "cold" | "error";
|
||||
|
||||
export function observationSessionVisualState(
|
||||
session: ObservationSessionSummary,
|
||||
|
|
@ -26,12 +26,14 @@ export function observationSessionVisualState(
|
|||
if (session.preparation.state === "ready" && session.replayable) return "ready";
|
||||
return "error";
|
||||
}
|
||||
return session.status === "recording" ? "processing" : "error";
|
||||
if (session.status === "recording") return "processing";
|
||||
return session.replayable ? "cold" : "error";
|
||||
}
|
||||
|
||||
export function observationSessionVisualLabel(state: SessionVisualState): string {
|
||||
if (state === "ready") return "Готово";
|
||||
if (state === "processing") return "Обработка";
|
||||
if (state === "cold") return "Подготовить";
|
||||
return "Ошибка";
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -137,8 +137,12 @@
|
|||
animation: observation-session-processing 1.1s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.observation-session-option i[data-session-visual-state="cold"],
|
||||
.observation-session-option i[data-session-visual-state="error"] {
|
||||
background: var(--nodedc-text-muted);
|
||||
}
|
||||
|
||||
.observation-session-option i[data-session-visual-state="error"] {
|
||||
opacity: 0.48;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ test("session catalog exposes authoritative background preparation state", () =>
|
|||
});
|
||||
});
|
||||
|
||||
test("saved-session rows use only authoritative ready, processing and error states", () => {
|
||||
test("saved-session rows distinguish durable, active, cold and failed states", () => {
|
||||
const makeDecoded = (state, overrides = {}) => decodeObservationSessionCatalog({
|
||||
items: [session({
|
||||
status: "interrupted",
|
||||
|
|
@ -187,7 +187,7 @@ test("saved-session rows use only authoritative ready, processing and error stat
|
|||
observationSessionVisualState(makeDecoded("failed", { status: "recording" })),
|
||||
"error",
|
||||
);
|
||||
assert.equal(observationSessionVisualState(makeDecoded(null)), "error");
|
||||
assert.equal(observationSessionVisualState(makeDecoded(null)), "cold");
|
||||
assert.equal(
|
||||
observationSessionVisualState(makeDecoded(null, { status: "recording" })),
|
||||
"processing",
|
||||
|
|
@ -206,6 +206,7 @@ test("saved-session rows use only authoritative ready, processing and error stat
|
|||
);
|
||||
assert.equal(observationSessionVisualLabel("ready"), "Готово");
|
||||
assert.equal(observationSessionVisualLabel("processing"), "Обработка");
|
||||
assert.equal(observationSessionVisualLabel("cold"), "Подготовить");
|
||||
assert.equal(observationSessionVisualLabel("error"), "Ошибка");
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -57,22 +57,26 @@ not mutate playback state, so changing accumulation, size, color or visibility
|
|||
does not pause the active replay or replace its camera view.
|
||||
|
||||
Completed and recovered sessions are discovered on startup and by the catalog
|
||||
reconciler, then deduplicated into one bounded, single-worker preparation queue.
|
||||
The worker validates the native source, exports point/pose data, verifies source
|
||||
stability and both SHA-256 digests, finalizes once, and atomically publishes the
|
||||
reconciler. The first scan is only a historical baseline: it never enqueues the
|
||||
existing archive. A session that becomes finalized after that baseline is
|
||||
deduplicated into the bounded, single-worker preparation queue. The worker
|
||||
validates the native source, exports point/pose data, verifies source stability
|
||||
and both SHA-256 digests, finalizes once, and atomically publishes the
|
||||
digest-bound derived RRD plus its cache sidecar. Before the same job becomes
|
||||
`ready`, it also parses and validates every archived camera index and retains
|
||||
the immutable manifest generation in memory. The catalog exposes preparation
|
||||
state and progress while this runs; a recording URL and camera descriptors are
|
||||
returned only after the complete launch generation has passed validation.
|
||||
`ready`, it also parses and validates every archived camera index and publishes
|
||||
the immutable media-manifest sidecar. The catalog exposes preparation state and
|
||||
progress while this runs; a recording URL and camera descriptors are returned
|
||||
only after the complete launch generation has passed validation.
|
||||
|
||||
Preparation is a backend lifecycle, not a viewer action: sealing or recovering
|
||||
a native session makes it eligible for the reconciler, which builds the derived
|
||||
RRD once in the background. Reopening, seeking or changing the workspace reads
|
||||
that prepared artifact and never reruns conversion. A cold process may validate
|
||||
an existing cache and rebuild its in-memory camera manifest in the worker, but
|
||||
no replay, status, RRD or manifest request hashes native evidence or parses a
|
||||
camera index synchronously.
|
||||
Preparation is a backend lifecycle, not a viewer computation. Reopening,
|
||||
seeking, refreshing the catalog or changing the workspace never reruns a
|
||||
published conversion. After a process restart, a catalog/replay read restores a
|
||||
complete compatible RRD and camera package from its durable sidecars using
|
||||
bounded schema and stat-identity checks; it does not hash native evidence, parse
|
||||
camera indexes or invoke the exporter. A historical session whose package is
|
||||
missing, stale or from an incompatible cache schema remains cold. Only an
|
||||
explicit replay/RRD request schedules that one session for preparation; listing
|
||||
the catalog and finalizing another session do not schedule it.
|
||||
|
||||
The session menu uses three operator indicators:
|
||||
|
||||
|
|
@ -80,6 +84,7 @@ The session menu uses three operator indicators:
|
|||
| --- | --- | --- |
|
||||
| Solid green + `Готово` | `ready` | Verified RRD is available. |
|
||||
| Pulsing green + `Обработка` | `queued`, `validating`, `exporting`, `finalizing` | Background preparation is active. |
|
||||
| Solid gray + `Подготовить` | replayable session with no compatible published package | No work is running; choosing the row schedules this session only. |
|
||||
| Dim gray + `Ошибка` | failed, cancelled, non-replayable or invalid session | No launchable recording; inspect the message or retry. |
|
||||
|
||||
Saved-session status never uses yellow or red. Switching sessions aborts only
|
||||
|
|
@ -93,7 +98,8 @@ Later openings reuse a verified, digest-bound cache. Derived RRD cache v9 is
|
|||
intentionally incompatible with v8 and older generations. In addition to plugin
|
||||
and ordered-artifact identity, it binds the durable clock origin and active
|
||||
envelope and materializes real `session_time = 0` origin and sealed completion
|
||||
rows. Older generations are rebuilt once in the background.
|
||||
rows. Older generations remain cold after startup and are rebuilt only when an
|
||||
operator explicitly opens that session.
|
||||
The browser may use the strict `source_url` with `If-Match`, while the embedded
|
||||
Rerun loader uses the canonical `viewer_source_url` whose lowercase SHA-256
|
||||
`generation` query is bound to the same launch descriptor. A missing or stale
|
||||
|
|
@ -194,11 +200,13 @@ Native `.k1mqtt` bytes with aligned metadata, the capture-clock artifacts and
|
|||
canonical camera fMP4 archives are the evidence source of truth. The derived RRD
|
||||
contains every decodable point and pose frame plus every valid `ModelingReport`
|
||||
distance/speed/scan-time sample from that source, but remains a rebuildable view
|
||||
rather than an evidence master. A cache entry is rebuilt only when native source
|
||||
identity/digests change or an incompatible derived-data export revision is
|
||||
introduced. A UI blueprint or workspace-layout revision never invalidates or
|
||||
rewrites the data RRD. Cache v8 and older payloads are not reusable as v9 because
|
||||
they do not bind both real capture-envelope endpoints.
|
||||
rather than an evidence master. A cache entry is eligible for rebuild only when
|
||||
native source identity/digests change or an incompatible derived-data export
|
||||
revision is introduced. Historical entries are not rebuilt during startup or
|
||||
because another session is finalized; an explicit open schedules the affected
|
||||
entry. A UI blueprint or workspace-layout revision never invalidates or rewrites
|
||||
the data RRD. Cache v8 and older payloads are not reusable as v9 because they do
|
||||
not bind both real capture-envelope endpoints.
|
||||
|
||||
`ModelingReport` time is the device's `ScanTime` counter at two ticks per second;
|
||||
distance and speed are the reported `MoveDistance`/`MoveSpeed` values. Live state
|
||||
|
|
@ -318,9 +326,10 @@ written under the private derived cache with a schema, generation and checksum.
|
|||
Publication uses a private temporary file, file and directory `fsync`, and atomic
|
||||
rename; startup scavenges crash-left temporary files. A restart reuses this
|
||||
sidecar after confined O(n) stat validation, without rereading, hashing or parsing
|
||||
media. A missing, corrupt or stale sidecar is rebuilt only by the background
|
||||
worker. No replay, status, manifest or payload request performs conversion or
|
||||
recalculates these intervals.
|
||||
media. A missing, corrupt or stale sidecar remains cold until an explicit open
|
||||
schedules the complete session package in the background worker. No catalog,
|
||||
status, manifest or payload request itself performs conversion or recalculates
|
||||
these intervals.
|
||||
|
||||
Every derived RRD contains a real `session_time = 0` row at the internal
|
||||
`/__mission_core/session_origin` entity. The anchor is deliberately outside
|
||||
|
|
|
|||
|
|
@ -73,11 +73,15 @@ committed metadata; replay stops at the last validated aligned boundary.
|
|||
|
||||
### Recorded spatial playback
|
||||
|
||||
When a native capture is normally sealed or recovery-sealed, a bounded
|
||||
single-worker reconciler automatically prepares its private derived `.rrd`.
|
||||
Preparation is a backend lifecycle and is never executed by an HTTP replay
|
||||
request. Reopening, seeking, switching tabs or changing display settings reads
|
||||
the already prepared artifact and does not rerun conversion. Export is lossless
|
||||
When a native capture is normally sealed or recovery-sealed during the current
|
||||
process lifetime, a bounded single-worker reconciler automatically prepares its
|
||||
private derived `.rrd`. The reconciler's first scan establishes a historical
|
||||
baseline and does not enqueue old sessions. Preparation is a backend lifecycle
|
||||
and is never executed inside an HTTP replay request: an explicit request may
|
||||
enqueue one cold historical package and returns `202` while the worker owns the
|
||||
conversion. Reopening, seeking, switching tabs, listing the catalog or changing
|
||||
display settings reads the already published artifact and does not rerun
|
||||
conversion. Export is lossless
|
||||
with respect to every decodable point and pose message in the native capture;
|
||||
it does not pass through the bounded live-preview queue. The RRD uses the
|
||||
recording-local `session_time` duration timeline, while the receive wall clock
|
||||
|
|
@ -123,21 +127,23 @@ future replay v3 will need distinct spatial and session-union ranges.
|
|||
The path-free descriptor is atomically persisted in the private derived cache
|
||||
with its checksum and a stat identity covering the native timing origin and
|
||||
every camera summary, index, init and segment file. Restart performs confined
|
||||
stat validation and reuses an unchanged descriptor without reading/hashing media;
|
||||
a missing, corrupt or stale sidecar is rebuilt in the background. An unparseable
|
||||
or ambiguous fragment fails readiness rather than exposing a fake seekable
|
||||
camera. Replay requests consume the prepared descriptor and never repeat media
|
||||
conversion or timing analysis. Launch and manifest aggregate byte counts must
|
||||
agree before the browser admits camera payload downloads.
|
||||
stat validation and reuses an unchanged descriptor without reading/hashing media.
|
||||
A missing, corrupt or stale historical sidecar stays cold until that session is
|
||||
explicitly opened; startup, catalog reads and completion of another session do
|
||||
not rebuild it. An unparseable or ambiguous fragment fails readiness rather than
|
||||
exposing a fake seekable camera. Replay requests consume the prepared descriptor
|
||||
and never repeat media conversion or timing analysis. Launch and manifest
|
||||
aggregate byte counts must agree before the browser admits camera payload
|
||||
downloads.
|
||||
|
||||
The derived RRD writes an actual zero-time anchor at the internal
|
||||
`/__mission_core/session_origin` entity. Keeping it outside `/world` prevents a
|
||||
synthetic visualization layer while ensuring that the decoded RRD timeline,
|
||||
not merely its summary document, begins at session time zero. This changes the
|
||||
derived payload contract. Cache v7 keeps that anchor and additionally binds the
|
||||
derived recording to the owning plugin ID, primary artifact and ordered source
|
||||
artifact set; it rejects v6 and older sidecars and performs one background
|
||||
rebuild.
|
||||
derived payload contract. Cache v9 keeps that anchor, binds the owning plugin,
|
||||
ordered source artifacts and capture-envelope endpoints, and materializes the
|
||||
sealed completion rows. Older sidecars are incompatible and remain cold until
|
||||
that session is explicitly opened.
|
||||
|
||||
Replay v2 exposes two paths to the same pinned generation. `source_url` retains
|
||||
the strict `If-Match: "sha256:…"` contract for Mission Core clients;
|
||||
|
|
|
|||
|
|
@ -175,6 +175,42 @@ class RecordedMediaInspector:
|
|||
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
||||
return manifest
|
||||
|
||||
def restore_prepared(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
replay: ReplayCommand,
|
||||
) -> RecordedMediaManifest | None:
|
||||
"""Restore one previously prepared manifest without rebuilding it.
|
||||
|
||||
A catalog read or process restart must never turn an old camera archive
|
||||
into fresh preparation work. The durable sidecar is accepted only when
|
||||
its complete source stat identity still matches the sealed archive.
|
||||
Missing or stale sidecars remain cold until the operator explicitly
|
||||
requests that session.
|
||||
"""
|
||||
|
||||
if artifact.session_id != replay.session_id:
|
||||
raise SessionIntegrityError("recorded media does not belong to replay session")
|
||||
epoch_paths = _epoch_paths(artifact.source_path)
|
||||
key = (artifact.session_id, artifact.artifact_id)
|
||||
with self._lock:
|
||||
cached = self._cache.get(key)
|
||||
if cached is not None:
|
||||
identity = _prepared_source_identity(
|
||||
replay,
|
||||
epoch_paths,
|
||||
cached.manifest.epochs,
|
||||
)
|
||||
if cached.identity == identity:
|
||||
return cached.manifest
|
||||
prepared = self._load_prepared_sidecar(artifact, replay, epoch_paths)
|
||||
if prepared is None:
|
||||
return None
|
||||
manifest, identity = prepared
|
||||
with self._lock:
|
||||
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
||||
return manifest
|
||||
|
||||
def _load_prepared_sidecar(
|
||||
self,
|
||||
artifact: RecordedMediaArtifact,
|
||||
|
|
|
|||
|
|
@ -111,6 +111,11 @@ class SessionRecordingPreparationManager:
|
|||
tuple[RecordedMediaManifest, ...],
|
||||
]
|
||||
| None = None,
|
||||
ready_restorer: Callable[
|
||||
[ReplayCommand, MaterializedRecording],
|
||||
tuple[RecordedMediaManifest, ...] | None,
|
||||
]
|
||||
| None = None,
|
||||
) -> None:
|
||||
if queue_capacity < 1:
|
||||
raise ValueError("recording preparation queue capacity must be positive")
|
||||
|
|
@ -123,6 +128,7 @@ class SessionRecordingPreparationManager:
|
|||
# hung exporter must become observable to the browser stall detector.
|
||||
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
||||
self._ready_preparer = ready_preparer
|
||||
self._ready_restorer = ready_restorer
|
||||
self._guard = threading.RLock()
|
||||
self._current_by_session: dict[str, _PreparationJob] = {}
|
||||
self._closed = True
|
||||
|
|
@ -237,6 +243,74 @@ class SessionRecordingPreparationManager:
|
|||
self._current_by_session[command.session_id] = ready
|
||||
return self._snapshot_locked(ready)
|
||||
|
||||
def restore_published(
|
||||
self,
|
||||
command: ReplayCommand,
|
||||
) -> RecordingPreparationSnapshot | None:
|
||||
"""Restore a complete durable package without enqueueing preparation.
|
||||
|
||||
The RRD and every recorded-media descriptor must already have been
|
||||
published by an earlier successful job. Missing/stale pieces return a
|
||||
cold miss; this method never invokes an exporter or media parser.
|
||||
"""
|
||||
|
||||
source_command = _source_command(command)
|
||||
identity = _source_identity(source_command)
|
||||
with self._guard:
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if (
|
||||
current is not None
|
||||
and current.source_identity == identity
|
||||
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
||||
):
|
||||
return self._snapshot_locked(current)
|
||||
|
||||
recording = self.materializer.restore_published(source_command)
|
||||
if recording is None:
|
||||
with self._guard:
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if current is not None and current.state == "ready":
|
||||
self._current_by_session.pop(command.session_id, None)
|
||||
return None
|
||||
|
||||
if self._ready_preparer is None:
|
||||
recorded_media: tuple[RecordedMediaManifest, ...] = ()
|
||||
else:
|
||||
if self._ready_restorer is None:
|
||||
return None
|
||||
restored_media = self._ready_restorer(source_command, recording)
|
||||
if restored_media is None:
|
||||
return None
|
||||
recorded_media = restored_media
|
||||
validate_recorded_media_timeline(
|
||||
recorded_media,
|
||||
recording_start_seconds=recording.timeline_start_ns / 1_000_000_000,
|
||||
recording_end_seconds=recording.timeline_end_ns / 1_000_000_000,
|
||||
)
|
||||
|
||||
with self._guard:
|
||||
current = self._current_by_session.get(command.session_id)
|
||||
if (
|
||||
current is not None
|
||||
and current.source_identity == identity
|
||||
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
||||
):
|
||||
if current.state == "ready":
|
||||
current.recording = recording
|
||||
current.recorded_media = recorded_media
|
||||
return self._snapshot_locked(current)
|
||||
ready = _PreparationJob(
|
||||
preparation_id=uuid4().hex,
|
||||
source_identity=identity,
|
||||
command=source_command,
|
||||
state="ready",
|
||||
progress=1.0,
|
||||
recording=recording,
|
||||
recorded_media=recorded_media,
|
||||
)
|
||||
self._current_by_session[command.session_id] = ready
|
||||
return self._snapshot_locked(ready)
|
||||
|
||||
def resolve_cached_pinned(
|
||||
self,
|
||||
command: ReplayCommand,
|
||||
|
|
@ -325,10 +399,7 @@ class SessionRecordingPreparationManager:
|
|||
job is None
|
||||
or job.state != "ready"
|
||||
or job.recording is None
|
||||
or (
|
||||
preparation_id is not None
|
||||
and job.preparation_id != preparation_id
|
||||
)
|
||||
or (preparation_id is not None and job.preparation_id != preparation_id)
|
||||
):
|
||||
return None
|
||||
recording = job.recording
|
||||
|
|
@ -456,9 +527,7 @@ class SessionRecordingPreparationManager:
|
|||
recording_start_seconds=(
|
||||
recording.timeline_start_ns / 1_000_000_000
|
||||
),
|
||||
recording_end_seconds=(
|
||||
recording.timeline_end_ns / 1_000_000_000
|
||||
),
|
||||
recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000),
|
||||
)
|
||||
except Exception:
|
||||
with self._guard:
|
||||
|
|
|
|||
|
|
@ -232,6 +232,26 @@ class SessionRecordingMaterializer:
|
|||
with self._lock_for(session_id):
|
||||
return self._load_cached_recording(session_id, _validate_source(command))
|
||||
|
||||
def restore_published(self, command: ReplayCommand) -> MaterializedRecording | None:
|
||||
"""Restore an immutable published cache using bounded metadata checks.
|
||||
|
||||
Full source/output digests are proved before the cache is published.
|
||||
On a later process start the private cache is restored only when its
|
||||
schema, source stat identities and derived-file stat identity still
|
||||
match the durable sidecar. This keeps a prepared recording durable
|
||||
across restarts without rereading gigabytes or starting an exporter.
|
||||
"""
|
||||
|
||||
session_id = _validate_command_shape(command)
|
||||
if not self._has_compatible_cache_candidate(session_id):
|
||||
return None
|
||||
with self._lock_for(session_id):
|
||||
return self._load_cached_recording(
|
||||
session_id,
|
||||
_validate_source(command),
|
||||
verify_digests=False,
|
||||
)
|
||||
|
||||
def get_cached_pinned(
|
||||
self,
|
||||
command: ReplayCommand,
|
||||
|
|
@ -430,6 +450,7 @@ class SessionRecordingMaterializer:
|
|||
source: _ValidatedSource,
|
||||
*,
|
||||
pin: bool = False,
|
||||
verify_digests: bool = True,
|
||||
) -> MaterializedRecording | None:
|
||||
# Cache validation and the optional lease are one transaction with
|
||||
# eviction. This makes it safe for FileResponse to open the path after
|
||||
|
|
@ -451,6 +472,7 @@ class SessionRecordingMaterializer:
|
|||
source=source,
|
||||
recording_path=recording_path,
|
||||
sidecar_path=sidecar_path,
|
||||
verify_digests=verify_digests,
|
||||
)
|
||||
if cached is None:
|
||||
return None
|
||||
|
|
@ -661,6 +683,7 @@ class SessionRecordingMaterializer:
|
|||
source: _ValidatedSource,
|
||||
recording_path: Path,
|
||||
sidecar_path: Path,
|
||||
verify_digests: bool = True,
|
||||
) -> MaterializedRecording | None:
|
||||
if recording_path.is_symlink() or sidecar_path.is_symlink():
|
||||
return None
|
||||
|
|
@ -698,12 +721,19 @@ class SessionRecordingMaterializer:
|
|||
|
||||
source_sha256: str | None = None
|
||||
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
|
||||
digest = _sha256_prefix_stable(
|
||||
digest = (
|
||||
_sha256_prefix_stable(
|
||||
artifact.path,
|
||||
artifact.file_stat,
|
||||
artifact.replay_byte_length,
|
||||
)
|
||||
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256:
|
||||
if verify_digests
|
||||
else cached["sha256"]
|
||||
)
|
||||
if (
|
||||
artifact.expected_sha256 is not None
|
||||
and cached["sha256"] != artifact.expected_sha256
|
||||
):
|
||||
raise RecordingMaterializationError(
|
||||
"recording artifact digest no longer matches catalog"
|
||||
)
|
||||
|
|
@ -713,6 +743,8 @@ class SessionRecordingMaterializer:
|
|||
source_sha256 = digest
|
||||
if source_sha256 is None or source_sha256 != document["source_sha256"]:
|
||||
return None
|
||||
recording_sha256 = document["recording_sha256"]
|
||||
if verify_digests:
|
||||
recording_sha256 = _sha256_stable(recording_path, recording_stat)
|
||||
if recording_sha256 != document["recording_sha256"]:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
|
@ -65,9 +65,25 @@ def _prepare_recorded_media_for_launch(
|
|||
)
|
||||
|
||||
|
||||
def _restore_recorded_media_for_launch(
|
||||
command: ReplayCommand,
|
||||
_: MaterializedRecording,
|
||||
) -> tuple[RecordedMediaManifest, ...] | None:
|
||||
"""Restore only previously published media descriptors."""
|
||||
|
||||
restored: list[RecordedMediaManifest] = []
|
||||
for artifact in session_store.list_recorded_media(command.session_id):
|
||||
manifest = session_recorded_media_inspector.restore_prepared(artifact, command)
|
||||
if manifest is None:
|
||||
return None
|
||||
restored.append(manifest)
|
||||
return tuple(restored)
|
||||
|
||||
|
||||
session_recording_preparation_manager = SessionRecordingPreparationManager(
|
||||
session_recording_materializer,
|
||||
ready_preparer=_prepare_recorded_media_for_launch,
|
||||
ready_restorer=_restore_recorded_media_for_launch,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -82,22 +98,30 @@ def refresh_observation_catalog() -> tuple[str, ...]:
|
|||
return tuple(dict.fromkeys(imported))
|
||||
|
||||
|
||||
def enqueue_replayable_recordings() -> tuple[str, ...]:
|
||||
"""Reconcile finalized catalog sessions into the durable RRD work queue."""
|
||||
def finalized_replayable_recording_ids() -> tuple[str, ...]:
|
||||
"""List finalized replayable catalog identities without scheduling work."""
|
||||
|
||||
enqueued: list[str] = []
|
||||
finalized: list[str] = []
|
||||
cursor: str | None = None
|
||||
while True:
|
||||
page = session_store.list_recent(limit=100, cursor=cursor)
|
||||
for summary in page.items:
|
||||
if not summary.replayable or summary.status not in {
|
||||
"ready",
|
||||
"interrupted",
|
||||
"failed",
|
||||
}:
|
||||
continue
|
||||
finalized.extend(
|
||||
summary.session_id
|
||||
for summary in page.items
|
||||
if summary.replayable and summary.status in {"ready", "interrupted", "failed"}
|
||||
)
|
||||
cursor = page.next_cursor
|
||||
if cursor is None:
|
||||
return tuple(finalized)
|
||||
|
||||
|
||||
def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]:
|
||||
"""Schedule only explicitly selected newly finalized sessions."""
|
||||
|
||||
enqueued: list[str] = []
|
||||
for session_id in dict.fromkeys(session_ids):
|
||||
try:
|
||||
command = session_store.prepare_replay(summary.session_id)
|
||||
command = session_store.prepare_replay(session_id)
|
||||
session_recording_preparation_manager.enqueue(
|
||||
command,
|
||||
retry_interrupted=True,
|
||||
|
|
@ -105,24 +129,42 @@ def enqueue_replayable_recordings() -> tuple[str, ...]:
|
|||
except RecordingPreparationQueueFull:
|
||||
return tuple(enqueued)
|
||||
except Exception:
|
||||
# One stale/corrupt catalog row must not starve every valid
|
||||
# finalized session that follows it in the reconciliation
|
||||
# page. The row remains visible as failed/interrupted evidence
|
||||
# and can be repaired independently.
|
||||
# One stale/corrupt row must not starve a later newly completed
|
||||
# session. Historical cold caches remain operator-triggered.
|
||||
continue
|
||||
enqueued.append(summary.session_id)
|
||||
cursor = page.next_cursor
|
||||
if cursor is None:
|
||||
enqueued.append(session_id)
|
||||
return tuple(enqueued)
|
||||
|
||||
|
||||
async def _recording_preparation_reconciler() -> None:
|
||||
"""Discover newly completed/recovered captures without blocking requests."""
|
||||
def newly_finalized_recording_ids(
|
||||
known_finalized: set[str] | None,
|
||||
current_finalized: Iterable[str],
|
||||
) -> tuple[str, ...]:
|
||||
"""Return only post-startup completions; the first scan is a baseline."""
|
||||
|
||||
if known_finalized is None:
|
||||
return ()
|
||||
return tuple(sorted(set(current_finalized) - known_finalized))
|
||||
|
||||
|
||||
async def _recording_preparation_reconciler() -> None:
|
||||
"""Prepare sessions finalized during this process, never historical rows."""
|
||||
|
||||
known_finalized: set[str] | None = None
|
||||
while True:
|
||||
try:
|
||||
await asyncio.to_thread(refresh_observation_catalog)
|
||||
await asyncio.to_thread(enqueue_replayable_recordings)
|
||||
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
||||
newly_finalized = newly_finalized_recording_ids(
|
||||
known_finalized,
|
||||
finalized,
|
||||
)
|
||||
if newly_finalized:
|
||||
await asyncio.to_thread(
|
||||
enqueue_replayable_recordings,
|
||||
newly_finalized,
|
||||
)
|
||||
known_finalized = finalized
|
||||
except Exception:
|
||||
# A transient filesystem/catalog failure must not permanently
|
||||
# disable preparation of sessions completed later in the run.
|
||||
|
|
|
|||
|
|
@ -45,9 +45,7 @@ SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
|
|||
SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
MAX_SAFE_INTEGER = 9_007_199_254_740_991
|
||||
REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform"
|
||||
IMMUTABLE_RECORDING_CACHE_CONTROL = (
|
||||
"private, max-age=31536000, immutable, no-transform"
|
||||
)
|
||||
IMMUTABLE_RECORDING_CACHE_CONTROL = "private, max-age=31536000, immutable, no-transform"
|
||||
|
||||
|
||||
class _ReleasingFileResponse(FileResponse):
|
||||
|
|
@ -321,9 +319,13 @@ def build_session_router(
|
|||
|
||||
if recording_preparation_manager is not None:
|
||||
try:
|
||||
# Enqueue performs only catalog identity/lstat checks. A cold
|
||||
# RRD cache and every camera index are validated exclusively
|
||||
# by the process-owned worker.
|
||||
snapshot = recording_preparation_manager.status(command.session_id)
|
||||
if snapshot is None:
|
||||
snapshot = await run_in_threadpool(
|
||||
recording_preparation_manager.restore_published,
|
||||
command,
|
||||
)
|
||||
if snapshot is None:
|
||||
snapshot = recording_preparation_manager.enqueue(
|
||||
command,
|
||||
retry_failed=True,
|
||||
|
|
@ -504,6 +506,11 @@ def build_session_router(
|
|||
False,
|
||||
False,
|
||||
)
|
||||
snapshot = await run_in_threadpool(
|
||||
recording_preparation_manager.restore_published,
|
||||
command,
|
||||
)
|
||||
if snapshot is None:
|
||||
snapshot = recording_preparation_manager.enqueue(command)
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
|
@ -1036,15 +1043,16 @@ def _catalog_preparation_document(
|
|||
snapshot = manager.status(session_id)
|
||||
if snapshot is None:
|
||||
try:
|
||||
snapshot = manager.enqueue(store.prepare_replay(session_id))
|
||||
snapshot = manager.restore_published(store.prepare_replay(session_id))
|
||||
except (
|
||||
RecordingPreparationQueueFull,
|
||||
SessionNotFoundError,
|
||||
SessionNotReplayableError,
|
||||
SessionIntegrityError,
|
||||
ValueError,
|
||||
):
|
||||
return None
|
||||
if snapshot is None:
|
||||
return None
|
||||
document: dict[str, Any] = {
|
||||
"preparation_id": snapshot.preparation_id,
|
||||
"state": snapshot.state,
|
||||
|
|
@ -1165,10 +1173,7 @@ def _recorded_media_manifest_document(
|
|||
"segments": [
|
||||
{
|
||||
"sequence": segment.sequence,
|
||||
"url": (
|
||||
f"{base}/epochs/{epoch.ordinal}/segments/"
|
||||
f"{segment.sequence}.m4s"
|
||||
),
|
||||
"url": (f"{base}/epochs/{epoch.ordinal}/segments/{segment.sequence}.m4s"),
|
||||
"byte_length": segment.byte_length,
|
||||
"sha256": segment.sha256,
|
||||
}
|
||||
|
|
@ -1198,19 +1203,13 @@ def _resolve_recorded_media_manifest(
|
|||
detail="Некорректный идентификатор записанного медиаканала.",
|
||||
) from exc
|
||||
snapshot = manager.status(session_id)
|
||||
if (
|
||||
snapshot is None
|
||||
or snapshot.state != "ready"
|
||||
or snapshot.recorded_media is None
|
||||
):
|
||||
if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Медиаканал ещё не подготовлен.",
|
||||
)
|
||||
matches = tuple(
|
||||
manifest
|
||||
for manifest in snapshot.recorded_media
|
||||
if manifest.artifact_id == artifact_id
|
||||
manifest for manifest in snapshot.recorded_media if manifest.artifact_id == artifact_id
|
||||
)
|
||||
if len(matches) != 1:
|
||||
raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.")
|
||||
|
|
|
|||
|
|
@ -574,6 +574,39 @@ def test_cold_replay_returns_quick_202_then_status_returns_ready_launch(
|
|||
manager.close()
|
||||
|
||||
|
||||
def test_catalog_read_never_prepares_a_cold_historical_session(tmp_path: Path) -> None:
|
||||
repository = tmp_path / "repo"
|
||||
sessions = repository / "sessions"
|
||||
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
|
||||
store = SessionStore(repository, data_dir=tmp_path / "data")
|
||||
store.reconcile_archive(xgrids_k1_archive_source(sessions))
|
||||
calls = 0
|
||||
|
||||
def exporter(_source: Path, _destination: Path) -> dict[str, object]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise AssertionError("catalog reads must not invoke the exporter")
|
||||
|
||||
materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter)
|
||||
manager = SessionRecordingPreparationManager(materializer)
|
||||
router = build_session_router(
|
||||
store,
|
||||
recording_materializer=materializer,
|
||||
recording_preparation_manager=manager,
|
||||
)
|
||||
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
|
||||
try:
|
||||
item = list_route(limit=20, cursor=None)["items"][0]
|
||||
time.sleep(0.02)
|
||||
|
||||
assert item["id"] == session.name
|
||||
assert item["preparation"] is None
|
||||
assert manager.status(session.name) is None
|
||||
assert calls == 0
|
||||
finally:
|
||||
manager.close()
|
||||
|
||||
|
||||
def test_recording_response_releases_pin_once_when_asgi_send_fails(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ def test_catalog_refresh_failure_is_a_stable_service_error(tmp_path: Path) -> No
|
|||
assert "private local path" not in str(error.value.detail)
|
||||
|
||||
|
||||
def test_startup_warmup_enqueues_every_finalized_replayable_session(
|
||||
def test_startup_scan_baselines_historical_sessions_without_enqueuing(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
|
@ -188,17 +188,14 @@ def test_startup_warmup_enqueues_every_finalized_replayable_session(
|
|||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
enqueued = app_module.enqueue_replayable_recordings()
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(session.name)
|
||||
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
snapshot = manager.status(session.name)
|
||||
historical = set(app_module.finalized_replayable_recording_ids())
|
||||
newly_finalized = app_module.newly_finalized_recording_ids(None, historical)
|
||||
enqueued = app_module.enqueue_replayable_recordings(newly_finalized)
|
||||
|
||||
assert enqueued == (session.name,)
|
||||
assert snapshot is not None
|
||||
assert snapshot.state == "ready"
|
||||
assert snapshot.recording is not None
|
||||
assert historical == {session.name}
|
||||
assert newly_finalized == ()
|
||||
assert enqueued == ()
|
||||
assert manager.status(session.name) is None
|
||||
finally:
|
||||
manager.close()
|
||||
|
||||
|
|
@ -233,7 +230,7 @@ def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session
|
|||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
enqueued = app_module.enqueue_replayable_recordings()
|
||||
enqueued = app_module.enqueue_replayable_recordings((stale.name, good.name))
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(good.name)
|
||||
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
|
||||
|
|
@ -283,7 +280,7 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
|
|||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
assert started.wait(timeout=1)
|
||||
manager.close(timeout=0.001)
|
||||
manager.start()
|
||||
|
|
@ -291,15 +288,13 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
|
|||
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(session.name)
|
||||
while (
|
||||
snapshot is None or snapshot.state != "cancelled"
|
||||
) and time.monotonic() < deadline:
|
||||
while (snapshot is None or snapshot.state != "cancelled") and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
snapshot = manager.status(session.name)
|
||||
assert snapshot is not None and snapshot.state == "cancelled"
|
||||
interrupted_id = snapshot.preparation_id
|
||||
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
deadline = time.monotonic() + 2
|
||||
while time.monotonic() < deadline:
|
||||
snapshot = manager.status(session.name)
|
||||
|
|
@ -336,18 +331,16 @@ def test_reconciler_does_not_loop_retry_a_genuine_failed_job(
|
|||
monkeypatch.setattr(app_module, "session_store", store)
|
||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||
try:
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
deadline = time.monotonic() + 2
|
||||
snapshot = manager.status(session.name)
|
||||
while (
|
||||
snapshot is None or snapshot.state != "failed"
|
||||
) and time.monotonic() < deadline:
|
||||
while (snapshot is None or snapshot.state != "failed") and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
snapshot = manager.status(session.name)
|
||||
assert snapshot is not None and snapshot.state == "failed"
|
||||
failed_id = snapshot.preparation_id
|
||||
|
||||
app_module.enqueue_replayable_recordings()
|
||||
app_module.enqueue_replayable_recordings((session.name,))
|
||||
time.sleep(0.05)
|
||||
unchanged = manager.status(session.name)
|
||||
assert unchanged is not None
|
||||
|
|
|
|||
|
|
@ -231,6 +231,46 @@ def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Pat
|
|||
manager.close()
|
||||
|
||||
|
||||
def test_published_package_restores_after_process_restart_without_export(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
command = _command(tmp_path / "session")
|
||||
private_root = tmp_path / "private"
|
||||
calls = 0
|
||||
|
||||
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return _summary(source, destination, b"durable-recording")
|
||||
|
||||
first = SessionRecordingPreparationManager(
|
||||
SessionRecordingMaterializer(private_root, exporter=exporter)
|
||||
)
|
||||
try:
|
||||
first.enqueue(command)
|
||||
ready = _wait_for_state(first, command.session_id, {"ready"})
|
||||
assert ready.recording is not None
|
||||
finally:
|
||||
first.close()
|
||||
|
||||
def forbidden_exporter(_source: Path, _destination: Path) -> dict[str, object]:
|
||||
raise AssertionError("a published package must not be exported again")
|
||||
|
||||
restarted = SessionRecordingPreparationManager(
|
||||
SessionRecordingMaterializer(private_root, exporter=forbidden_exporter)
|
||||
)
|
||||
try:
|
||||
restored = restarted.restore_published(command)
|
||||
|
||||
assert restored is not None
|
||||
assert restored.state == "ready"
|
||||
assert restored.recording is not None
|
||||
assert restored.recording.path.read_bytes() == b"durable-recording"
|
||||
assert calls == 1
|
||||
finally:
|
||||
restarted.close()
|
||||
|
||||
|
||||
def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Reference in New Issue