fix(archive): preserve prepared sessions across restarts
This commit is contained in:
parent
aa3680948f
commit
574a494759
|
|
@ -11,7 +11,7 @@ import type {
|
||||||
} from "../core/observation/useObservationSessions";
|
} from "../core/observation/useObservationSessions";
|
||||||
import "../styles/observation-sessions.css";
|
import "../styles/observation-sessions.css";
|
||||||
|
|
||||||
type SessionVisualState = "ready" | "processing" | "error";
|
type SessionVisualState = "ready" | "processing" | "cold" | "error";
|
||||||
|
|
||||||
export function observationSessionVisualState(
|
export function observationSessionVisualState(
|
||||||
session: ObservationSessionSummary,
|
session: ObservationSessionSummary,
|
||||||
|
|
@ -26,12 +26,14 @@ export function observationSessionVisualState(
|
||||||
if (session.preparation.state === "ready" && session.replayable) return "ready";
|
if (session.preparation.state === "ready" && session.replayable) return "ready";
|
||||||
return "error";
|
return "error";
|
||||||
}
|
}
|
||||||
return session.status === "recording" ? "processing" : "error";
|
if (session.status === "recording") return "processing";
|
||||||
|
return session.replayable ? "cold" : "error";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function observationSessionVisualLabel(state: SessionVisualState): string {
|
export function observationSessionVisualLabel(state: SessionVisualState): string {
|
||||||
if (state === "ready") return "Готово";
|
if (state === "ready") return "Готово";
|
||||||
if (state === "processing") return "Обработка";
|
if (state === "processing") return "Обработка";
|
||||||
|
if (state === "cold") return "Подготовить";
|
||||||
return "Ошибка";
|
return "Ошибка";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -137,8 +137,12 @@
|
||||||
animation: observation-session-processing 1.1s ease-in-out infinite;
|
animation: observation-session-processing 1.1s ease-in-out infinite;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.observation-session-option i[data-session-visual-state="cold"],
|
||||||
.observation-session-option i[data-session-visual-state="error"] {
|
.observation-session-option i[data-session-visual-state="error"] {
|
||||||
background: var(--nodedc-text-muted);
|
background: var(--nodedc-text-muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
.observation-session-option i[data-session-visual-state="error"] {
|
||||||
opacity: 0.48;
|
opacity: 0.48;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -160,7 +160,7 @@ test("session catalog exposes authoritative background preparation state", () =>
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("saved-session rows use only authoritative ready, processing and error states", () => {
|
test("saved-session rows distinguish durable, active, cold and failed states", () => {
|
||||||
const makeDecoded = (state, overrides = {}) => decodeObservationSessionCatalog({
|
const makeDecoded = (state, overrides = {}) => decodeObservationSessionCatalog({
|
||||||
items: [session({
|
items: [session({
|
||||||
status: "interrupted",
|
status: "interrupted",
|
||||||
|
|
@ -187,7 +187,7 @@ test("saved-session rows use only authoritative ready, processing and error stat
|
||||||
observationSessionVisualState(makeDecoded("failed", { status: "recording" })),
|
observationSessionVisualState(makeDecoded("failed", { status: "recording" })),
|
||||||
"error",
|
"error",
|
||||||
);
|
);
|
||||||
assert.equal(observationSessionVisualState(makeDecoded(null)), "error");
|
assert.equal(observationSessionVisualState(makeDecoded(null)), "cold");
|
||||||
assert.equal(
|
assert.equal(
|
||||||
observationSessionVisualState(makeDecoded(null, { status: "recording" })),
|
observationSessionVisualState(makeDecoded(null, { status: "recording" })),
|
||||||
"processing",
|
"processing",
|
||||||
|
|
@ -206,6 +206,7 @@ test("saved-session rows use only authoritative ready, processing and error stat
|
||||||
);
|
);
|
||||||
assert.equal(observationSessionVisualLabel("ready"), "Готово");
|
assert.equal(observationSessionVisualLabel("ready"), "Готово");
|
||||||
assert.equal(observationSessionVisualLabel("processing"), "Обработка");
|
assert.equal(observationSessionVisualLabel("processing"), "Обработка");
|
||||||
|
assert.equal(observationSessionVisualLabel("cold"), "Подготовить");
|
||||||
assert.equal(observationSessionVisualLabel("error"), "Ошибка");
|
assert.equal(observationSessionVisualLabel("error"), "Ошибка");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -57,22 +57,26 @@ not mutate playback state, so changing accumulation, size, color or visibility
|
||||||
does not pause the active replay or replace its camera view.
|
does not pause the active replay or replace its camera view.
|
||||||
|
|
||||||
Completed and recovered sessions are discovered on startup and by the catalog
|
Completed and recovered sessions are discovered on startup and by the catalog
|
||||||
reconciler, then deduplicated into one bounded, single-worker preparation queue.
|
reconciler. The first scan is only a historical baseline: it never enqueues the
|
||||||
The worker validates the native source, exports point/pose data, verifies source
|
existing archive. A session that becomes finalized after that baseline is
|
||||||
stability and both SHA-256 digests, finalizes once, and atomically publishes the
|
deduplicated into the bounded, single-worker preparation queue. The worker
|
||||||
|
validates the native source, exports point/pose data, verifies source stability
|
||||||
|
and both SHA-256 digests, finalizes once, and atomically publishes the
|
||||||
digest-bound derived RRD plus its cache sidecar. Before the same job becomes
|
digest-bound derived RRD plus its cache sidecar. Before the same job becomes
|
||||||
`ready`, it also parses and validates every archived camera index and retains
|
`ready`, it also parses and validates every archived camera index and publishes
|
||||||
the immutable manifest generation in memory. The catalog exposes preparation
|
the immutable media-manifest sidecar. The catalog exposes preparation state and
|
||||||
state and progress while this runs; a recording URL and camera descriptors are
|
progress while this runs; a recording URL and camera descriptors are returned
|
||||||
returned only after the complete launch generation has passed validation.
|
only after the complete launch generation has passed validation.
|
||||||
|
|
||||||
Preparation is a backend lifecycle, not a viewer action: sealing or recovering
|
Preparation is a backend lifecycle, not a viewer computation. Reopening,
|
||||||
a native session makes it eligible for the reconciler, which builds the derived
|
seeking, refreshing the catalog or changing the workspace never reruns a
|
||||||
RRD once in the background. Reopening, seeking or changing the workspace reads
|
published conversion. After a process restart, a catalog/replay read restores a
|
||||||
that prepared artifact and never reruns conversion. A cold process may validate
|
complete compatible RRD and camera package from its durable sidecars using
|
||||||
an existing cache and rebuild its in-memory camera manifest in the worker, but
|
bounded schema and stat-identity checks; it does not hash native evidence, parse
|
||||||
no replay, status, RRD or manifest request hashes native evidence or parses a
|
camera indexes or invoke the exporter. A historical session whose package is
|
||||||
camera index synchronously.
|
missing, stale or from an incompatible cache schema remains cold. Only an
|
||||||
|
explicit replay/RRD request schedules that one session for preparation; listing
|
||||||
|
the catalog and finalizing another session do not schedule it.
|
||||||
|
|
||||||
The session menu uses three operator indicators:
|
The session menu uses three operator indicators:
|
||||||
|
|
||||||
|
|
@ -80,6 +84,7 @@ The session menu uses three operator indicators:
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Solid green + `Готово` | `ready` | Verified RRD is available. |
|
| Solid green + `Готово` | `ready` | Verified RRD is available. |
|
||||||
| Pulsing green + `Обработка` | `queued`, `validating`, `exporting`, `finalizing` | Background preparation is active. |
|
| Pulsing green + `Обработка` | `queued`, `validating`, `exporting`, `finalizing` | Background preparation is active. |
|
||||||
|
| Solid gray + `Подготовить` | replayable session with no compatible published package | No work is running; choosing the row schedules this session only. |
|
||||||
| Dim gray + `Ошибка` | failed, cancelled, non-replayable or invalid session | No launchable recording; inspect the message or retry. |
|
| Dim gray + `Ошибка` | failed, cancelled, non-replayable or invalid session | No launchable recording; inspect the message or retry. |
|
||||||
|
|
||||||
Saved-session status never uses yellow or red. Switching sessions aborts only
|
Saved-session status never uses yellow or red. Switching sessions aborts only
|
||||||
|
|
@ -93,7 +98,8 @@ Later openings reuse a verified, digest-bound cache. Derived RRD cache v9 is
|
||||||
intentionally incompatible with v8 and older generations. In addition to plugin
|
intentionally incompatible with v8 and older generations. In addition to plugin
|
||||||
and ordered-artifact identity, it binds the durable clock origin and active
|
and ordered-artifact identity, it binds the durable clock origin and active
|
||||||
envelope and materializes real `session_time = 0` origin and sealed completion
|
envelope and materializes real `session_time = 0` origin and sealed completion
|
||||||
rows. Older generations are rebuilt once in the background.
|
rows. Older generations remain cold after startup and are rebuilt only when an
|
||||||
|
operator explicitly opens that session.
|
||||||
The browser may use the strict `source_url` with `If-Match`, while the embedded
|
The browser may use the strict `source_url` with `If-Match`, while the embedded
|
||||||
Rerun loader uses the canonical `viewer_source_url` whose lowercase SHA-256
|
Rerun loader uses the canonical `viewer_source_url` whose lowercase SHA-256
|
||||||
`generation` query is bound to the same launch descriptor. A missing or stale
|
`generation` query is bound to the same launch descriptor. A missing or stale
|
||||||
|
|
@ -194,11 +200,13 @@ Native `.k1mqtt` bytes with aligned metadata, the capture-clock artifacts and
|
||||||
canonical camera fMP4 archives are the evidence source of truth. The derived RRD
|
canonical camera fMP4 archives are the evidence source of truth. The derived RRD
|
||||||
contains every decodable point and pose frame plus every valid `ModelingReport`
|
contains every decodable point and pose frame plus every valid `ModelingReport`
|
||||||
distance/speed/scan-time sample from that source, but remains a rebuildable view
|
distance/speed/scan-time sample from that source, but remains a rebuildable view
|
||||||
rather than an evidence master. A cache entry is rebuilt only when native source
|
rather than an evidence master. A cache entry is eligible for rebuild only when
|
||||||
identity/digests change or an incompatible derived-data export revision is
|
native source identity/digests change or an incompatible derived-data export
|
||||||
introduced. A UI blueprint or workspace-layout revision never invalidates or
|
revision is introduced. Historical entries are not rebuilt during startup or
|
||||||
rewrites the data RRD. Cache v8 and older payloads are not reusable as v9 because
|
because another session is finalized; an explicit open schedules the affected
|
||||||
they do not bind both real capture-envelope endpoints.
|
entry. A UI blueprint or workspace-layout revision never invalidates or rewrites
|
||||||
|
the data RRD. Cache v8 and older payloads are not reusable as v9 because they do
|
||||||
|
not bind both real capture-envelope endpoints.
|
||||||
|
|
||||||
`ModelingReport` time is the device's `ScanTime` counter at two ticks per second;
|
`ModelingReport` time is the device's `ScanTime` counter at two ticks per second;
|
||||||
distance and speed are the reported `MoveDistance`/`MoveSpeed` values. Live state
|
distance and speed are the reported `MoveDistance`/`MoveSpeed` values. Live state
|
||||||
|
|
@ -318,9 +326,10 @@ written under the private derived cache with a schema, generation and checksum.
|
||||||
Publication uses a private temporary file, file and directory `fsync`, and atomic
|
Publication uses a private temporary file, file and directory `fsync`, and atomic
|
||||||
rename; startup scavenges crash-left temporary files. A restart reuses this
|
rename; startup scavenges crash-left temporary files. A restart reuses this
|
||||||
sidecar after confined O(n) stat validation, without rereading, hashing or parsing
|
sidecar after confined O(n) stat validation, without rereading, hashing or parsing
|
||||||
media. A missing, corrupt or stale sidecar is rebuilt only by the background
|
media. A missing, corrupt or stale sidecar remains cold until an explicit open
|
||||||
worker. No replay, status, manifest or payload request performs conversion or
|
schedules the complete session package in the background worker. No catalog,
|
||||||
recalculates these intervals.
|
status, manifest or payload request itself performs conversion or recalculates
|
||||||
|
these intervals.
|
||||||
|
|
||||||
Every derived RRD contains a real `session_time = 0` row at the internal
|
Every derived RRD contains a real `session_time = 0` row at the internal
|
||||||
`/__mission_core/session_origin` entity. The anchor is deliberately outside
|
`/__mission_core/session_origin` entity. The anchor is deliberately outside
|
||||||
|
|
|
||||||
|
|
@ -73,11 +73,15 @@ committed metadata; replay stops at the last validated aligned boundary.
|
||||||
|
|
||||||
### Recorded spatial playback
|
### Recorded spatial playback
|
||||||
|
|
||||||
When a native capture is normally sealed or recovery-sealed, a bounded
|
When a native capture is normally sealed or recovery-sealed during the current
|
||||||
single-worker reconciler automatically prepares its private derived `.rrd`.
|
process lifetime, a bounded single-worker reconciler automatically prepares its
|
||||||
Preparation is a backend lifecycle and is never executed by an HTTP replay
|
private derived `.rrd`. The reconciler's first scan establishes a historical
|
||||||
request. Reopening, seeking, switching tabs or changing display settings reads
|
baseline and does not enqueue old sessions. Preparation is a backend lifecycle
|
||||||
the already prepared artifact and does not rerun conversion. Export is lossless
|
and is never executed inside an HTTP replay request: an explicit request may
|
||||||
|
enqueue one cold historical package and returns `202` while the worker owns the
|
||||||
|
conversion. Reopening, seeking, switching tabs, listing the catalog or changing
|
||||||
|
display settings reads the already published artifact and does not rerun
|
||||||
|
conversion. Export is lossless
|
||||||
with respect to every decodable point and pose message in the native capture;
|
with respect to every decodable point and pose message in the native capture;
|
||||||
it does not pass through the bounded live-preview queue. The RRD uses the
|
it does not pass through the bounded live-preview queue. The RRD uses the
|
||||||
recording-local `session_time` duration timeline, while the receive wall clock
|
recording-local `session_time` duration timeline, while the receive wall clock
|
||||||
|
|
@ -123,21 +127,23 @@ future replay v3 will need distinct spatial and session-union ranges.
|
||||||
The path-free descriptor is atomically persisted in the private derived cache
|
The path-free descriptor is atomically persisted in the private derived cache
|
||||||
with its checksum and a stat identity covering the native timing origin and
|
with its checksum and a stat identity covering the native timing origin and
|
||||||
every camera summary, index, init and segment file. Restart performs confined
|
every camera summary, index, init and segment file. Restart performs confined
|
||||||
stat validation and reuses an unchanged descriptor without reading/hashing media;
|
stat validation and reuses an unchanged descriptor without reading/hashing media.
|
||||||
a missing, corrupt or stale sidecar is rebuilt in the background. An unparseable
|
A missing, corrupt or stale historical sidecar stays cold until that session is
|
||||||
or ambiguous fragment fails readiness rather than exposing a fake seekable
|
explicitly opened; startup, catalog reads and completion of another session do
|
||||||
camera. Replay requests consume the prepared descriptor and never repeat media
|
not rebuild it. An unparseable or ambiguous fragment fails readiness rather than
|
||||||
conversion or timing analysis. Launch and manifest aggregate byte counts must
|
exposing a fake seekable camera. Replay requests consume the prepared descriptor
|
||||||
agree before the browser admits camera payload downloads.
|
and never repeat media conversion or timing analysis. Launch and manifest
|
||||||
|
aggregate byte counts must agree before the browser admits camera payload
|
||||||
|
downloads.
|
||||||
|
|
||||||
The derived RRD writes an actual zero-time anchor at the internal
|
The derived RRD writes an actual zero-time anchor at the internal
|
||||||
`/__mission_core/session_origin` entity. Keeping it outside `/world` prevents a
|
`/__mission_core/session_origin` entity. Keeping it outside `/world` prevents a
|
||||||
synthetic visualization layer while ensuring that the decoded RRD timeline,
|
synthetic visualization layer while ensuring that the decoded RRD timeline,
|
||||||
not merely its summary document, begins at session time zero. This changes the
|
not merely its summary document, begins at session time zero. This changes the
|
||||||
derived payload contract. Cache v7 keeps that anchor and additionally binds the
|
derived payload contract. Cache v9 keeps that anchor, binds the owning plugin,
|
||||||
derived recording to the owning plugin ID, primary artifact and ordered source
|
ordered source artifacts and capture-envelope endpoints, and materializes the
|
||||||
artifact set; it rejects v6 and older sidecars and performs one background
|
sealed completion rows. Older sidecars are incompatible and remain cold until
|
||||||
rebuild.
|
that session is explicitly opened.
|
||||||
|
|
||||||
Replay v2 exposes two paths to the same pinned generation. `source_url` retains
|
Replay v2 exposes two paths to the same pinned generation. `source_url` retains
|
||||||
the strict `If-Match: "sha256:…"` contract for Mission Core clients;
|
the strict `If-Match: "sha256:…"` contract for Mission Core clients;
|
||||||
|
|
|
||||||
|
|
@ -175,6 +175,42 @@ class RecordedMediaInspector:
|
||||||
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
||||||
return manifest
|
return manifest
|
||||||
|
|
||||||
|
def restore_prepared(
|
||||||
|
self,
|
||||||
|
artifact: RecordedMediaArtifact,
|
||||||
|
replay: ReplayCommand,
|
||||||
|
) -> RecordedMediaManifest | None:
|
||||||
|
"""Restore one previously prepared manifest without rebuilding it.
|
||||||
|
|
||||||
|
A catalog read or process restart must never turn an old camera archive
|
||||||
|
into fresh preparation work. The durable sidecar is accepted only when
|
||||||
|
its complete source stat identity still matches the sealed archive.
|
||||||
|
Missing or stale sidecars remain cold until the operator explicitly
|
||||||
|
requests that session.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if artifact.session_id != replay.session_id:
|
||||||
|
raise SessionIntegrityError("recorded media does not belong to replay session")
|
||||||
|
epoch_paths = _epoch_paths(artifact.source_path)
|
||||||
|
key = (artifact.session_id, artifact.artifact_id)
|
||||||
|
with self._lock:
|
||||||
|
cached = self._cache.get(key)
|
||||||
|
if cached is not None:
|
||||||
|
identity = _prepared_source_identity(
|
||||||
|
replay,
|
||||||
|
epoch_paths,
|
||||||
|
cached.manifest.epochs,
|
||||||
|
)
|
||||||
|
if cached.identity == identity:
|
||||||
|
return cached.manifest
|
||||||
|
prepared = self._load_prepared_sidecar(artifact, replay, epoch_paths)
|
||||||
|
if prepared is None:
|
||||||
|
return None
|
||||||
|
manifest, identity = prepared
|
||||||
|
with self._lock:
|
||||||
|
self._cache[key] = _CachedManifest(identity=identity, manifest=manifest)
|
||||||
|
return manifest
|
||||||
|
|
||||||
def _load_prepared_sidecar(
|
def _load_prepared_sidecar(
|
||||||
self,
|
self,
|
||||||
artifact: RecordedMediaArtifact,
|
artifact: RecordedMediaArtifact,
|
||||||
|
|
|
||||||
|
|
@ -111,6 +111,11 @@ class SessionRecordingPreparationManager:
|
||||||
tuple[RecordedMediaManifest, ...],
|
tuple[RecordedMediaManifest, ...],
|
||||||
]
|
]
|
||||||
| None = None,
|
| None = None,
|
||||||
|
ready_restorer: Callable[
|
||||||
|
[ReplayCommand, MaterializedRecording],
|
||||||
|
tuple[RecordedMediaManifest, ...] | None,
|
||||||
|
]
|
||||||
|
| None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
if queue_capacity < 1:
|
if queue_capacity < 1:
|
||||||
raise ValueError("recording preparation queue capacity must be positive")
|
raise ValueError("recording preparation queue capacity must be positive")
|
||||||
|
|
@ -123,6 +128,7 @@ class SessionRecordingPreparationManager:
|
||||||
# hung exporter must become observable to the browser stall detector.
|
# hung exporter must become observable to the browser stall detector.
|
||||||
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
||||||
self._ready_preparer = ready_preparer
|
self._ready_preparer = ready_preparer
|
||||||
|
self._ready_restorer = ready_restorer
|
||||||
self._guard = threading.RLock()
|
self._guard = threading.RLock()
|
||||||
self._current_by_session: dict[str, _PreparationJob] = {}
|
self._current_by_session: dict[str, _PreparationJob] = {}
|
||||||
self._closed = True
|
self._closed = True
|
||||||
|
|
@ -237,6 +243,74 @@ class SessionRecordingPreparationManager:
|
||||||
self._current_by_session[command.session_id] = ready
|
self._current_by_session[command.session_id] = ready
|
||||||
return self._snapshot_locked(ready)
|
return self._snapshot_locked(ready)
|
||||||
|
|
||||||
|
def restore_published(
|
||||||
|
self,
|
||||||
|
command: ReplayCommand,
|
||||||
|
) -> RecordingPreparationSnapshot | None:
|
||||||
|
"""Restore a complete durable package without enqueueing preparation.
|
||||||
|
|
||||||
|
The RRD and every recorded-media descriptor must already have been
|
||||||
|
published by an earlier successful job. Missing/stale pieces return a
|
||||||
|
cold miss; this method never invokes an exporter or media parser.
|
||||||
|
"""
|
||||||
|
|
||||||
|
source_command = _source_command(command)
|
||||||
|
identity = _source_identity(source_command)
|
||||||
|
with self._guard:
|
||||||
|
current = self._current_by_session.get(command.session_id)
|
||||||
|
if (
|
||||||
|
current is not None
|
||||||
|
and current.source_identity == identity
|
||||||
|
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
||||||
|
):
|
||||||
|
return self._snapshot_locked(current)
|
||||||
|
|
||||||
|
recording = self.materializer.restore_published(source_command)
|
||||||
|
if recording is None:
|
||||||
|
with self._guard:
|
||||||
|
current = self._current_by_session.get(command.session_id)
|
||||||
|
if current is not None and current.state == "ready":
|
||||||
|
self._current_by_session.pop(command.session_id, None)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if self._ready_preparer is None:
|
||||||
|
recorded_media: tuple[RecordedMediaManifest, ...] = ()
|
||||||
|
else:
|
||||||
|
if self._ready_restorer is None:
|
||||||
|
return None
|
||||||
|
restored_media = self._ready_restorer(source_command, recording)
|
||||||
|
if restored_media is None:
|
||||||
|
return None
|
||||||
|
recorded_media = restored_media
|
||||||
|
validate_recorded_media_timeline(
|
||||||
|
recorded_media,
|
||||||
|
recording_start_seconds=recording.timeline_start_ns / 1_000_000_000,
|
||||||
|
recording_end_seconds=recording.timeline_end_ns / 1_000_000_000,
|
||||||
|
)
|
||||||
|
|
||||||
|
with self._guard:
|
||||||
|
current = self._current_by_session.get(command.session_id)
|
||||||
|
if (
|
||||||
|
current is not None
|
||||||
|
and current.source_identity == identity
|
||||||
|
and current.state in ACTIVE_PREPARATION_STATES | {"ready"}
|
||||||
|
):
|
||||||
|
if current.state == "ready":
|
||||||
|
current.recording = recording
|
||||||
|
current.recorded_media = recorded_media
|
||||||
|
return self._snapshot_locked(current)
|
||||||
|
ready = _PreparationJob(
|
||||||
|
preparation_id=uuid4().hex,
|
||||||
|
source_identity=identity,
|
||||||
|
command=source_command,
|
||||||
|
state="ready",
|
||||||
|
progress=1.0,
|
||||||
|
recording=recording,
|
||||||
|
recorded_media=recorded_media,
|
||||||
|
)
|
||||||
|
self._current_by_session[command.session_id] = ready
|
||||||
|
return self._snapshot_locked(ready)
|
||||||
|
|
||||||
def resolve_cached_pinned(
|
def resolve_cached_pinned(
|
||||||
self,
|
self,
|
||||||
command: ReplayCommand,
|
command: ReplayCommand,
|
||||||
|
|
@ -325,10 +399,7 @@ class SessionRecordingPreparationManager:
|
||||||
job is None
|
job is None
|
||||||
or job.state != "ready"
|
or job.state != "ready"
|
||||||
or job.recording is None
|
or job.recording is None
|
||||||
or (
|
or (preparation_id is not None and job.preparation_id != preparation_id)
|
||||||
preparation_id is not None
|
|
||||||
and job.preparation_id != preparation_id
|
|
||||||
)
|
|
||||||
):
|
):
|
||||||
return None
|
return None
|
||||||
recording = job.recording
|
recording = job.recording
|
||||||
|
|
@ -456,9 +527,7 @@ class SessionRecordingPreparationManager:
|
||||||
recording_start_seconds=(
|
recording_start_seconds=(
|
||||||
recording.timeline_start_ns / 1_000_000_000
|
recording.timeline_start_ns / 1_000_000_000
|
||||||
),
|
),
|
||||||
recording_end_seconds=(
|
recording_end_seconds=(recording.timeline_end_ns / 1_000_000_000),
|
||||||
recording.timeline_end_ns / 1_000_000_000
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
with self._guard:
|
with self._guard:
|
||||||
|
|
|
||||||
|
|
@ -232,6 +232,26 @@ class SessionRecordingMaterializer:
|
||||||
with self._lock_for(session_id):
|
with self._lock_for(session_id):
|
||||||
return self._load_cached_recording(session_id, _validate_source(command))
|
return self._load_cached_recording(session_id, _validate_source(command))
|
||||||
|
|
||||||
|
def restore_published(self, command: ReplayCommand) -> MaterializedRecording | None:
|
||||||
|
"""Restore an immutable published cache using bounded metadata checks.
|
||||||
|
|
||||||
|
Full source/output digests are proved before the cache is published.
|
||||||
|
On a later process start the private cache is restored only when its
|
||||||
|
schema, source stat identities and derived-file stat identity still
|
||||||
|
match the durable sidecar. This keeps a prepared recording durable
|
||||||
|
across restarts without rereading gigabytes or starting an exporter.
|
||||||
|
"""
|
||||||
|
|
||||||
|
session_id = _validate_command_shape(command)
|
||||||
|
if not self._has_compatible_cache_candidate(session_id):
|
||||||
|
return None
|
||||||
|
with self._lock_for(session_id):
|
||||||
|
return self._load_cached_recording(
|
||||||
|
session_id,
|
||||||
|
_validate_source(command),
|
||||||
|
verify_digests=False,
|
||||||
|
)
|
||||||
|
|
||||||
def get_cached_pinned(
|
def get_cached_pinned(
|
||||||
self,
|
self,
|
||||||
command: ReplayCommand,
|
command: ReplayCommand,
|
||||||
|
|
@ -430,6 +450,7 @@ class SessionRecordingMaterializer:
|
||||||
source: _ValidatedSource,
|
source: _ValidatedSource,
|
||||||
*,
|
*,
|
||||||
pin: bool = False,
|
pin: bool = False,
|
||||||
|
verify_digests: bool = True,
|
||||||
) -> MaterializedRecording | None:
|
) -> MaterializedRecording | None:
|
||||||
# Cache validation and the optional lease are one transaction with
|
# Cache validation and the optional lease are one transaction with
|
||||||
# eviction. This makes it safe for FileResponse to open the path after
|
# eviction. This makes it safe for FileResponse to open the path after
|
||||||
|
|
@ -451,6 +472,7 @@ class SessionRecordingMaterializer:
|
||||||
source=source,
|
source=source,
|
||||||
recording_path=recording_path,
|
recording_path=recording_path,
|
||||||
sidecar_path=sidecar_path,
|
sidecar_path=sidecar_path,
|
||||||
|
verify_digests=verify_digests,
|
||||||
)
|
)
|
||||||
if cached is None:
|
if cached is None:
|
||||||
return None
|
return None
|
||||||
|
|
@ -661,6 +683,7 @@ class SessionRecordingMaterializer:
|
||||||
source: _ValidatedSource,
|
source: _ValidatedSource,
|
||||||
recording_path: Path,
|
recording_path: Path,
|
||||||
sidecar_path: Path,
|
sidecar_path: Path,
|
||||||
|
verify_digests: bool = True,
|
||||||
) -> MaterializedRecording | None:
|
) -> MaterializedRecording | None:
|
||||||
if recording_path.is_symlink() or sidecar_path.is_symlink():
|
if recording_path.is_symlink() or sidecar_path.is_symlink():
|
||||||
return None
|
return None
|
||||||
|
|
@ -698,12 +721,19 @@ class SessionRecordingMaterializer:
|
||||||
|
|
||||||
source_sha256: str | None = None
|
source_sha256: str | None = None
|
||||||
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
|
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
|
||||||
digest = _sha256_prefix_stable(
|
digest = (
|
||||||
artifact.path,
|
_sha256_prefix_stable(
|
||||||
artifact.file_stat,
|
artifact.path,
|
||||||
artifact.replay_byte_length,
|
artifact.file_stat,
|
||||||
|
artifact.replay_byte_length,
|
||||||
|
)
|
||||||
|
if verify_digests
|
||||||
|
else cached["sha256"]
|
||||||
)
|
)
|
||||||
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256:
|
if (
|
||||||
|
artifact.expected_sha256 is not None
|
||||||
|
and cached["sha256"] != artifact.expected_sha256
|
||||||
|
):
|
||||||
raise RecordingMaterializationError(
|
raise RecordingMaterializationError(
|
||||||
"recording artifact digest no longer matches catalog"
|
"recording artifact digest no longer matches catalog"
|
||||||
)
|
)
|
||||||
|
|
@ -713,9 +743,11 @@ class SessionRecordingMaterializer:
|
||||||
source_sha256 = digest
|
source_sha256 = digest
|
||||||
if source_sha256 is None or source_sha256 != document["source_sha256"]:
|
if source_sha256 is None or source_sha256 != document["source_sha256"]:
|
||||||
return None
|
return None
|
||||||
recording_sha256 = _sha256_stable(recording_path, recording_stat)
|
recording_sha256 = document["recording_sha256"]
|
||||||
if recording_sha256 != document["recording_sha256"]:
|
if verify_digests:
|
||||||
return None
|
recording_sha256 = _sha256_stable(recording_path, recording_stat)
|
||||||
|
if recording_sha256 != document["recording_sha256"]:
|
||||||
|
return None
|
||||||
_chmod_best_effort(recording_path, 0o600)
|
_chmod_best_effort(recording_path, 0o600)
|
||||||
_chmod_best_effort(sidecar_path, 0o600)
|
_chmod_best_effort(sidecar_path, 0o600)
|
||||||
if document["schema_version"] != CACHE_SCHEMA:
|
if document["schema_version"] != CACHE_SCHEMA:
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator, Iterable
|
||||||
from contextlib import asynccontextmanager, suppress
|
from contextlib import asynccontextmanager, suppress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
@ -65,9 +65,25 @@ def _prepare_recorded_media_for_launch(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_recorded_media_for_launch(
|
||||||
|
command: ReplayCommand,
|
||||||
|
_: MaterializedRecording,
|
||||||
|
) -> tuple[RecordedMediaManifest, ...] | None:
|
||||||
|
"""Restore only previously published media descriptors."""
|
||||||
|
|
||||||
|
restored: list[RecordedMediaManifest] = []
|
||||||
|
for artifact in session_store.list_recorded_media(command.session_id):
|
||||||
|
manifest = session_recorded_media_inspector.restore_prepared(artifact, command)
|
||||||
|
if manifest is None:
|
||||||
|
return None
|
||||||
|
restored.append(manifest)
|
||||||
|
return tuple(restored)
|
||||||
|
|
||||||
|
|
||||||
session_recording_preparation_manager = SessionRecordingPreparationManager(
|
session_recording_preparation_manager = SessionRecordingPreparationManager(
|
||||||
session_recording_materializer,
|
session_recording_materializer,
|
||||||
ready_preparer=_prepare_recorded_media_for_launch,
|
ready_preparer=_prepare_recorded_media_for_launch,
|
||||||
|
ready_restorer=_restore_recorded_media_for_launch,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -82,47 +98,73 @@ def refresh_observation_catalog() -> tuple[str, ...]:
|
||||||
return tuple(dict.fromkeys(imported))
|
return tuple(dict.fromkeys(imported))
|
||||||
|
|
||||||
|
|
||||||
def enqueue_replayable_recordings() -> tuple[str, ...]:
|
def finalized_replayable_recording_ids() -> tuple[str, ...]:
|
||||||
"""Reconcile finalized catalog sessions into the durable RRD work queue."""
|
"""List finalized replayable catalog identities without scheduling work."""
|
||||||
|
|
||||||
enqueued: list[str] = []
|
finalized: list[str] = []
|
||||||
cursor: str | None = None
|
cursor: str | None = None
|
||||||
while True:
|
while True:
|
||||||
page = session_store.list_recent(limit=100, cursor=cursor)
|
page = session_store.list_recent(limit=100, cursor=cursor)
|
||||||
for summary in page.items:
|
finalized.extend(
|
||||||
if not summary.replayable or summary.status not in {
|
summary.session_id
|
||||||
"ready",
|
for summary in page.items
|
||||||
"interrupted",
|
if summary.replayable and summary.status in {"ready", "interrupted", "failed"}
|
||||||
"failed",
|
)
|
||||||
}:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
command = session_store.prepare_replay(summary.session_id)
|
|
||||||
session_recording_preparation_manager.enqueue(
|
|
||||||
command,
|
|
||||||
retry_interrupted=True,
|
|
||||||
)
|
|
||||||
except RecordingPreparationQueueFull:
|
|
||||||
return tuple(enqueued)
|
|
||||||
except Exception:
|
|
||||||
# One stale/corrupt catalog row must not starve every valid
|
|
||||||
# finalized session that follows it in the reconciliation
|
|
||||||
# page. The row remains visible as failed/interrupted evidence
|
|
||||||
# and can be repaired independently.
|
|
||||||
continue
|
|
||||||
enqueued.append(summary.session_id)
|
|
||||||
cursor = page.next_cursor
|
cursor = page.next_cursor
|
||||||
if cursor is None:
|
if cursor is None:
|
||||||
|
return tuple(finalized)
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue_replayable_recordings(session_ids: Iterable[str]) -> tuple[str, ...]:
|
||||||
|
"""Schedule only explicitly selected newly finalized sessions."""
|
||||||
|
|
||||||
|
enqueued: list[str] = []
|
||||||
|
for session_id in dict.fromkeys(session_ids):
|
||||||
|
try:
|
||||||
|
command = session_store.prepare_replay(session_id)
|
||||||
|
session_recording_preparation_manager.enqueue(
|
||||||
|
command,
|
||||||
|
retry_interrupted=True,
|
||||||
|
)
|
||||||
|
except RecordingPreparationQueueFull:
|
||||||
return tuple(enqueued)
|
return tuple(enqueued)
|
||||||
|
except Exception:
|
||||||
|
# One stale/corrupt row must not starve a later newly completed
|
||||||
|
# session. Historical cold caches remain operator-triggered.
|
||||||
|
continue
|
||||||
|
enqueued.append(session_id)
|
||||||
|
return tuple(enqueued)
|
||||||
|
|
||||||
|
|
||||||
|
def newly_finalized_recording_ids(
|
||||||
|
known_finalized: set[str] | None,
|
||||||
|
current_finalized: Iterable[str],
|
||||||
|
) -> tuple[str, ...]:
|
||||||
|
"""Return only post-startup completions; the first scan is a baseline."""
|
||||||
|
|
||||||
|
if known_finalized is None:
|
||||||
|
return ()
|
||||||
|
return tuple(sorted(set(current_finalized) - known_finalized))
|
||||||
|
|
||||||
|
|
||||||
async def _recording_preparation_reconciler() -> None:
|
async def _recording_preparation_reconciler() -> None:
|
||||||
"""Discover newly completed/recovered captures without blocking requests."""
|
"""Prepare sessions finalized during this process, never historical rows."""
|
||||||
|
|
||||||
|
known_finalized: set[str] | None = None
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
await asyncio.to_thread(refresh_observation_catalog)
|
await asyncio.to_thread(refresh_observation_catalog)
|
||||||
await asyncio.to_thread(enqueue_replayable_recordings)
|
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
|
||||||
|
newly_finalized = newly_finalized_recording_ids(
|
||||||
|
known_finalized,
|
||||||
|
finalized,
|
||||||
|
)
|
||||||
|
if newly_finalized:
|
||||||
|
await asyncio.to_thread(
|
||||||
|
enqueue_replayable_recordings,
|
||||||
|
newly_finalized,
|
||||||
|
)
|
||||||
|
known_finalized = finalized
|
||||||
except Exception:
|
except Exception:
|
||||||
# A transient filesystem/catalog failure must not permanently
|
# A transient filesystem/catalog failure must not permanently
|
||||||
# disable preparation of sessions completed later in the run.
|
# disable preparation of sessions completed later in the run.
|
||||||
|
|
|
||||||
|
|
@ -45,9 +45,7 @@ SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
|
||||||
SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
SAFE_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||||
MAX_SAFE_INTEGER = 9_007_199_254_740_991
|
MAX_SAFE_INTEGER = 9_007_199_254_740_991
|
||||||
REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform"
|
REVALIDATED_RECORDING_CACHE_CONTROL = "private, no-cache, no-transform"
|
||||||
IMMUTABLE_RECORDING_CACHE_CONTROL = (
|
IMMUTABLE_RECORDING_CACHE_CONTROL = "private, max-age=31536000, immutable, no-transform"
|
||||||
"private, max-age=31536000, immutable, no-transform"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class _ReleasingFileResponse(FileResponse):
|
class _ReleasingFileResponse(FileResponse):
|
||||||
|
|
@ -321,13 +319,17 @@ def build_session_router(
|
||||||
|
|
||||||
if recording_preparation_manager is not None:
|
if recording_preparation_manager is not None:
|
||||||
try:
|
try:
|
||||||
# Enqueue performs only catalog identity/lstat checks. A cold
|
snapshot = recording_preparation_manager.status(command.session_id)
|
||||||
# RRD cache and every camera index are validated exclusively
|
if snapshot is None:
|
||||||
# by the process-owned worker.
|
snapshot = await run_in_threadpool(
|
||||||
snapshot = recording_preparation_manager.enqueue(
|
recording_preparation_manager.restore_published,
|
||||||
command,
|
command,
|
||||||
retry_failed=True,
|
)
|
||||||
)
|
if snapshot is None:
|
||||||
|
snapshot = recording_preparation_manager.enqueue(
|
||||||
|
command,
|
||||||
|
retry_failed=True,
|
||||||
|
)
|
||||||
except RecordingPreparationQueueFull as exc:
|
except RecordingPreparationQueueFull as exc:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=503,
|
status_code=503,
|
||||||
|
|
@ -504,7 +506,12 @@ def build_session_router(
|
||||||
False,
|
False,
|
||||||
False,
|
False,
|
||||||
)
|
)
|
||||||
snapshot = recording_preparation_manager.enqueue(command)
|
snapshot = await run_in_threadpool(
|
||||||
|
recording_preparation_manager.restore_published,
|
||||||
|
command,
|
||||||
|
)
|
||||||
|
if snapshot is None:
|
||||||
|
snapshot = recording_preparation_manager.enqueue(command)
|
||||||
except SessionNotFoundError as exc:
|
except SessionNotFoundError as exc:
|
||||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||||
except (SessionNotReplayableError, SessionIntegrityError) as exc:
|
except (SessionNotReplayableError, SessionIntegrityError) as exc:
|
||||||
|
|
@ -1036,15 +1043,16 @@ def _catalog_preparation_document(
|
||||||
snapshot = manager.status(session_id)
|
snapshot = manager.status(session_id)
|
||||||
if snapshot is None:
|
if snapshot is None:
|
||||||
try:
|
try:
|
||||||
snapshot = manager.enqueue(store.prepare_replay(session_id))
|
snapshot = manager.restore_published(store.prepare_replay(session_id))
|
||||||
except (
|
except (
|
||||||
RecordingPreparationQueueFull,
|
|
||||||
SessionNotFoundError,
|
SessionNotFoundError,
|
||||||
SessionNotReplayableError,
|
SessionNotReplayableError,
|
||||||
SessionIntegrityError,
|
SessionIntegrityError,
|
||||||
ValueError,
|
ValueError,
|
||||||
):
|
):
|
||||||
return None
|
return None
|
||||||
|
if snapshot is None:
|
||||||
|
return None
|
||||||
document: dict[str, Any] = {
|
document: dict[str, Any] = {
|
||||||
"preparation_id": snapshot.preparation_id,
|
"preparation_id": snapshot.preparation_id,
|
||||||
"state": snapshot.state,
|
"state": snapshot.state,
|
||||||
|
|
@ -1165,10 +1173,7 @@ def _recorded_media_manifest_document(
|
||||||
"segments": [
|
"segments": [
|
||||||
{
|
{
|
||||||
"sequence": segment.sequence,
|
"sequence": segment.sequence,
|
||||||
"url": (
|
"url": (f"{base}/epochs/{epoch.ordinal}/segments/{segment.sequence}.m4s"),
|
||||||
f"{base}/epochs/{epoch.ordinal}/segments/"
|
|
||||||
f"{segment.sequence}.m4s"
|
|
||||||
),
|
|
||||||
"byte_length": segment.byte_length,
|
"byte_length": segment.byte_length,
|
||||||
"sha256": segment.sha256,
|
"sha256": segment.sha256,
|
||||||
}
|
}
|
||||||
|
|
@ -1198,19 +1203,13 @@ def _resolve_recorded_media_manifest(
|
||||||
detail="Некорректный идентификатор записанного медиаканала.",
|
detail="Некорректный идентификатор записанного медиаканала.",
|
||||||
) from exc
|
) from exc
|
||||||
snapshot = manager.status(session_id)
|
snapshot = manager.status(session_id)
|
||||||
if (
|
if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None:
|
||||||
snapshot is None
|
|
||||||
or snapshot.state != "ready"
|
|
||||||
or snapshot.recorded_media is None
|
|
||||||
):
|
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=409,
|
status_code=409,
|
||||||
detail="Медиаканал ещё не подготовлен.",
|
detail="Медиаканал ещё не подготовлен.",
|
||||||
)
|
)
|
||||||
matches = tuple(
|
matches = tuple(
|
||||||
manifest
|
manifest for manifest in snapshot.recorded_media if manifest.artifact_id == artifact_id
|
||||||
for manifest in snapshot.recorded_media
|
|
||||||
if manifest.artifact_id == artifact_id
|
|
||||||
)
|
)
|
||||||
if len(matches) != 1:
|
if len(matches) != 1:
|
||||||
raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.")
|
raise HTTPException(status_code=404, detail="Записанный медиаканал не найден.")
|
||||||
|
|
|
||||||
|
|
@ -574,6 +574,39 @@ def test_cold_replay_returns_quick_202_then_status_returns_ready_launch(
|
||||||
manager.close()
|
manager.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_catalog_read_never_prepares_a_cold_historical_session(tmp_path: Path) -> None:
|
||||||
|
repository = tmp_path / "repo"
|
||||||
|
sessions = repository / "sessions"
|
||||||
|
session = make_legacy_session(sessions, "20260716T205632Z_viewer_live")
|
||||||
|
store = SessionStore(repository, data_dir=tmp_path / "data")
|
||||||
|
store.reconcile_archive(xgrids_k1_archive_source(sessions))
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
def exporter(_source: Path, _destination: Path) -> dict[str, object]:
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise AssertionError("catalog reads must not invoke the exporter")
|
||||||
|
|
||||||
|
materializer = SessionRecordingMaterializer(store.data_dir, exporter=exporter)
|
||||||
|
manager = SessionRecordingPreparationManager(materializer)
|
||||||
|
router = build_session_router(
|
||||||
|
store,
|
||||||
|
recording_materializer=materializer,
|
||||||
|
recording_preparation_manager=manager,
|
||||||
|
)
|
||||||
|
list_route = endpoint(router, "/api/v1/observation-sessions", "GET")
|
||||||
|
try:
|
||||||
|
item = list_route(limit=20, cursor=None)["items"][0]
|
||||||
|
time.sleep(0.02)
|
||||||
|
|
||||||
|
assert item["id"] == session.name
|
||||||
|
assert item["preparation"] is None
|
||||||
|
assert manager.status(session.name) is None
|
||||||
|
assert calls == 0
|
||||||
|
finally:
|
||||||
|
manager.close()
|
||||||
|
|
||||||
|
|
||||||
def test_recording_response_releases_pin_once_when_asgi_send_fails(
|
def test_recording_response_releases_pin_once_when_asgi_send_fails(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
|
||||||
|
|
@ -160,7 +160,7 @@ def test_catalog_refresh_failure_is_a_stable_service_error(tmp_path: Path) -> No
|
||||||
assert "private local path" not in str(error.value.detail)
|
assert "private local path" not in str(error.value.detail)
|
||||||
|
|
||||||
|
|
||||||
def test_startup_warmup_enqueues_every_finalized_replayable_session(
|
def test_startup_scan_baselines_historical_sessions_without_enqueuing(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
@ -188,17 +188,14 @@ def test_startup_warmup_enqueues_every_finalized_replayable_session(
|
||||||
monkeypatch.setattr(app_module, "session_store", store)
|
monkeypatch.setattr(app_module, "session_store", store)
|
||||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||||
try:
|
try:
|
||||||
enqueued = app_module.enqueue_replayable_recordings()
|
historical = set(app_module.finalized_replayable_recording_ids())
|
||||||
deadline = time.monotonic() + 2
|
newly_finalized = app_module.newly_finalized_recording_ids(None, historical)
|
||||||
snapshot = manager.status(session.name)
|
enqueued = app_module.enqueue_replayable_recordings(newly_finalized)
|
||||||
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
|
|
||||||
time.sleep(0.005)
|
|
||||||
snapshot = manager.status(session.name)
|
|
||||||
|
|
||||||
assert enqueued == (session.name,)
|
assert historical == {session.name}
|
||||||
assert snapshot is not None
|
assert newly_finalized == ()
|
||||||
assert snapshot.state == "ready"
|
assert enqueued == ()
|
||||||
assert snapshot.recording is not None
|
assert manager.status(session.name) is None
|
||||||
finally:
|
finally:
|
||||||
manager.close()
|
manager.close()
|
||||||
|
|
||||||
|
|
@ -233,7 +230,7 @@ def test_reconciliation_skips_one_stale_session_and_prepares_later_valid_session
|
||||||
monkeypatch.setattr(app_module, "session_store", store)
|
monkeypatch.setattr(app_module, "session_store", store)
|
||||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||||
try:
|
try:
|
||||||
enqueued = app_module.enqueue_replayable_recordings()
|
enqueued = app_module.enqueue_replayable_recordings((stale.name, good.name))
|
||||||
deadline = time.monotonic() + 2
|
deadline = time.monotonic() + 2
|
||||||
snapshot = manager.status(good.name)
|
snapshot = manager.status(good.name)
|
||||||
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
|
while snapshot is not None and snapshot.state != "ready" and time.monotonic() < deadline:
|
||||||
|
|
@ -283,7 +280,7 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
|
||||||
monkeypatch.setattr(app_module, "session_store", store)
|
monkeypatch.setattr(app_module, "session_store", store)
|
||||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||||
try:
|
try:
|
||||||
app_module.enqueue_replayable_recordings()
|
app_module.enqueue_replayable_recordings((session.name,))
|
||||||
assert started.wait(timeout=1)
|
assert started.wait(timeout=1)
|
||||||
manager.close(timeout=0.001)
|
manager.close(timeout=0.001)
|
||||||
manager.start()
|
manager.start()
|
||||||
|
|
@ -291,15 +288,13 @@ def test_reconciler_requeues_only_a_restart_interrupted_job(
|
||||||
|
|
||||||
deadline = time.monotonic() + 2
|
deadline = time.monotonic() + 2
|
||||||
snapshot = manager.status(session.name)
|
snapshot = manager.status(session.name)
|
||||||
while (
|
while (snapshot is None or snapshot.state != "cancelled") and time.monotonic() < deadline:
|
||||||
snapshot is None or snapshot.state != "cancelled"
|
|
||||||
) and time.monotonic() < deadline:
|
|
||||||
time.sleep(0.005)
|
time.sleep(0.005)
|
||||||
snapshot = manager.status(session.name)
|
snapshot = manager.status(session.name)
|
||||||
assert snapshot is not None and snapshot.state == "cancelled"
|
assert snapshot is not None and snapshot.state == "cancelled"
|
||||||
interrupted_id = snapshot.preparation_id
|
interrupted_id = snapshot.preparation_id
|
||||||
|
|
||||||
app_module.enqueue_replayable_recordings()
|
app_module.enqueue_replayable_recordings((session.name,))
|
||||||
deadline = time.monotonic() + 2
|
deadline = time.monotonic() + 2
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
snapshot = manager.status(session.name)
|
snapshot = manager.status(session.name)
|
||||||
|
|
@ -336,18 +331,16 @@ def test_reconciler_does_not_loop_retry_a_genuine_failed_job(
|
||||||
monkeypatch.setattr(app_module, "session_store", store)
|
monkeypatch.setattr(app_module, "session_store", store)
|
||||||
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
monkeypatch.setattr(app_module, "session_recording_preparation_manager", manager)
|
||||||
try:
|
try:
|
||||||
app_module.enqueue_replayable_recordings()
|
app_module.enqueue_replayable_recordings((session.name,))
|
||||||
deadline = time.monotonic() + 2
|
deadline = time.monotonic() + 2
|
||||||
snapshot = manager.status(session.name)
|
snapshot = manager.status(session.name)
|
||||||
while (
|
while (snapshot is None or snapshot.state != "failed") and time.monotonic() < deadline:
|
||||||
snapshot is None or snapshot.state != "failed"
|
|
||||||
) and time.monotonic() < deadline:
|
|
||||||
time.sleep(0.005)
|
time.sleep(0.005)
|
||||||
snapshot = manager.status(session.name)
|
snapshot = manager.status(session.name)
|
||||||
assert snapshot is not None and snapshot.state == "failed"
|
assert snapshot is not None and snapshot.state == "failed"
|
||||||
failed_id = snapshot.preparation_id
|
failed_id = snapshot.preparation_id
|
||||||
|
|
||||||
app_module.enqueue_replayable_recordings()
|
app_module.enqueue_replayable_recordings((session.name,))
|
||||||
time.sleep(0.05)
|
time.sleep(0.05)
|
||||||
unchanged = manager.status(session.name)
|
unchanged = manager.status(session.name)
|
||||||
assert unchanged is not None
|
assert unchanged is not None
|
||||||
|
|
|
||||||
|
|
@ -231,6 +231,46 @@ def test_manager_can_restart_across_repeated_application_lifespans(tmp_path: Pat
|
||||||
manager.close()
|
manager.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_published_package_restores_after_process_restart_without_export(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
command = _command(tmp_path / "session")
|
||||||
|
private_root = tmp_path / "private"
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
def exporter(source: Path, destination: Path) -> dict[str, object]:
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
return _summary(source, destination, b"durable-recording")
|
||||||
|
|
||||||
|
first = SessionRecordingPreparationManager(
|
||||||
|
SessionRecordingMaterializer(private_root, exporter=exporter)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
first.enqueue(command)
|
||||||
|
ready = _wait_for_state(first, command.session_id, {"ready"})
|
||||||
|
assert ready.recording is not None
|
||||||
|
finally:
|
||||||
|
first.close()
|
||||||
|
|
||||||
|
def forbidden_exporter(_source: Path, _destination: Path) -> dict[str, object]:
|
||||||
|
raise AssertionError("a published package must not be exported again")
|
||||||
|
|
||||||
|
restarted = SessionRecordingPreparationManager(
|
||||||
|
SessionRecordingMaterializer(private_root, exporter=forbidden_exporter)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
restored = restarted.restore_published(command)
|
||||||
|
|
||||||
|
assert restored is not None
|
||||||
|
assert restored.state == "ready"
|
||||||
|
assert restored.recording is not None
|
||||||
|
assert restored.recording.path.read_bytes() == b"durable-recording"
|
||||||
|
assert calls == 1
|
||||||
|
finally:
|
||||||
|
restarted.close()
|
||||||
|
|
||||||
|
|
||||||
def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer(
|
def test_noncooperative_exporter_has_no_fake_heartbeat_and_restart_waits_for_old_writer(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue