SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): publish immutable lab session instances

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-23 20:13:40 +03:00
parent f1f8e21b78
commit 10019a454a
16 changed files with 1970 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/sessions/__init__.py
+4
View File
@@ -5,6 +5,7 @@ from .active import (
ActiveSessionLeaseError,
recover_stale_active_session_marker,
)
from .lab_cache import publish_lab_replay_cache
from .media import (
RECORDED_MEDIA_MANIFEST_SCHEMA,
RecordedMediaFile,
@@ -14,6 +15,7 @@ from .media import (
validate_recorded_media_timeline,
)
from .models import (
LabSessionBinding,
LayoutConflictError,
ObservationArtifactCandidate,
ObservationSessionCandidate,
@@ -50,6 +52,7 @@ from .store import (
__all__ = [
"LayoutConflictError",
"LabSessionBinding",
"ActiveSessionLease",
"ActiveSessionLeaseError",
"MaterializedRecording",
@@ -59,6 +62,7 @@ __all__ = [
"ObservationSessionCandidate",
"PluginRecordingExportCancelled",
"PluginRecordingExportError",
"publish_lab_replay_cache",
"RecordingMaterializationCancelled",
"RecordedMediaArtifact",
"RECORDED_MEDIA_MANIFEST_SCHEMA",
src/k1link/sessions/lab_cache.py
+139
View File
@@ -0,0 +1,139 @@
"""Publish zero-copy replay caches for immutable LAB session aliases."""
from __future__ import annotations
import hashlib
import json
import os
import stat
from pathlib import Path
from typing import Any
from k1link.artifacts import write_json_atomic
from .models import SessionIntegrityError
from .recording import (
COMPATIBLE_CACHE_SCHEMAS,
RECORDING_CACHE_FILENAME,
RECORDING_CACHE_SIDECAR_FILENAME,
)
def publish_lab_replay_cache(
data_dir: Path,
*,
source_session_id: str,
lab_session_id: str,
) -> None:
"""Hard-link a validated source RRD and clone its path-free sidecars.
The RRD bytes are immutable and independent of the catalog id, so a hard
link gives every LAB instance a normal cache endpoint without consuming a
second copy of the multi-gigabyte recording.
"""
root = data_dir.expanduser().resolve(strict=True)
recordings = (root / "recordings").resolve(strict=True)
source_root = recordings / source_session_id
source_rrd = source_root / RECORDING_CACHE_FILENAME
source_sidecar = source_root / RECORDING_CACHE_SIDECAR_FILENAME
_require_regular(source_rrd, source_root)
_require_regular(source_sidecar, source_root)
document = _read_object(source_sidecar)
if (
document.get("schema_version") not in COMPATIBLE_CACHE_SCHEMAS
or document.get("session_id") != source_session_id
):
raise SessionIntegrityError("source recording cache is incompatible")
target_root = recordings / lab_session_id
target_root.mkdir(mode=0o700, parents=False, exist_ok=True)
if target_root.is_symlink() or target_root.resolve() != target_root:
raise SessionIntegrityError("LAB recording cache root is unsafe")
target_rrd = target_root / RECORDING_CACHE_FILENAME
if target_rrd.exists():
source_stat = _require_regular(source_rrd, source_root)
target_stat = _require_regular(target_rrd, target_root)
if (source_stat.st_dev, source_stat.st_ino) != (target_stat.st_dev, target_stat.st_ino):
raise SessionIntegrityError("LAB recording cache already contains different bytes")
else:
temporary = target_root / f".{RECORDING_CACHE_FILENAME}.{os.getpid()}.tmp"
try:
os.link(source_rrd, temporary, follow_symlinks=False)
os.replace(temporary, target_rrd)
finally:
temporary.unlink(missing_ok=True)
target_stat = _require_regular(target_rrd, target_root)
cloned = {
**document,
"session_id": lab_session_id,
"recording_mtime_ns": target_stat.st_mtime_ns,
}
write_json_atomic(target_root / RECORDING_CACHE_SIDECAR_FILENAME, cloned)
os.chmod(target_root / RECORDING_CACHE_SIDECAR_FILENAME, 0o600)
_clone_recorded_media_sidecars(
root / "recorded-media-preparations",
source_session_id=source_session_id,
lab_session_id=lab_session_id,
)
def _clone_recorded_media_sidecars(
root: Path,
*,
source_session_id: str,
lab_session_id: str,
) -> None:
if not root.is_dir():
return
for source in root.iterdir():
if source.is_symlink() or not source.is_file():
continue
try:
document = _read_object(source)
except (OSError, SessionIntegrityError):
continue
if document.get("session_id") != source_session_id:
continue
artifact_id = document.get("artifact_id")
if not isinstance(artifact_id, str):
raise SessionIntegrityError("recorded media sidecar has no artifact identity")
body = {key: value for key, value in document.items() if key != "checksum_sha256"}
body["session_id"] = lab_session_id
checksum = hashlib.sha256(_canonical_json(body)).hexdigest()
target = root / _media_sidecar_name(lab_session_id, artifact_id)
write_json_atomic(target, {**body, "checksum_sha256": checksum})
os.chmod(target, 0o600)
def _media_sidecar_name(session_id: str, artifact_id: str) -> str:
key = f"{session_id}\0{artifact_id}".encode()
return f"{hashlib.sha256(key).hexdigest()}.json"
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
def _read_object(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, dict):
raise SessionIntegrityError("cache sidecar is not a JSON object")
return value
def _require_regular(path: Path, parent: Path) -> os.stat_result:
metadata = path.lstat()
if (
stat.S_ISLNK(metadata.st_mode)
or not stat.S_ISREG(metadata.st_mode)
or path.parent.resolve(strict=True) != parent.resolve(strict=True)
):
raise SessionIntegrityError("cache artifact is not a confined regular file")
return metadata
src/k1link/sessions/models.py
+34 -1
View File
@@ -28,6 +28,35 @@ class LayoutConflictError(SessionStoreError):
"""A workspace layout revision changed since the caller loaded it."""
@dataclass(frozen=True, slots=True)
class LabSessionBinding:
"""Immutable provenance for one derived laboratory replay."""
session_id: str
source_session_id: str
lab_id: str
result_kind: str
result_id: str
source_result_id: str | None
config_sha256: str | None
run_created_at_utc: str
published_at_utc: str
provenance: dict[str, Any]
def as_dict(self) -> dict[str, Any]:
return {
"lab_id": self.lab_id,
"source_session_id": self.source_session_id,
"result_kind": self.result_kind,
"result_id": self.result_id,
"source_result_id": self.source_result_id,
"config_sha256": self.config_sha256,
"run_created_at_utc": self.run_created_at_utc,
"published_at_utc": self.published_at_utc,
"provenance": self.provenance,
}
@dataclass(frozen=True, slots=True)
class SessionSource:
source_id: str
@@ -81,9 +110,10 @@ class SessionSummary:
total_bytes: int
replayable: bool
origin: str
lab: LabSessionBinding | None = None
def as_dict(self) -> dict[str, Any]:
return {
document: dict[str, Any] = {
"schema_version": "missioncore.observation-session-summary/v1",
"session_id": self.session_id,
"display_name": self.display_name,
@@ -97,6 +127,9 @@ class SessionSummary:
"replayable": self.replayable,
"origin": self.origin,
}
if self.lab is not None:
document["lab"] = self.lab.as_dict()
return document
@dataclass(frozen=True, slots=True)
src/k1link/sessions/store.py
+324 -4
View File
@@ -16,6 +16,7 @@ from uuid import uuid4
from k1link.artifacts import utc_now_iso
from .models import (
LabSessionBinding,
LayoutConflictError,
ObservationArtifactCandidate,
ObservationSessionCandidate,
@@ -39,6 +40,10 @@ from .plugin_contract import ObservationArchiveSource
IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
MAX_LAYOUT_BYTES = 256 * 1024
DATABASE_NAME = "mission-core.sqlite3"
LAB_ARCHIVE_ID = "missioncore.lab-instances"
LAB_ORIGIN = "missioncore.lab-instance/v1"
LAB_ID_PATTERN = re.compile(r"^LAB [A-Z][A-Z0-9._-]{0,31}$")
SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$")
SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS observation_sessions (
@@ -94,6 +99,24 @@ CREATE TABLE IF NOT EXISTS observation_session_sources (
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS observation_lab_instances (
session_id TEXT PRIMARY KEY
REFERENCES observation_sessions(session_id) ON DELETE CASCADE,
source_session_id TEXT NOT NULL
REFERENCES observation_sessions(session_id) ON DELETE RESTRICT,
lab_id TEXT NOT NULL,
result_kind TEXT NOT NULL,
result_id TEXT NOT NULL,
source_result_id TEXT,
config_sha256 TEXT,
run_created_at_utc TEXT NOT NULL,
published_at_utc TEXT NOT NULL,
provenance_json TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS observation_lab_instances_source
ON observation_lab_instances(source_session_id, published_at_utc DESC);
CREATE TABLE IF NOT EXISTS workspace_layouts (
workspace_id TEXT PRIMARY KEY,
layout_schema_version INTEGER NOT NULL,
@@ -162,7 +185,18 @@ class SessionStore:
"WHERE plugin_id = ? AND archive_id = ? AND allowed_root = ?",
(source.plugin_id, source.archive_id, str(allowed_root)),
).fetchall()
stale = [row["session_id"] for row in indexed if row["session_id"] not in discovered]
referenced_sources = {
row["source_session_id"]
for row in connection.execute(
"SELECT DISTINCT source_session_id FROM observation_lab_instances"
).fetchall()
}
stale = [
row["session_id"]
for row in indexed
if row["session_id"] not in discovered
and row["session_id"] not in referenced_sources
]
connection.executemany(
"DELETE FROM observation_sessions WHERE session_id = ?",
((session_id,) for session_id in stale),
@@ -193,9 +227,31 @@ class SessionStore:
"ORDER BY COALESCE(started_at_utc, '') DESC, session_id DESC LIMIT ?",
parameters,
).fetchall()
lab_rows = (
{
row["session_id"]: row
for row in connection.execute(
"SELECT * FROM observation_lab_instances WHERE session_id IN "
f"({','.join('?' for _ in rows)})", # noqa: S608 - placeholder count only
tuple(row["session_id"] for row in rows),
).fetchall()
}
if rows
else {}
)
has_more = len(rows) > limit
selected = rows[:limit]
items = tuple(_summary_from_row(row) for row in selected)
items = tuple(
_summary_from_row(
row,
lab=(
_lab_binding_from_row(lab_rows[row["session_id"]])
if row["session_id"] in lab_rows
else None
),
)
for row in selected
)
next_cursor = items[-1].session_id if has_more and items else None
return SessionPage(items=items, next_cursor=next_cursor)
@@ -218,6 +274,10 @@ class SessionStore:
"FROM observation_session_artifacts WHERE session_id = ? ORDER BY artifact_id",
(session_id,),
).fetchall()
lab_row = connection.execute(
"SELECT * FROM observation_lab_instances WHERE session_id = ?",
(session_id,),
).fetchone()
sources = tuple(
SessionSource(
source_id=source["source_id"],
@@ -240,7 +300,191 @@ class SessionStore:
)
for artifact in artifact_rows
)
return SessionDetail(summary=_summary_from_row(row), sources=sources, artifacts=artifacts)
return SessionDetail(
summary=_summary_from_row(
row,
lab=None if lab_row is None else _lab_binding_from_row(lab_row),
),
sources=sources,
artifacts=artifacts,
)
def get_lab_instance(self, session_id: str) -> LabSessionBinding | None:
"""Return immutable LAB provenance without exposing filesystem locators."""
_validate_identifier(session_id, "session id")
with self._connect() as connection:
row = connection.execute(
"SELECT * FROM observation_lab_instances WHERE session_id = ?",
(session_id,),
).fetchone()
return None if row is None else _lab_binding_from_row(row)
def publish_lab_instance(
self,
*,
session_id: str,
source_session_id: str,
display_name: str,
lab_id: str,
result_kind: str,
result_id: str,
run_created_at_utc: str,
source_result_id: str | None = None,
config_sha256: str | None = None,
provenance: dict[str, Any] | None = None,
) -> LabSessionBinding:
"""Append one immutable catalog projection over an existing source session.
Artifact/source rows are copied as references to the sealed evidence.
No raw payload is copied and the LAB row uses a distinct archive id so
device-plugin reconciliation cannot replace it.
"""
_validate_identifier(session_id, "LAB session id")
_validate_identifier(source_session_id, "source session id")
if session_id == source_session_id:
raise ValueError("LAB session id must differ from its source")
if LAB_ID_PATTERN.fullmatch(lab_id) is None:
raise ValueError("LAB id must use the form 'LAB E21'")
_validate_identifier(result_kind, "LAB result kind")
_validate_identifier(result_id, "LAB result id")
if source_result_id is not None:
_validate_identifier(source_result_id, "LAB source result id")
if config_sha256 is not None and SHA256_PATTERN.fullmatch(config_sha256) is None:
raise ValueError("LAB configuration SHA-256 is invalid")
normalized_name = display_name.strip()
if not 1 <= len(normalized_name) <= 160:
raise ValueError("LAB display name must contain 1..160 characters")
_require_utc_timestamp(run_created_at_utc, "LAB run creation time")
serialized_provenance = _serialize_provenance(provenance or {})
published_at = utc_now_iso()
with self._lock, self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
source = connection.execute(
"SELECT * FROM observation_sessions WHERE session_id = ?",
(source_session_id,),
).fetchone()
if source is None:
connection.rollback()
raise SessionNotFoundError("LAB source observation session was not found")
if (
connection.execute(
"SELECT 1 FROM observation_lab_instances WHERE session_id = ?",
(source_session_id,),
).fetchone()
is not None
):
connection.rollback()
raise SessionIntegrityError("LAB instances cannot be chained")
existing = connection.execute(
"SELECT * FROM observation_lab_instances WHERE session_id = ?",
(session_id,),
).fetchone()
expected = {
"source_session_id": source_session_id,
"lab_id": lab_id,
"result_kind": result_kind,
"result_id": result_id,
"source_result_id": source_result_id,
"config_sha256": config_sha256,
"run_created_at_utc": run_created_at_utc,
"provenance_json": serialized_provenance,
}
if existing is not None:
if any(existing[key] != value for key, value in expected.items()):
connection.rollback()
raise SessionIntegrityError(
"LAB session id is already bound to different provenance"
)
connection.commit()
return _lab_binding_from_row(existing)
if (
connection.execute(
"SELECT 1 FROM observation_sessions WHERE session_id = ?",
(session_id,),
).fetchone()
is not None
):
connection.rollback()
raise SessionIntegrityError("LAB session id already exists in the catalog")
connection.execute(
"INSERT INTO observation_sessions "
"(session_id, plugin_id, archive_id, display_name, status, "
"started_at_utc, completed_at_utc, duration_seconds, modalities_json, "
"replayable, origin, source_count, total_bytes, "
"primary_replay_artifact_id, timeline_origin_epoch_ns, "
"timeline_origin_monotonic_ns, allowed_root, session_root, "
"created_at_utc, updated_at_utc) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
session_id,
source["plugin_id"],
LAB_ARCHIVE_ID,
normalized_name,
source["status"],
run_created_at_utc,
run_created_at_utc,
source["duration_seconds"],
source["modalities_json"],
source["replayable"],
LAB_ORIGIN,
source["source_count"],
source["total_bytes"],
source["primary_replay_artifact_id"],
source["timeline_origin_epoch_ns"],
source["timeline_origin_monotonic_ns"],
source["allowed_root"],
source["session_root"],
published_at,
published_at,
),
)
connection.execute(
"INSERT INTO observation_session_artifacts "
"(session_id, artifact_id, kind, media_type, byte_length, sha256, "
"integrity_status, locator, replay_byte_length) "
"SELECT ?, artifact_id, kind, media_type, byte_length, sha256, "
"integrity_status, locator, replay_byte_length "
"FROM observation_session_artifacts WHERE session_id = ?",
(session_id, source_session_id),
)
connection.execute(
"INSERT INTO observation_session_sources "
"(session_id, source_id, semantic_channel_id, modality, status, "
"seekable, artifact_id) "
"SELECT ?, source_id, semantic_channel_id, modality, status, "
"seekable, artifact_id FROM observation_session_sources "
"WHERE session_id = ?",
(session_id, source_session_id),
)
connection.execute(
"INSERT INTO observation_lab_instances "
"(session_id, source_session_id, lab_id, result_kind, result_id, "
"source_result_id, config_sha256, run_created_at_utc, "
"published_at_utc, provenance_json) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
session_id,
source_session_id,
lab_id,
result_kind,
result_id,
source_result_id,
config_sha256,
run_created_at_utc,
published_at,
serialized_provenance,
),
)
connection.commit()
binding = self.get_lab_instance(session_id)
if binding is None:
raise SessionIntegrityError("LAB session publication was not durable")
return binding
def delete_session(self, session_id: str) -> None:
"""Permanently delete one exact catalogued evidence directory and row."""
@@ -248,6 +492,29 @@ class SessionStore:
_validate_identifier(session_id, "session id")
with self._lock:
with self._connect() as connection:
lab = connection.execute(
"SELECT 1 FROM observation_lab_instances WHERE session_id = ?",
(session_id,),
).fetchone()
if lab is not None:
connection.execute("BEGIN IMMEDIATE")
deleted = connection.execute(
"DELETE FROM observation_sessions WHERE session_id = ?",
(session_id,),
).rowcount
connection.commit()
if deleted != 1:
raise SessionNotFoundError("observation session was not found")
return
dependent = connection.execute(
"SELECT session_id FROM observation_lab_instances "
"WHERE source_session_id = ? LIMIT 1",
(session_id,),
).fetchone()
if dependent is not None:
raise SessionIntegrityError(
"source session has LAB instances; remove those instances first"
)
row = connection.execute(
"SELECT allowed_root, session_root FROM observation_sessions "
"WHERE session_id = ?",
@@ -737,7 +1004,11 @@ def _validate_candidate_replay(
raise SessionIntegrityError("non-replayable observation declares replay artifacts")
def _summary_from_row(row: sqlite3.Row) -> SessionSummary:
def _summary_from_row(
row: sqlite3.Row,
*,
lab: LabSessionBinding | None = None,
) -> SessionSummary:
raw_modalities = json.loads(row["modalities_json"])
modalities = tuple(cast(SessionModality, value) for value in raw_modalities)
return SessionSummary(
@@ -752,9 +1023,58 @@ def _summary_from_row(row: sqlite3.Row) -> SessionSummary:
total_bytes=row["total_bytes"],
replayable=bool(row["replayable"]),
origin=row["origin"],
lab=lab,
)
def _lab_binding_from_row(row: sqlite3.Row) -> LabSessionBinding:
try:
provenance = json.loads(row["provenance_json"])
except (TypeError, json.JSONDecodeError) as exc:
raise SessionIntegrityError("stored LAB provenance is invalid") from exc
if not isinstance(provenance, dict):
raise SessionIntegrityError("stored LAB provenance is not an object")
return LabSessionBinding(
session_id=row["session_id"],
source_session_id=row["source_session_id"],
lab_id=row["lab_id"],
result_kind=row["result_kind"],
result_id=row["result_id"],
source_result_id=row["source_result_id"],
config_sha256=row["config_sha256"],
run_created_at_utc=row["run_created_at_utc"],
published_at_utc=row["published_at_utc"],
provenance=provenance,
)
def _serialize_provenance(value: dict[str, Any]) -> str:
try:
serialized = json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
except (TypeError, ValueError) as exc:
raise ValueError("LAB provenance must be JSON-compatible") from exc
if len(serialized.encode("utf-8")) > 256 * 1024:
raise ValueError("LAB provenance exceeds 256 KiB")
return serialized
def _require_utc_timestamp(value: str, field: str) -> None:
from datetime import datetime
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise ValueError(f"{field} is invalid") from exc
if parsed.tzinfo is None:
raise ValueError(f"{field} must include a timezone")
def _layout_from_row(row: sqlite3.Row) -> WorkspaceLayout:
payload = json.loads(row["layout_json"])
if not isinstance(payload, dict):