SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): integrate calibrated operator pipeline

Browse Source 
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/web/camera_archive.py
+183 -60
View File
@@ -26,9 +26,9 @@ _EPOCH_DIRECTORY = re.compile(r"^epoch-([1-9][0-9]*)$")
_SEGMENT_FILE = re.compile(r"^([1-9][0-9]*)\.m4s$")
_DEFAULT_COMMIT_INTERVAL_SECONDS = 0.25
_DEFAULT_COMMIT_BYTES = 4 * 1024 * 1024
_MAX_RECOVERY_INDEX_BYTES = 32 * 1024 * 1024
_MAX_RECOVERY_SUMMARY_BYTES = 2 * 1024 * 1024
_MAX_RECOVERY_INDEX_LINE_BYTES = 64 * 1024
_MAX_RECOVERY_SEGMENT_BYTES = 8 * 1024 * 1024
_MAX_RECOVERY_SEGMENTS = 500_000
_ACTIVE_ARCHIVES_LOCK = threading.Lock()
_ACTIVE_ARCHIVES: set[Path] = set()
@@ -394,9 +394,11 @@ def recover_incomplete_camera_archives(
``interrupted`` summary is atomically written.
This is intentionally a startup/catalog-refresh operation, not a hot-path
operation: it hashes media fragments. The callable is safe to repeat, but a
composition layer should normally execute it once per server process before
the first catalog import.
operation. A clean sealed epoch is validated from its summary, streaming
index and segment stat metadata without loading payloads; only an incomplete
or damaged epoch enters fragment recovery and hashes candidate payloads. The
callable is safe to repeat, but a composition layer should normally execute
it once per server process before the first catalog import.
"""
root = sessions_root.expanduser().resolve()
@@ -468,30 +470,25 @@ def _recover_epoch(
if segments_fd is None:
return None
try:
old_index = _read_regular_at(
if _sealed_epoch_is_valid_on_disk(
epoch_fd,
segments_fd,
source_id=source_id,
init=init,
):
return None
old_index = _read_regular_at_current_size(
epoch_fd,
"index.jsonl",
_MAX_RECOVERY_INDEX_BYTES,
allow_empty=True,
)
old_summary = _read_regular_at(
epoch_fd,
"summary.json",
_MAX_RECOVERY_INDEX_BYTES,
_MAX_RECOVERY_SUMMARY_BYTES,
allow_empty=False,
)
segment_payloads, segment_timestamps, orphans = _read_recovery_segments(
segments_fd
)
if _sealed_epoch_is_valid(
source_id=source_id,
summary_bytes=old_summary,
index_bytes=old_index,
segment_payloads=segment_payloads,
orphans=orphans,
):
return None
segment_timestamps, orphans = _read_recovery_segment_catalog(segments_fd)
old_entries = _parse_index_prefix(old_index)
old_by_sequence = {
int(entry["sequence"]): entry
@@ -502,9 +499,17 @@ def _recover_epoch(
stream_hash = hashlib.sha256(init)
valid_bytes = len(init)
expected = 1
while expected <= _MAX_RECOVERY_SEGMENTS:
payload = segment_payloads.get(expected)
while True:
if expected not in segment_timestamps:
break
payload = _read_regular_at(
segments_fd,
f"{expected}.m4s",
_MAX_RECOVERY_SEGMENT_BYTES,
allow_empty=False,
)
if payload is None:
orphans.append(f"{expected}.m4s")
break
digest = hashlib.sha256(payload).hexdigest()
previous = old_by_sequence.get(expected)
@@ -528,11 +533,10 @@ def _recover_epoch(
valid_bytes += len(payload)
expected += 1
valid_sequences = {int(entry["sequence"]) for entry in recovered_entries}
orphans.extend(
f"{sequence}.m4s"
for sequence in segment_payloads
if sequence not in valid_sequences
for sequence in segment_timestamps
if sequence >= expected
)
orphans = sorted(set(orphans))
if not recovered_entries:
@@ -611,48 +615,145 @@ def _recover_epoch(
os.close(epoch_fd)
def _sealed_epoch_is_valid(
def _sealed_epoch_is_valid_on_disk(
epoch_fd: int,
segments_fd: int,
*,
source_id: str,
summary_bytes: bytes | None,
index_bytes: bytes | None,
segment_payloads: dict[int, bytes],
orphans: list[str],
init: bytes,
) -> bool:
if not summary_bytes or index_bytes is None or orphans:
"""Recognize a clean seal without loading a multi-hour archive into RAM.
Recovery only needs to prove that the durable commit envelope is complete.
Full fragment digests and ISO-BMFF timing are revalidated by the recorded
media preparation path before browser publication.
"""
summary_bytes = _read_regular_at(
epoch_fd,
"summary.json",
_MAX_RECOVERY_SUMMARY_BYTES,
allow_empty=False,
)
if not summary_bytes:
return False
try:
summary = json.loads(summary_bytes)
except (UnicodeDecodeError, json.JSONDecodeError):
return False
if not isinstance(summary, dict) or summary.get("source_id") != source_id:
return False
segment_count = summary.get("segment_count")
segment_count = summary.get("segment_count") if isinstance(summary, dict) else None
if (
not isinstance(segment_count, int)
not isinstance(summary, dict)
or summary.get("schema_version") != CAMERA_ARCHIVE_SCHEMA
or summary.get("source_id") != source_id
or not isinstance(segment_count, int)
or isinstance(segment_count, bool)
or segment_count < 1
or set(segment_payloads) != set(range(1, segment_count + 1))
or summary.get("entry_count") != segment_count
or summary.get("media_segment_count") != segment_count
or summary.get("commit_policy") != CAMERA_COMMIT_POLICY
or summary.get("init_sha256") != hashlib.sha256(init).hexdigest()
):
return False
entries = _parse_index_prefix(index_bytes)
if len(entries) != segment_count:
return False
return all(
_index_entry_matches(
entry,
sequence,
len(segment_payloads[sequence]),
hashlib.sha256(segment_payloads[sequence]).hexdigest(),
try:
descriptor = os.open(
"index.jsonl",
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
dir_fd=epoch_fd,
)
for sequence, entry in enumerate(entries, start=1)
except OSError:
return False
digest = hashlib.sha256()
valid_bytes = len(init)
try:
before = os.fstat(descriptor)
if not stat.S_ISREG(before.st_mode) or before.st_size <= 0:
return False
with os.fdopen(descriptor, "rb") as stream:
descriptor = -1
for sequence in range(1, segment_count + 1):
raw_line = stream.readline(_MAX_RECOVERY_INDEX_LINE_BYTES + 1)
if (
not raw_line
or len(raw_line) > _MAX_RECOVERY_INDEX_LINE_BYTES
or not raw_line.endswith(b"\n")
):
return False
digest.update(raw_line)
try:
entry = json.loads(raw_line)
except (UnicodeDecodeError, json.JSONDecodeError):
return False
length = entry.get("length") if isinstance(entry, dict) else None
if (
not isinstance(length, int)
or isinstance(length, bool)
or not 0 < length <= _MAX_RECOVERY_SEGMENT_BYTES
or not _index_entry_shape_matches(entry, sequence)
):
return False
try:
segment_stat = os.stat(
f"{sequence}.m4s",
dir_fd=segments_fd,
follow_symlinks=False,
)
except OSError:
return False
if not stat.S_ISREG(segment_stat.st_mode) or segment_stat.st_size != length:
return False
valid_bytes += length
if stream.read(1):
return False
after = os.fstat(stream.fileno())
if (
(before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns)
!= (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns)
):
return False
except OSError:
return False
finally:
if descriptor >= 0:
os.close(descriptor)
try:
segment_names = [
name
for name in os.listdir(segments_fd)
if _SEGMENT_FILE.fullmatch(name) is not None
]
except OSError:
return False
if len(segment_names) != segment_count or any(
name != f"{sequence}.m4s"
for sequence, name in enumerate(
sorted(segment_names, key=lambda name: int(name.removesuffix(".m4s"))),
start=1,
)
):
return False
return (
summary.get("index_sha256") == digest.hexdigest()
and summary.get("valid_bytes") == valid_bytes
)
def _read_recovery_segments(
def _index_entry_shape_matches(entry: object, sequence: int) -> bool:
return (
isinstance(entry, dict)
and entry.get("schema_version") == CAMERA_INDEX_SCHEMA
and entry.get("sequence") == sequence
and entry.get("kind") == "media"
and entry.get("path") == f"segments/{sequence}.m4s"
and isinstance(entry.get("sha256"), str)
and re.fullmatch(r"[a-f0-9]{64}", str(entry["sha256"])) is not None
)
def _read_recovery_segment_catalog(
segments_fd: int,
) -> tuple[dict[int, bytes], dict[int, int], list[str]]:
payloads: dict[int, bytes] = {}
) -> tuple[dict[int, int], list[str]]:
timestamps: dict[int, int] = {}
orphans: list[str] = []
try:
@@ -664,22 +765,22 @@ def _read_recovery_segments(
if match is None:
continue
sequence = int(match.group(1))
if name != f"{sequence}.m4s" or sequence in payloads:
if name != f"{sequence}.m4s" or sequence in timestamps:
orphans.append(name)
continue
read_result = _read_regular_with_metadata_at(
segments_fd,
name,
_MAX_RECOVERY_SEGMENT_BYTES,
allow_empty=False,
)
if read_result is None:
try:
metadata = os.stat(name, dir_fd=segments_fd, follow_symlinks=False)
except OSError:
orphans.append(name)
continue
if (
not stat.S_ISREG(metadata.st_mode)
or not 0 < metadata.st_size <= _MAX_RECOVERY_SEGMENT_BYTES
):
orphans.append(name)
continue
payload, metadata = read_result
payloads[sequence] = payload
timestamps[sequence] = metadata.st_mtime_ns
return payloads, timestamps, orphans
return timestamps, orphans
def _parse_index_prefix(payload: bytes | None) -> list[dict[str, Any]]:
@@ -908,6 +1009,28 @@ def _read_regular_at(
return result[0] if result is not None else None
def _read_regular_at_current_size(
parent_fd: int,
name: str,
*,
allow_empty: bool,
) -> bytes | None:
"""Read one private recovery artifact without a duration-derived ceiling."""
try:
metadata = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
except OSError:
return None
if not stat.S_ISREG(metadata.st_mode):
return None
return _read_regular_at(
parent_fd,
name,
max(1, metadata.st_size),
allow_empty=allow_empty,
)
def _read_regular_with_metadata_at(
parent_fd: int,
name: str,