feat(k1): integrate durable evidence with acquisition lifecycle

tests/test_xgrids_camera_gateway.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import sys
+import time
 from io import BytesIO
 from pathlib import Path
 
 import pytest
 
+import k1link.web.xgrids_k1_camera as camera_module
 from k1link.web.xgrids_k1_camera import (
     CAMERA_MEDIA_TYPE,
     XgridsK1CameraGateway,
@@ -40,6 +43,63 @@ def _fake_ffmpeg(tmp_path: Path) -> Path:
     return executable
 
 
+def _burst_ffmpeg(tmp_path: Path) -> Path:
+    executable = tmp_path / "burst-ffmpeg"
+    executable.write_text(
+        f"#!{sys.executable}\n"
+        "import sys, time\n"
+        "def box(kind, payload=b''):\n"
+        "    return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n"
+        "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov'))\n"
+        "sys.stdout.buffer.flush()\n"
+        "time.sleep(0.2)\n"
+        "for index in range(10):\n"
+        "    sys.stdout.buffer.write(box(b'moof') + box(b'mdat', bytes([index])))\n"
+        "    sys.stdout.buffer.flush()\n"
+        "    time.sleep(0.01)\n"
+        "time.sleep(10)\n",
+        encoding="utf-8",
+    )
+    executable.chmod(0o700)
+    return executable
+
+
+def _buffered_tail_ffmpeg(tmp_path: Path, sentinel: Path, *, fragments: int) -> Path:
+    executable = tmp_path / "buffered-tail-ffmpeg"
+    executable.write_text(
+        f"#!{sys.executable}\n"
+        "import pathlib, sys, time\n"
+        "def box(kind, payload=b''):\n"
+        "    return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n"
+        "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov'))\n"
+        f"for index in range({fragments}):\n"
+        "    sys.stdout.buffer.write(box(b'moof') + box(b'mdat', index.to_bytes(2, 'big')))\n"
+        "sys.stdout.buffer.flush()\n"
+        f"pathlib.Path({str(sentinel)!r}).write_text('ready')\n"
+        "time.sleep(10)\n",
+        encoding="utf-8",
+    )
+    executable.chmod(0o700)
+    return executable
+
+
+def _incomplete_tail_ffmpeg(tmp_path: Path, sentinel: Path) -> Path:
+    executable = tmp_path / "incomplete-tail-ffmpeg"
+    executable.write_text(
+        f"#!{sys.executable}\n"
+        "import pathlib, sys, time\n"
+        "def box(kind, payload=b''):\n"
+        "    return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n"
+        "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov') + box(b'moof'))\n"
+        "sys.stdout.buffer.flush()\n"
+        f"pathlib.Path({str(sentinel)!r}).write_text('ready')\n"
+        "time.sleep(10)\n",
+        encoding="utf-8",
+    )
+    executable.chmod(0o700)
+    return executable
+
+
 def _gateway(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
@@ -48,6 +108,16 @@ def _gateway(
     return XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID)
 
 
+def _wait_until(predicate: object, *, timeout: float = 3.0) -> None:
+    assert callable(predicate)
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if predicate():
+            return
+        time.sleep(0.01)
+    raise AssertionError("condition was not reached before timeout")
+
+
 def test_camera_selection_is_exclusive_and_hides_device_transport(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
@@ -90,9 +160,7 @@ def test_camera_ffmpeg_command_is_allowlisted_copy_remux() -> None:
     )
 
     assert argv[0] == "/trusted/ffmpeg"
-    assert argv[argv.index("-i") + 1] == (
-        "rtsp://10.0.0.24:8554/live/chn_left_main"
-    )
+    assert argv[argv.index("-i") + 1] == ("rtsp://10.0.0.24:8554/live/chn_left_main")
     assert argv[argv.index("-c:v") + 1] == "copy"
     assert argv[argv.index("-allowed_media_types") + 1] == "video"
     assert argv[argv.index("-flush_packets") + 1] == "1"
@@ -145,6 +213,279 @@ def test_gateway_emits_init_and_complete_media_segments_without_transcoding(
         gateway.close()
 
 
+def test_acquisition_records_without_browser_and_source_switch_seals_epochs(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    gateway = _gateway(tmp_path, monkeypatch)
+    session = tmp_path / "sessions" / "camera-acquisition"
+    try:
+        left = gateway.select("sensor.camera.left", "192.168.1.20")
+        with pytest.raises(ValueError, match="does not exist"):
+            gateway.start_recording(session)
+        assert session.exists() is False
+        session.mkdir(parents=True)
+        gateway.start_recording(session)
+        left_epoch = session / "media" / "sensor.camera.left" / f"epoch-{left['generation']}"
+
+        # No open_delivery/WebSocket exists: acquisition ownership alone starts
+        # FFmpeg and commits the init segment before any preview consumer.
+        _wait_until(
+            lambda: (
+                (left_epoch / "init.mp4").is_file()
+                and len(list((left_epoch / "segments").glob("*.m4s"))) == 1
+            )
+        )
+        _wait_until(lambda: gateway.snapshot()["phase"] == "streaming")
+
+        right = gateway.select("sensor.camera.right", "192.168.1.20")
+        right_epoch = session / "media" / "sensor.camera.right" / f"epoch-{right['generation']}"
+        _wait_until(lambda: gateway.snapshot()["phase"] == "streaming")
+        _wait_until(lambda: len(list((right_epoch / "segments").glob("*.m4s"))) == 1)
+
+        lease = gateway.open_delivery(right["generation"])
+        init_segment = lease.segments.get(timeout=1)
+        assert init_segment is not None and init_segment[0] == "init"
+        gateway.release_delivery(lease, client_closed=True)
+
+        # Browser disposal is not producer disposal while recording is active.
+        assert lease.process.poll() is None
+        assert gateway.snapshot()["recording"]["active"] is True
+        assert gateway.snapshot()["phase"] == "streaming"
+
+        stopped = gateway.stop_recording(status="complete")
+        assert stopped["recording"]["active"] is False
+        assert stopped["recording"]["completed_epochs"] == 2
+
+        summaries = []
+        for epoch in (left_epoch, right_epoch):
+            init = (epoch / "init.mp4").read_bytes()
+            entries = [
+                json.loads(line)
+                for line in (epoch / "index.jsonl").read_text(encoding="utf-8").splitlines()
+            ]
+            summary = json.loads((epoch / "summary.json").read_text(encoding="utf-8"))
+            summaries.append(summary)
+            assert [entry["kind"] for entry in entries] == ["media"]
+            assert all(
+                entry["length"] == len((epoch / entry["path"]).read_bytes())
+                and hashlib.sha256((epoch / entry["path"]).read_bytes()).hexdigest()
+                == entry["sha256"]
+                for entry in entries
+            )
+            assert summary["status"] == "complete"
+            assert summary["segment_count"] == 1
+            assert summary["entry_count"] == 1
+            assert summary["valid_bytes"] == len(init) + sum(
+                entry["length"] for entry in entries
+            )
+
+        assert summaries[0]["failure_code"] == "source-switch"
+        assert summaries[1]["failure_code"] is None
+        serialized = json.dumps(stopped)
+        assert "192.168.1.20" not in serialized
+        assert "rtsp://" not in serialized
+    finally:
+        gateway.close()
+
+
+def test_camera_storage_open_failure_is_loud_and_never_starts_ffmpeg(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    gateway = _gateway(tmp_path, monkeypatch)
+    session = tmp_path / "session"
+    session.mkdir()
+    popen_called = False
+
+    class FailingArchive:
+        def __init__(self, *_: object, **__: object) -> None:
+            raise camera_module.CameraArchiveError("synthetic storage failure")
+
+    def forbidden_popen(*_: object, **__: object) -> object:
+        nonlocal popen_called
+        popen_called = True
+        raise AssertionError("FFmpeg must not start without durable storage")
+
+    monkeypatch.setattr(camera_module, "CameraArchiveWriter", FailingArchive)
+    monkeypatch.setattr(camera_module.subprocess, "Popen", forbidden_popen)
+    try:
+        gateway.select("sensor.camera.left", "192.168.1.20")
+        with pytest.raises(RuntimeError, match="хранилище"):
+            gateway.start_recording(session)
+
+        state = gateway.snapshot()
+        assert popen_called is False
+        assert state["phase"] == "error"
+        assert state["error"]["code"] == "camera-storage-failed"
+        assert "192.168.1.20" not in json.dumps(state)
+    finally:
+        gateway.close()
+
+
+def test_camera_storage_append_failure_fails_producer_and_epoch_loudly(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    original_append = camera_module.CameraArchiveWriter.append
+
+    def failing_append(
+        writer: object,
+        kind: object,
+        payload: bytes,
+        **timestamps: object,
+    ) -> dict[str, object]:
+        if kind == "media":
+            raise camera_module.CameraArchiveError("synthetic media commit failure")
+        return original_append(writer, kind, payload, **timestamps)  # type: ignore[arg-type]
+
+    monkeypatch.setattr(camera_module.CameraArchiveWriter, "append", failing_append)
+    gateway = _gateway(tmp_path, monkeypatch)
+    session = tmp_path / "session"
+    session.mkdir()
+    try:
+        selected = gateway.select("sensor.camera.left", "192.168.1.20")
+        gateway.start_recording(session)
+
+        _wait_until(lambda: gateway.snapshot()["phase"] == "error")
+        state = gateway.snapshot()
+        assert state["error"]["code"] == "camera-storage-failed"
+        summary_path = (
+            session
+            / "media"
+            / "sensor.camera.left"
+            / f"epoch-{selected['generation']}"
+            / "summary.json"
+        )
+        _wait_until(
+            lambda: (
+                summary_path.is_file()
+                and json.loads(summary_path.read_text(encoding="utf-8"))["status"]
+                == "failed"
+            )
+        )
+        summary = json.loads(summary_path.read_text(encoding="utf-8"))
+        assert summary["status"] == "failed"
+        assert summary["failure_code"] == "camera-storage-failed"
+        assert "192.168.1.20" not in json.dumps(state)
+    finally:
+        gateway.close()
+
+
+def test_slow_browser_is_dropped_without_stopping_archive_producer(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(_burst_ffmpeg(tmp_path)))
+    gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID)
+    session = tmp_path / "session"
+    session.mkdir()
+    try:
+        selected = gateway.select("sensor.camera.left", "192.168.1.20")
+        gateway.start_recording(session)
+        lease = gateway.open_delivery(selected["generation"])
+
+        # Deliberately never drain the bounded queue.
+        _wait_until(lambda: lease.failure_code == "consumer-too-slow")
+        _wait_until(lambda: gateway.snapshot()["phase"] == "streaming")
+        assert lease.process.poll() is None
+        assert gateway.snapshot()["recording"]["active"] is True
+
+        gateway.release_delivery(lease, client_closed=False)
+        epoch = (
+            session
+            / "media"
+            / "sensor.camera.left"
+            / f"epoch-{selected['generation']}"
+        )
+        _wait_until(
+            lambda: len(list((epoch / "segments").glob("*.m4s"))) == 10,
+        )
+        gateway.stop_recording(status="complete")
+
+        summary_path = epoch / "summary.json"
+        summary = json.loads(summary_path.read_text(encoding="utf-8"))
+        assert summary["status"] == "complete"
+        assert summary["segment_count"] == 10
+        assert summary["media_segment_count"] == 10
+    finally:
+        gateway.close()
+
+
+def test_clean_stop_drains_ffmpeg_stdout_before_sealing_archive(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    fragment_count = 100
+    sentinel = tmp_path / "ffmpeg-wrote-buffered-tail"
+    monkeypatch.setenv(
+        "MISSIONCORE_FFMPEG_BINARY",
+        str(_buffered_tail_ffmpeg(tmp_path, sentinel, fragments=fragment_count)),
+    )
+    gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID)
+    session = tmp_path / "session"
+    session.mkdir()
+    try:
+        selected = gateway.select("sensor.camera.left", "192.168.1.20")
+        gateway.start_recording(session)
+        _wait_until(sentinel.is_file)
+
+        gateway.stop_recording(status="complete")
+
+        epoch = (
+            session
+            / "media"
+            / "sensor.camera.left"
+            / f"epoch-{selected['generation']}"
+        )
+        summary = json.loads((epoch / "summary.json").read_text(encoding="utf-8"))
+        assert summary["status"] == "complete"
+        assert summary["segment_count"] == fragment_count
+        assert len(list((epoch / "segments").glob("*.m4s"))) == fragment_count
+        assert len((epoch / "index.jsonl").read_text(encoding="utf-8").splitlines()) == (
+            fragment_count
+        )
+    finally:
+        gateway.close()
+
+
+def test_clean_stop_marks_incomplete_fragment_tail_interrupted(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    sentinel = tmp_path / "ffmpeg-wrote-incomplete-tail"
+    monkeypatch.setenv(
+        "MISSIONCORE_FFMPEG_BINARY",
+        str(_incomplete_tail_ffmpeg(tmp_path, sentinel)),
+    )
+    gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID)
+    session = tmp_path / "session"
+    session.mkdir()
+    try:
+        selected = gateway.select("sensor.camera.left", "192.168.1.20")
+        gateway.start_recording(session)
+        _wait_until(sentinel.is_file)
+
+        gateway.stop_recording(status="complete")
+
+        summary_path = (
+            session
+            / "media"
+            / "sensor.camera.left"
+            / f"epoch-{selected['generation']}"
+            / "summary.json"
+        )
+        summary = json.loads(summary_path.read_text(encoding="utf-8"))
+        assert summary["status"] == "interrupted"
+        assert summary["failure_code"] == "incomplete-fmp4-fragment"
+        assert summary["segment_count"] == 0
+        state = gateway.snapshot()
+        assert state["phase"] == "error"
+        assert state["error"]["code"] == "incomplete-fmp4-fragment"
+    finally:
+        gateway.close()
+
+
 def test_service_publishes_two_dynamic_camera_rows_and_stale_stop_is_safe(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,