perf(observatory): pack camera epoch source transfer
This commit is contained in:
@@ -16,11 +16,12 @@ import os
|
|||||||
import re
|
import re
|
||||||
import secrets
|
import secrets
|
||||||
import stat
|
import stat
|
||||||
|
import tarfile
|
||||||
from collections.abc import AsyncIterable, Mapping, Sequence
|
from collections.abc import AsyncIterable, Mapping, Sequence
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Final, Literal
|
from typing import BinaryIO, Final, Literal, cast
|
||||||
|
|
||||||
from k1link.observatory.portable_result_contract import (
|
from k1link.observatory.portable_result_contract import (
|
||||||
OBSERVATION_ONLY_AUTHORITY,
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
@@ -59,6 +60,16 @@ from k1link.sessions.store import SessionStore
|
|||||||
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA: Final = (
|
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA: Final = (
|
||||||
"missioncore.observatory-portable-source-materialization/v1"
|
"missioncore.observatory-portable-source-materialization/v1"
|
||||||
)
|
)
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-camera-epoch-archive/v1"
|
||||||
|
)
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_BINDING_SCHEMA: Final = (
|
||||||
|
"missioncore.observatory-portable-camera-epoch-archive-binding/v1"
|
||||||
|
)
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE: Final = "application/x-tar"
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_ID_HEADER: Final = (
|
||||||
|
"X-Mission-Core-Camera-Epoch-Archive-Id"
|
||||||
|
)
|
||||||
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA: Final = (
|
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA: Final = (
|
||||||
"missioncore.observatory-portable-result-upload-plan/v1"
|
"missioncore.observatory-portable-result-upload-plan/v1"
|
||||||
)
|
)
|
||||||
@@ -73,6 +84,10 @@ PORTABLE_RESULT_STAGING_DIRECTORY: Final = "observatory-worker-result-staging"
|
|||||||
|
|
||||||
MAX_SOURCE_MEMBERS: Final = 100_000
|
MAX_SOURCE_MEMBERS: Final = 100_000
|
||||||
MAX_SOURCE_BYTES: Final = 2 * 1024 * 1024 * 1024 * 1024
|
MAX_SOURCE_BYTES: Final = 2 * 1024 * 1024 * 1024 * 1024
|
||||||
|
MAX_CAMERA_EPOCH_ARCHIVE_BYTES: Final = (
|
||||||
|
MAX_SOURCE_BYTES + MAX_SOURCE_MEMBERS * 1024 + tarfile.RECORDSIZE
|
||||||
|
)
|
||||||
|
MAX_CAMERA_EPOCH_ARCHIVE_BINDING_BYTES: Final = 8 * 1024 * 1024
|
||||||
MAX_RESULT_MEMBER_BYTES: Final = 64 * 1024 * 1024 * 1024
|
MAX_RESULT_MEMBER_BYTES: Final = 64 * 1024 * 1024 * 1024
|
||||||
MAX_RESULT_PACKAGE_BYTES: Final = 256 * 1024 * 1024 * 1024
|
MAX_RESULT_PACKAGE_BYTES: Final = 256 * 1024 * 1024 * 1024
|
||||||
MAX_RESULT_MANIFEST_BYTES: Final = 1024 * 1024
|
MAX_RESULT_MANIFEST_BYTES: Final = 1024 * 1024
|
||||||
@@ -228,6 +243,53 @@ class PortableSourceMaterializationManifest:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PortableCameraEpochArchive:
|
||||||
|
archive_id: str
|
||||||
|
job_identity_sha256: str
|
||||||
|
source_bundle_sha256: str
|
||||||
|
artifact_id: str
|
||||||
|
camera_epoch: int
|
||||||
|
member_ids: tuple[str, ...]
|
||||||
|
byte_length: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
_digest(self.archive_id, "camera epoch archive id")
|
||||||
|
_digest(self.job_identity_sha256, "camera epoch archive job identity")
|
||||||
|
_digest(self.source_bundle_sha256, "camera epoch archive source bundle")
|
||||||
|
_session_id(self.artifact_id)
|
||||||
|
_positive_int(self.camera_epoch, "camera epoch archive ordinal")
|
||||||
|
if (
|
||||||
|
not 2 <= len(self.member_ids) <= MAX_SOURCE_MEMBERS
|
||||||
|
or len(set(self.member_ids)) != len(self.member_ids)
|
||||||
|
):
|
||||||
|
raise ValueError("camera epoch archive members are invalid")
|
||||||
|
for member_id in self.member_ids:
|
||||||
|
_digest(member_id, "camera epoch archive member id")
|
||||||
|
if not 1 <= self.byte_length <= MAX_CAMERA_EPOCH_ARCHIVE_BYTES:
|
||||||
|
raise ValueError("camera epoch archive byte length is invalid")
|
||||||
|
_digest(self.sha256, "camera epoch archive sha256")
|
||||||
|
if self.archive_id != portable_camera_epoch_archive_id(
|
||||||
|
job_identity_sha256=self.job_identity_sha256,
|
||||||
|
source_bundle_sha256=self.source_bundle_sha256,
|
||||||
|
artifact_id=self.artifact_id,
|
||||||
|
camera_epoch=self.camera_epoch,
|
||||||
|
member_ids=self.member_ids,
|
||||||
|
byte_length=self.byte_length,
|
||||||
|
sha256=self.sha256,
|
||||||
|
):
|
||||||
|
raise ValueError("camera epoch archive identity is invalid")
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"archive_id": self.archive_id,
|
||||||
|
"media_type": PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE,
|
||||||
|
"byte_length": self.byte_length,
|
||||||
|
"sha256": self.sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class PortableResultUploadMember:
|
class PortableResultUploadMember:
|
||||||
member_id: str
|
member_id: str
|
||||||
@@ -443,6 +505,39 @@ class PortableObservatoryArtifactTransport:
|
|||||||
)
|
)
|
||||||
return member, destination
|
return member, destination
|
||||||
|
|
||||||
|
def materialize_camera_epoch_archive(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
job_id: str,
|
||||||
|
claim_token: str,
|
||||||
|
claim_generation: int,
|
||||||
|
claimant_id: str,
|
||||||
|
) -> tuple[PortableCameraEpochArchive, Path]:
|
||||||
|
"""Publish one claim-bound deterministic tar for the admitted camera epoch."""
|
||||||
|
|
||||||
|
job = self._authorize(
|
||||||
|
job_id=job_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=claimant_id,
|
||||||
|
allowed_states=("claimed", "running"),
|
||||||
|
)
|
||||||
|
manifest = self._resolve_source_manifest(job)
|
||||||
|
members = _ordered_camera_epoch_members(manifest.members)
|
||||||
|
archive, path = _publish_camera_epoch_archive(
|
||||||
|
self._source_cas,
|
||||||
|
job=job,
|
||||||
|
members=members,
|
||||||
|
)
|
||||||
|
self._authorize(
|
||||||
|
job_id=job_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=claimant_id,
|
||||||
|
allowed_states=("claimed", "running"),
|
||||||
|
)
|
||||||
|
return archive, path
|
||||||
|
|
||||||
def stage_result_manifest(
|
def stage_result_manifest(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -1033,6 +1128,418 @@ class PortableObservatoryArtifactTransport:
|
|||||||
return receipt
|
return receipt
|
||||||
|
|
||||||
|
|
||||||
|
def portable_camera_epoch_archive_id(
|
||||||
|
*,
|
||||||
|
job_identity_sha256: str,
|
||||||
|
source_bundle_sha256: str,
|
||||||
|
artifact_id: str,
|
||||||
|
camera_epoch: int,
|
||||||
|
member_ids: Sequence[str],
|
||||||
|
byte_length: int,
|
||||||
|
sha256: str,
|
||||||
|
) -> str:
|
||||||
|
"""Return the fixed identity of one packed, content-sealed camera epoch."""
|
||||||
|
|
||||||
|
_digest(job_identity_sha256, "camera epoch archive job identity")
|
||||||
|
_digest(source_bundle_sha256, "camera epoch archive source bundle")
|
||||||
|
_session_id(artifact_id)
|
||||||
|
_positive_int(camera_epoch, "camera epoch archive ordinal")
|
||||||
|
canonical_member_ids = tuple(member_ids)
|
||||||
|
if (
|
||||||
|
not 2 <= len(canonical_member_ids) <= MAX_SOURCE_MEMBERS
|
||||||
|
or len(set(canonical_member_ids)) != len(canonical_member_ids)
|
||||||
|
):
|
||||||
|
raise ValueError("camera epoch archive members are invalid")
|
||||||
|
for member_id in canonical_member_ids:
|
||||||
|
_digest(member_id, "camera epoch archive member id")
|
||||||
|
if not 1 <= byte_length <= MAX_CAMERA_EPOCH_ARCHIVE_BYTES:
|
||||||
|
raise ValueError("camera epoch archive byte length is invalid")
|
||||||
|
_digest(sha256, "camera epoch archive sha256")
|
||||||
|
return hashlib.sha256(
|
||||||
|
canonical_json(
|
||||||
|
{
|
||||||
|
"schema_version": PORTABLE_CAMERA_EPOCH_ARCHIVE_SCHEMA,
|
||||||
|
"job_identity_sha256": job_identity_sha256,
|
||||||
|
"source_bundle_sha256": source_bundle_sha256,
|
||||||
|
"artifact_id": artifact_id,
|
||||||
|
"camera_epoch": camera_epoch,
|
||||||
|
"member_ids": list(canonical_member_ids),
|
||||||
|
"media_type": PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE,
|
||||||
|
"byte_length": byte_length,
|
||||||
|
"sha256": sha256,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _ordered_camera_epoch_members(
|
||||||
|
members: Sequence[PortableSourceMember],
|
||||||
|
) -> tuple[PortableSourceMember, ...]:
|
||||||
|
inits = tuple(member for member in members if member.kind == "camera-init")
|
||||||
|
segments = tuple(member for member in members if member.kind == "camera-segment")
|
||||||
|
if len(inits) != 1 or not segments:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera epoch member inventory is incomplete"
|
||||||
|
)
|
||||||
|
init = inits[0]
|
||||||
|
if init.artifact_id is None or init.camera_epoch is None:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera epoch identity is incomplete"
|
||||||
|
)
|
||||||
|
ordered_segments = tuple(
|
||||||
|
sorted(segments, key=lambda member: member.camera_sequence or 0)
|
||||||
|
)
|
||||||
|
if tuple(member.camera_sequence for member in ordered_segments) != tuple(
|
||||||
|
range(1, len(ordered_segments) + 1)
|
||||||
|
) or any(
|
||||||
|
member.artifact_id != init.artifact_id
|
||||||
|
or member.camera_epoch != init.camera_epoch
|
||||||
|
for member in ordered_segments
|
||||||
|
):
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera epoch member order changed"
|
||||||
|
)
|
||||||
|
return (init, *ordered_segments)
|
||||||
|
|
||||||
|
|
||||||
|
def _camera_archive_relative_name(member: PortableSourceMember) -> str:
|
||||||
|
if member.kind == "camera-init":
|
||||||
|
return "init.mp4"
|
||||||
|
if member.kind != "camera-segment" or member.camera_sequence is None:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera archive member role is invalid"
|
||||||
|
)
|
||||||
|
return f"segments/{member.camera_sequence}.m4s"
|
||||||
|
|
||||||
|
|
||||||
|
def _camera_epoch_archive_byte_length(
|
||||||
|
members: Sequence[PortableSourceMember],
|
||||||
|
) -> int:
|
||||||
|
content_bytes = sum(
|
||||||
|
512 + ((member.byte_length + 511) // 512) * 512 for member in members
|
||||||
|
)
|
||||||
|
logical_bytes = content_bytes + 1024
|
||||||
|
return ((logical_bytes + tarfile.RECORDSIZE - 1) // tarfile.RECORDSIZE) * (
|
||||||
|
tarfile.RECORDSIZE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _camera_epoch_archive_binding_identity(
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
members: Sequence[PortableSourceMember],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
ordered = _ordered_camera_epoch_members(members)
|
||||||
|
init = ordered[0]
|
||||||
|
assert init.artifact_id is not None
|
||||||
|
assert init.camera_epoch is not None
|
||||||
|
return {
|
||||||
|
"schema_version": PORTABLE_CAMERA_EPOCH_ARCHIVE_BINDING_SCHEMA,
|
||||||
|
"job_identity_sha256": job.identity_sha256,
|
||||||
|
"source_bundle_sha256": job.source_bundle_sha256,
|
||||||
|
"artifact_id": init.artifact_id,
|
||||||
|
"camera_epoch": init.camera_epoch,
|
||||||
|
"member_ids": [member.member_id for member in ordered],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _load_camera_epoch_archive_binding(
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
binding_id: str,
|
||||||
|
identity: Mapping[str, object],
|
||||||
|
archive_root: Path,
|
||||||
|
) -> tuple[PortableCameraEpochArchive, Path]:
|
||||||
|
try:
|
||||||
|
payload = _read_exact_regular_file(
|
||||||
|
path,
|
||||||
|
maximum_bytes=MAX_CAMERA_EPOCH_ARCHIVE_BINDING_BYTES,
|
||||||
|
)
|
||||||
|
document = object_document(
|
||||||
|
json.loads(payload.decode("utf-8")),
|
||||||
|
"camera epoch archive binding",
|
||||||
|
)
|
||||||
|
if canonical_json(document) != payload:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"camera epoch archive binding is not canonical"
|
||||||
|
)
|
||||||
|
except PortableArtifactTransportError:
|
||||||
|
raise
|
||||||
|
except (
|
||||||
|
OSError,
|
||||||
|
UnicodeDecodeError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
PortableResultPackageIntegrityError,
|
||||||
|
ValueError,
|
||||||
|
) as exc:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"camera epoch archive binding is unreadable"
|
||||||
|
) from exc
|
||||||
|
if (
|
||||||
|
set(document) != {*identity, "binding_id", "archive", "authority"}
|
||||||
|
or any(document.get(key) != value for key, value in identity.items())
|
||||||
|
or document.get("binding_id") != binding_id
|
||||||
|
or document.get("authority") != OBSERVATION_ONLY_AUTHORITY
|
||||||
|
):
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"camera epoch archive binding identity changed"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
archive_document = object_document(
|
||||||
|
document.get("archive"),
|
||||||
|
"camera epoch archive",
|
||||||
|
)
|
||||||
|
except PortableResultPackageIntegrityError as exc:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"camera epoch archive seal is invalid"
|
||||||
|
) from exc
|
||||||
|
if (
|
||||||
|
set(archive_document)
|
||||||
|
!= {"archive_id", "media_type", "byte_length", "sha256"}
|
||||||
|
or archive_document.get("media_type")
|
||||||
|
!= PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE
|
||||||
|
):
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"camera epoch archive seal changed"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
member_ids_value = identity["member_ids"]
|
||||||
|
if not isinstance(member_ids_value, list) or any(
|
||||||
|
not isinstance(member_id, str) for member_id in member_ids_value
|
||||||
|
):
|
||||||
|
raise ValueError("camera epoch archive member ids are invalid")
|
||||||
|
result = PortableCameraEpochArchive(
|
||||||
|
archive_id=string(archive_document.get("archive_id"), "archive id"),
|
||||||
|
job_identity_sha256=string(
|
||||||
|
identity["job_identity_sha256"],
|
||||||
|
"archive job identity",
|
||||||
|
),
|
||||||
|
source_bundle_sha256=string(
|
||||||
|
identity["source_bundle_sha256"],
|
||||||
|
"archive source bundle",
|
||||||
|
),
|
||||||
|
artifact_id=string(identity["artifact_id"], "archive artifact id"),
|
||||||
|
camera_epoch=_positive_int(
|
||||||
|
identity["camera_epoch"],
|
||||||
|
"archive camera epoch",
|
||||||
|
),
|
||||||
|
member_ids=tuple(member_ids_value),
|
||||||
|
byte_length=_positive_int(
|
||||||
|
archive_document.get("byte_length"),
|
||||||
|
"archive byte length",
|
||||||
|
),
|
||||||
|
sha256=string(archive_document.get("sha256"), "archive sha256"),
|
||||||
|
)
|
||||||
|
except (PortableResultPackageIntegrityError, ValueError) as exc:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"camera epoch archive binding is invalid"
|
||||||
|
) from exc
|
||||||
|
archive_path = archive_root / result.sha256[:2] / result.sha256
|
||||||
|
if not _matches_exact_file(
|
||||||
|
archive_path,
|
||||||
|
expected_sha256=result.sha256,
|
||||||
|
expected_byte_length=result.byte_length,
|
||||||
|
):
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"camera epoch archive differs from its immutable binding"
|
||||||
|
)
|
||||||
|
return result, archive_path
|
||||||
|
|
||||||
|
|
||||||
|
def _publish_camera_epoch_archive(
|
||||||
|
root: Path,
|
||||||
|
*,
|
||||||
|
job: ObservatoryRecordedJob,
|
||||||
|
members: Sequence[PortableSourceMember],
|
||||||
|
) -> tuple[PortableCameraEpochArchive, Path]:
|
||||||
|
ordered = _ordered_camera_epoch_members(members)
|
||||||
|
init = ordered[0]
|
||||||
|
assert init.artifact_id is not None
|
||||||
|
assert init.camera_epoch is not None
|
||||||
|
archive_root = _prepare_secure_root(root / "camera-epoch-archives")
|
||||||
|
binding_identity = _camera_epoch_archive_binding_identity(job, ordered)
|
||||||
|
binding_id = hashlib.sha256(canonical_json(binding_identity)).hexdigest()
|
||||||
|
binding_root = _prepare_secure_root(archive_root / "bindings")
|
||||||
|
binding_path = binding_root / f"{binding_id}.json"
|
||||||
|
if binding_path.exists():
|
||||||
|
return _load_camera_epoch_archive_binding(
|
||||||
|
binding_path,
|
||||||
|
binding_id=binding_id,
|
||||||
|
identity=binding_identity,
|
||||||
|
archive_root=archive_root,
|
||||||
|
)
|
||||||
|
for member in ordered:
|
||||||
|
if member._source_path is None:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera archive member has no admitted source"
|
||||||
|
)
|
||||||
|
temporary = archive_root / f".tmp-{secrets.token_hex(16)}"
|
||||||
|
descriptor = os.open(
|
||||||
|
temporary,
|
||||||
|
os.O_WRONLY
|
||||||
|
| os.O_CREAT
|
||||||
|
| os.O_EXCL
|
||||||
|
| getattr(os, "O_NOFOLLOW", 0),
|
||||||
|
0o600,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with os.fdopen(descriptor, "wb") as stream:
|
||||||
|
descriptor = -1
|
||||||
|
with tarfile.open(
|
||||||
|
fileobj=stream,
|
||||||
|
mode="w",
|
||||||
|
format=tarfile.USTAR_FORMAT,
|
||||||
|
) as archive:
|
||||||
|
for member in ordered:
|
||||||
|
info = tarfile.TarInfo(_camera_archive_relative_name(member))
|
||||||
|
info.size = member.byte_length
|
||||||
|
info.mode = 0o600
|
||||||
|
info.mtime = 0
|
||||||
|
info.uid = 0
|
||||||
|
info.gid = 0
|
||||||
|
info.uname = ""
|
||||||
|
info.gname = ""
|
||||||
|
assert member._source_path is not None
|
||||||
|
_add_exact_camera_archive_member(
|
||||||
|
archive,
|
||||||
|
info,
|
||||||
|
member._source_path,
|
||||||
|
expected_sha256=member.sha256,
|
||||||
|
expected_byte_length=member.byte_length,
|
||||||
|
)
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
sha256, byte_length = _hash_regular_file(
|
||||||
|
temporary,
|
||||||
|
maximum_bytes=MAX_CAMERA_EPOCH_ARCHIVE_BYTES,
|
||||||
|
)
|
||||||
|
if byte_length != _camera_epoch_archive_byte_length(ordered):
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera epoch archive size is not deterministic"
|
||||||
|
)
|
||||||
|
result = PortableCameraEpochArchive(
|
||||||
|
archive_id=portable_camera_epoch_archive_id(
|
||||||
|
job_identity_sha256=job.identity_sha256,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
artifact_id=init.artifact_id,
|
||||||
|
camera_epoch=init.camera_epoch,
|
||||||
|
member_ids=tuple(member.member_id for member in ordered),
|
||||||
|
byte_length=byte_length,
|
||||||
|
sha256=sha256,
|
||||||
|
),
|
||||||
|
job_identity_sha256=job.identity_sha256,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
artifact_id=init.artifact_id,
|
||||||
|
camera_epoch=init.camera_epoch,
|
||||||
|
member_ids=tuple(member.member_id for member in ordered),
|
||||||
|
byte_length=byte_length,
|
||||||
|
sha256=sha256,
|
||||||
|
)
|
||||||
|
destination_root = _prepare_secure_root(
|
||||||
|
archive_root / result.sha256[:2]
|
||||||
|
)
|
||||||
|
destination = destination_root / result.sha256
|
||||||
|
_publish_uploaded_file(
|
||||||
|
temporary,
|
||||||
|
destination,
|
||||||
|
expected_sha256=result.sha256,
|
||||||
|
expected_byte_length=result.byte_length,
|
||||||
|
)
|
||||||
|
_write_immutable_file(
|
||||||
|
binding_path,
|
||||||
|
canonical_json(
|
||||||
|
{
|
||||||
|
**binding_identity,
|
||||||
|
"binding_id": binding_id,
|
||||||
|
"archive": result.as_dict(),
|
||||||
|
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return result, destination
|
||||||
|
except PortableArtifactTransportError:
|
||||||
|
raise
|
||||||
|
except (OSError, tarfile.TarError, ValueError) as exc:
|
||||||
|
raise PortableArtifactTransportUnavailableError(
|
||||||
|
"portable camera epoch archive could not be materialized"
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
if descriptor >= 0:
|
||||||
|
os.close(descriptor)
|
||||||
|
with suppress(FileNotFoundError):
|
||||||
|
temporary.unlink()
|
||||||
|
|
||||||
|
|
||||||
|
class _ExactArchiveSource:
|
||||||
|
def __init__(self, stream: BinaryIO, *, expected_byte_length: int) -> None:
|
||||||
|
self._stream = stream
|
||||||
|
self._expected_byte_length = expected_byte_length
|
||||||
|
self._digest = hashlib.sha256()
|
||||||
|
self.byte_length = 0
|
||||||
|
|
||||||
|
def read(self, size: int = -1) -> bytes:
|
||||||
|
payload = self._stream.read(size)
|
||||||
|
self.byte_length += len(payload)
|
||||||
|
if self.byte_length > self._expected_byte_length:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera archive source grew while read"
|
||||||
|
)
|
||||||
|
if not payload and self.byte_length != self._expected_byte_length:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera archive source ended early"
|
||||||
|
)
|
||||||
|
self._digest.update(payload)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sha256(self) -> str:
|
||||||
|
return self._digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _add_exact_camera_archive_member(
|
||||||
|
archive: tarfile.TarFile,
|
||||||
|
info: tarfile.TarInfo,
|
||||||
|
source: Path,
|
||||||
|
*,
|
||||||
|
expected_sha256: str,
|
||||||
|
expected_byte_length: int,
|
||||||
|
) -> None:
|
||||||
|
descriptor = os.open(
|
||||||
|
source,
|
||||||
|
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
||||||
|
)
|
||||||
|
with os.fdopen(descriptor, "rb") as source_stream:
|
||||||
|
before = os.fstat(source_stream.fileno())
|
||||||
|
if not stat.S_ISREG(before.st_mode) or before.st_size != expected_byte_length:
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera archive source is not the admitted regular file"
|
||||||
|
)
|
||||||
|
verified = _ExactArchiveSource(
|
||||||
|
source_stream,
|
||||||
|
expected_byte_length=expected_byte_length,
|
||||||
|
)
|
||||||
|
archive.addfile(info, cast(BinaryIO, verified))
|
||||||
|
after = os.fstat(source_stream.fileno())
|
||||||
|
stable = (
|
||||||
|
before.st_dev,
|
||||||
|
before.st_ino,
|
||||||
|
before.st_size,
|
||||||
|
before.st_mtime_ns,
|
||||||
|
) == (
|
||||||
|
after.st_dev,
|
||||||
|
after.st_ino,
|
||||||
|
after.st_size,
|
||||||
|
after.st_mtime_ns,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
verified.byte_length != expected_byte_length
|
||||||
|
or verified.sha256 != expected_sha256
|
||||||
|
or not stable
|
||||||
|
):
|
||||||
|
raise PortableArtifactTransportIntegrityError(
|
||||||
|
"portable camera archive source changed while read"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _source_member(
|
def _source_member(
|
||||||
job: ObservatoryRecordedJob,
|
job: ObservatoryRecordedJob,
|
||||||
*,
|
*,
|
||||||
|
|||||||
@@ -14,21 +14,28 @@ import json
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import secrets
|
import secrets
|
||||||
|
import shutil
|
||||||
import stat
|
import stat
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Final, Literal, cast
|
from typing import BinaryIO, Final, Literal, cast
|
||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from k1link.observatory.portable_artifact_transport import (
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
MAX_CAMERA_EPOCH_ARCHIVE_BYTES,
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_ID_HEADER,
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE,
|
||||||
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
||||||
PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
||||||
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
||||||
|
portable_camera_epoch_archive_id,
|
||||||
)
|
)
|
||||||
from k1link.observatory.portable_result_contract import (
|
from k1link.observatory.portable_result_contract import (
|
||||||
OBSERVATION_ONLY_AUTHORITY,
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
@@ -54,6 +61,7 @@ from k1link.observatory.worker_agent import (
|
|||||||
WORKER_HTTP_MAX_JSON_BYTES: Final = 16 * 1024 * 1024
|
WORKER_HTTP_MAX_JSON_BYTES: Final = 16 * 1024 * 1024
|
||||||
WORKER_HTTP_COPY_CHUNK_BYTES: Final = 1024 * 1024
|
WORKER_HTTP_COPY_CHUNK_BYTES: Final = 1024 * 1024
|
||||||
WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS: Final = 30.0
|
WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS: Final = 30.0
|
||||||
|
WORKER_HTTP_CAMERA_ARCHIVE_READ_TIMEOUT_SECONDS: Final = 30 * 60.0
|
||||||
|
|
||||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||||
_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$")
|
_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$")
|
||||||
@@ -136,6 +144,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
_CONTOUR_HEADER: contour_id,
|
_CONTOUR_HEADER: contour_id,
|
||||||
"Accept": "application/json",
|
"Accept": "application/json",
|
||||||
}
|
}
|
||||||
|
self._timeout_seconds = timeout_seconds
|
||||||
self._work_root = _secure_directory(work_root)
|
self._work_root = _secure_directory(work_root)
|
||||||
self._client = httpx.Client(
|
self._client = httpx.Client(
|
||||||
base_url=self._base_url,
|
base_url=self._base_url,
|
||||||
@@ -304,6 +313,21 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
"source materialization members select the same local role"
|
"source materialization members select the same local role"
|
||||||
)
|
)
|
||||||
destinations[destination] = member
|
destinations[destination] = member
|
||||||
|
camera_members = _ordered_camera_epoch_members(members)
|
||||||
|
if not all(
|
||||||
|
_matches_file(
|
||||||
|
_source_destination(root, member),
|
||||||
|
member.sha256,
|
||||||
|
member.byte_length,
|
||||||
|
)
|
||||||
|
for member in camera_members
|
||||||
|
):
|
||||||
|
self._download_camera_epoch_archive(
|
||||||
|
job=job,
|
||||||
|
context=context,
|
||||||
|
members=camera_members,
|
||||||
|
root=root,
|
||||||
|
)
|
||||||
for destination, member in destinations.items():
|
for destination, member in destinations.items():
|
||||||
if _matches_file(destination, member.sha256, member.byte_length):
|
if _matches_file(destination, member.sha256, member.byte_length):
|
||||||
continue
|
continue
|
||||||
@@ -463,6 +487,123 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
|||||||
result_sha256=draft.result_sha256,
|
result_sha256=draft.result_sha256,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _download_camera_epoch_archive(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
context: _ClaimContext,
|
||||||
|
members: tuple[_SourceMember, ...],
|
||||||
|
root: Path,
|
||||||
|
) -> None:
|
||||||
|
transfer_root = _secure_directory(self._work_root / ".source-transfers")
|
||||||
|
temporary = transfer_root / f".camera-epoch-{secrets.token_hex(16)}.tar"
|
||||||
|
descriptor = -1
|
||||||
|
try:
|
||||||
|
with self._client.stream(
|
||||||
|
"GET",
|
||||||
|
self._job_path(job.job_id, "source-camera-epoch-archive"),
|
||||||
|
headers=self._claim_headers(context),
|
||||||
|
timeout=httpx.Timeout(
|
||||||
|
connect=self._timeout_seconds,
|
||||||
|
read=WORKER_HTTP_CAMERA_ARCHIVE_READ_TIMEOUT_SECONDS,
|
||||||
|
write=self._timeout_seconds,
|
||||||
|
pool=self._timeout_seconds,
|
||||||
|
),
|
||||||
|
) as response:
|
||||||
|
if response.status_code == 404:
|
||||||
|
return
|
||||||
|
self._raise_for_status(response)
|
||||||
|
if (
|
||||||
|
response.headers.get("content-type")
|
||||||
|
!= PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive media type changed"
|
||||||
|
)
|
||||||
|
declared = response.headers.get("content-length")
|
||||||
|
if (
|
||||||
|
declared is None
|
||||||
|
or not declared.isascii()
|
||||||
|
or not declared.isdigit()
|
||||||
|
or len(declared) > len(str(MAX_CAMERA_EPOCH_ARCHIVE_BYTES))
|
||||||
|
or str(int(declared)) != declared
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive length is invalid"
|
||||||
|
)
|
||||||
|
expected_bytes = int(declared)
|
||||||
|
expected_sha256 = response.headers.get(_CONTENT_SHA_HEADER, "")
|
||||||
|
archive_id = response.headers.get(
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_ID_HEADER,
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not 1 <= expected_bytes <= MAX_CAMERA_EPOCH_ARCHIVE_BYTES
|
||||||
|
or _SHA256.fullmatch(expected_sha256) is None
|
||||||
|
or _SHA256.fullmatch(archive_id) is None
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive seal is invalid"
|
||||||
|
)
|
||||||
|
descriptor = os.open(
|
||||||
|
temporary,
|
||||||
|
os.O_WRONLY
|
||||||
|
| os.O_CREAT
|
||||||
|
| os.O_EXCL
|
||||||
|
| getattr(os, "O_NOFOLLOW", 0),
|
||||||
|
0o600,
|
||||||
|
)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
byte_length = 0
|
||||||
|
with os.fdopen(descriptor, "wb") as stream:
|
||||||
|
descriptor = -1
|
||||||
|
for chunk in response.iter_bytes(WORKER_HTTP_COPY_CHUNK_BYTES):
|
||||||
|
byte_length += len(chunk)
|
||||||
|
if byte_length > expected_bytes:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive exceeds its declared length"
|
||||||
|
)
|
||||||
|
digest.update(chunk)
|
||||||
|
stream.write(chunk)
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
if byte_length != expected_bytes or digest.hexdigest() != expected_sha256:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive content differs from its seal"
|
||||||
|
)
|
||||||
|
init = members[0]
|
||||||
|
assert init.artifact_id is not None
|
||||||
|
assert init.camera_epoch is not None
|
||||||
|
expected_archive_id = portable_camera_epoch_archive_id(
|
||||||
|
job_identity_sha256=job.identity_sha256,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
artifact_id=init.artifact_id,
|
||||||
|
camera_epoch=init.camera_epoch,
|
||||||
|
member_ids=tuple(member.member_id for member in members),
|
||||||
|
byte_length=byte_length,
|
||||||
|
sha256=expected_sha256,
|
||||||
|
)
|
||||||
|
if archive_id != expected_archive_id:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive identity changed"
|
||||||
|
)
|
||||||
|
_extract_camera_epoch_archive(
|
||||||
|
temporary,
|
||||||
|
root=root,
|
||||||
|
members=members,
|
||||||
|
)
|
||||||
|
except ObservatoryWorkerHttpError:
|
||||||
|
raise
|
||||||
|
except httpx.HTTPError as exc:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive transport is unavailable"
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
if descriptor >= 0:
|
||||||
|
os.close(descriptor)
|
||||||
|
with suppress(FileNotFoundError):
|
||||||
|
temporary.unlink()
|
||||||
|
|
||||||
def _download_member(
|
def _download_member(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -868,6 +1009,192 @@ def _source_member(value: object) -> _SourceMember:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ordered_camera_epoch_members(
|
||||||
|
members: tuple[_SourceMember, ...],
|
||||||
|
) -> tuple[_SourceMember, ...]:
|
||||||
|
inits = tuple(member for member in members if member.kind == "camera-init")
|
||||||
|
segments = tuple(member for member in members if member.kind == "camera-segment")
|
||||||
|
if len(inits) != 1 or not segments:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"source materialization camera epoch is incomplete"
|
||||||
|
)
|
||||||
|
init = inits[0]
|
||||||
|
if init.artifact_id is None or init.camera_epoch is None:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"source materialization camera identity is incomplete"
|
||||||
|
)
|
||||||
|
ordered_segments = tuple(
|
||||||
|
sorted(segments, key=lambda member: member.camera_sequence or 0)
|
||||||
|
)
|
||||||
|
if tuple(member.camera_sequence for member in ordered_segments) != tuple(
|
||||||
|
range(1, len(ordered_segments) + 1)
|
||||||
|
) or any(
|
||||||
|
member.artifact_id != init.artifact_id
|
||||||
|
or member.camera_epoch != init.camera_epoch
|
||||||
|
for member in ordered_segments
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"source materialization camera member order changed"
|
||||||
|
)
|
||||||
|
return (init, *ordered_segments)
|
||||||
|
|
||||||
|
|
||||||
|
def _camera_archive_relative_name(member: _SourceMember) -> str:
|
||||||
|
if member.kind == "camera-init":
|
||||||
|
return "init.mp4"
|
||||||
|
if member.kind != "camera-segment" or member.camera_sequence is None:
|
||||||
|
raise ObservatoryWorkerHttpError("camera archive member role is invalid")
|
||||||
|
return f"segments/{member.camera_sequence}.m4s"
|
||||||
|
|
||||||
|
|
||||||
|
def _camera_epoch_archive_byte_length(members: tuple[_SourceMember, ...]) -> int:
|
||||||
|
content_bytes = sum(
|
||||||
|
512 + ((member.byte_length + 511) // 512) * 512 for member in members
|
||||||
|
)
|
||||||
|
logical_bytes = content_bytes + 1024
|
||||||
|
return ((logical_bytes + tarfile.RECORDSIZE - 1) // tarfile.RECORDSIZE) * (
|
||||||
|
tarfile.RECORDSIZE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_camera_epoch_archive(
|
||||||
|
archive_path: Path,
|
||||||
|
*,
|
||||||
|
root: Path,
|
||||||
|
members: tuple[_SourceMember, ...],
|
||||||
|
) -> None:
|
||||||
|
expected_bytes = _camera_epoch_archive_byte_length(members)
|
||||||
|
try:
|
||||||
|
archive_metadata = archive_path.lstat()
|
||||||
|
except OSError as exc:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive is unavailable"
|
||||||
|
) from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(archive_metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(archive_metadata.st_mode)
|
||||||
|
or archive_metadata.st_size != expected_bytes
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive size differs from its member inventory"
|
||||||
|
)
|
||||||
|
transfer_root = _secure_directory(archive_path.parent)
|
||||||
|
staging = Path(
|
||||||
|
tempfile.mkdtemp(prefix=".camera-epoch-extract-", dir=transfer_root)
|
||||||
|
)
|
||||||
|
staged: list[Path] = []
|
||||||
|
archive_descriptor = -1
|
||||||
|
try:
|
||||||
|
archive_descriptor = os.open(
|
||||||
|
archive_path,
|
||||||
|
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
||||||
|
)
|
||||||
|
with os.fdopen(archive_descriptor, "rb") as archive_stream:
|
||||||
|
archive_descriptor = -1
|
||||||
|
with tarfile.open(fileobj=archive_stream, mode="r:") as archive:
|
||||||
|
for member in members:
|
||||||
|
info = archive.next()
|
||||||
|
if info is None:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive member inventory changed"
|
||||||
|
)
|
||||||
|
relative_name = _camera_archive_relative_name(member)
|
||||||
|
if (
|
||||||
|
info.name != relative_name
|
||||||
|
or info.type != tarfile.REGTYPE
|
||||||
|
or info.size != member.byte_length
|
||||||
|
or info.mode != 0o600
|
||||||
|
or info.mtime != 0
|
||||||
|
or info.uid != 0
|
||||||
|
or info.gid != 0
|
||||||
|
or info.uname != ""
|
||||||
|
or info.gname != ""
|
||||||
|
or info.linkname != ""
|
||||||
|
or info.pax_headers
|
||||||
|
):
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive contains an unsafe member"
|
||||||
|
)
|
||||||
|
source = archive.extractfile(info)
|
||||||
|
if source is None:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive member is unreadable"
|
||||||
|
)
|
||||||
|
target = staging.joinpath(*relative_name.split("/"))
|
||||||
|
_secure_directory(target.parent)
|
||||||
|
with source:
|
||||||
|
_copy_camera_archive_member(
|
||||||
|
cast(BinaryIO, source),
|
||||||
|
target,
|
||||||
|
expected_sha256=member.sha256,
|
||||||
|
expected_byte_length=member.byte_length,
|
||||||
|
)
|
||||||
|
staged.append(target)
|
||||||
|
if archive.next() is not None:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive member inventory changed"
|
||||||
|
)
|
||||||
|
for temporary, member in zip(staged, members, strict=True):
|
||||||
|
_publish_local_file(
|
||||||
|
temporary,
|
||||||
|
_source_destination(root, member),
|
||||||
|
member.sha256,
|
||||||
|
member.byte_length,
|
||||||
|
)
|
||||||
|
except ObservatoryWorkerHttpError:
|
||||||
|
raise
|
||||||
|
except (OSError, tarfile.TarError) as exc:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive could not be extracted"
|
||||||
|
) from exc
|
||||||
|
finally:
|
||||||
|
if archive_descriptor >= 0:
|
||||||
|
os.close(archive_descriptor)
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_camera_archive_member(
|
||||||
|
source: BinaryIO,
|
||||||
|
destination: Path,
|
||||||
|
*,
|
||||||
|
expected_sha256: str,
|
||||||
|
expected_byte_length: int,
|
||||||
|
) -> None:
|
||||||
|
descriptor = os.open(
|
||||||
|
destination,
|
||||||
|
os.O_WRONLY
|
||||||
|
| os.O_CREAT
|
||||||
|
| os.O_EXCL
|
||||||
|
| getattr(os, "O_NOFOLLOW", 0),
|
||||||
|
0o600,
|
||||||
|
)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
byte_length = 0
|
||||||
|
try:
|
||||||
|
with os.fdopen(descriptor, "wb") as stream:
|
||||||
|
descriptor = -1
|
||||||
|
while chunk := source.read(WORKER_HTTP_COPY_CHUNK_BYTES):
|
||||||
|
byte_length += len(chunk)
|
||||||
|
if byte_length > expected_byte_length:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive member exceeds its declared length"
|
||||||
|
)
|
||||||
|
digest.update(chunk)
|
||||||
|
stream.write(chunk)
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
if byte_length != expected_byte_length or digest.hexdigest() != expected_sha256:
|
||||||
|
raise ObservatoryWorkerHttpError(
|
||||||
|
"camera epoch archive member content changed"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if descriptor >= 0:
|
||||||
|
os.close(descriptor)
|
||||||
|
if not _matches_file(destination, expected_sha256, expected_byte_length):
|
||||||
|
with suppress(OSError):
|
||||||
|
destination.unlink()
|
||||||
|
|
||||||
|
|
||||||
def _source_destination(root: Path, member: _SourceMember) -> Path:
|
def _source_destination(root: Path, member: _SourceMember) -> Path:
|
||||||
if member.kind == "source-bundle":
|
if member.kind == "source-bundle":
|
||||||
return root / "source-bundle.json"
|
return root / "source-bundle.json"
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ from pydantic import BaseModel, ConfigDict, Field
|
|||||||
|
|
||||||
from k1link.observatory.portable_artifact_transport import (
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
MAX_RESULT_MANIFEST_BYTES,
|
MAX_RESULT_MANIFEST_BYTES,
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_ID_HEADER,
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE,
|
||||||
PortableArtifactTransportError,
|
PortableArtifactTransportError,
|
||||||
PortableArtifactTransportIntegrityError,
|
PortableArtifactTransportIntegrityError,
|
||||||
PortableArtifactTransportUnavailableError,
|
PortableArtifactTransportUnavailableError,
|
||||||
@@ -381,6 +383,41 @@ def build_observatory_worker_router(
|
|||||||
.as_dict()
|
.as_dict()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@router.get("/recorded-jobs/{job_id}/source-camera-epoch-archive")
|
||||||
|
def source_camera_epoch_archive(
|
||||||
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
claim_token: Annotated[
|
||||||
|
str,
|
||||||
|
Header(
|
||||||
|
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||||
|
pattern=_CLAIM_TOKEN_PATTERN,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
claim_generation: Annotated[
|
||||||
|
int,
|
||||||
|
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||||
|
],
|
||||||
|
) -> FileResponse:
|
||||||
|
archive, path = _artifact_call(
|
||||||
|
lambda: artifact_transport.materialize_camera_epoch_archive(
|
||||||
|
job_id=job_id,
|
||||||
|
claim_token=claim_token,
|
||||||
|
claim_generation=claim_generation,
|
||||||
|
claimant_id=authentication.contour_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return FileResponse(
|
||||||
|
path,
|
||||||
|
media_type=PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE,
|
||||||
|
headers={
|
||||||
|
"ETag": f'"{archive.sha256}"',
|
||||||
|
"Cache-Control": "private, no-store",
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
"X-Mission-Core-Content-Sha256": archive.sha256,
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_ID_HEADER: archive.archive_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
@router.get("/recorded-jobs/{job_id}/source-members/{member_id}")
|
@router.get("/recorded-jobs/{job_id}/source-members/{member_id}")
|
||||||
def source_member(
|
def source_member(
|
||||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import asyncio
|
|||||||
import copy
|
import copy
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import tarfile
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import cast
|
from typing import cast
|
||||||
@@ -484,6 +485,51 @@ def test_source_member_rejects_tampering_and_stale_generation(tmp_path: Path) ->
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_camera_epoch_archive_is_sealed_deterministic_and_cached(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
fixture = _fixture(tmp_path)
|
||||||
|
|
||||||
|
archive, archive_path = fixture.service.materialize_camera_epoch_archive(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert archive_path.name == archive.sha256
|
||||||
|
assert archive_path.stat().st_size == archive.byte_length
|
||||||
|
assert hashlib.sha256(archive_path.read_bytes()).hexdigest() == archive.sha256
|
||||||
|
with tarfile.open(archive_path, mode="r:") as packed:
|
||||||
|
members = packed.getmembers()
|
||||||
|
assert [member.name for member in members] == ["init.mp4", "segments/1.m4s"]
|
||||||
|
assert all(
|
||||||
|
member.type == tarfile.REGTYPE
|
||||||
|
and member.mode == 0o600
|
||||||
|
and member.mtime == 0
|
||||||
|
and member.uid == 0
|
||||||
|
and member.gid == 0
|
||||||
|
and member.uname == ""
|
||||||
|
and member.gname == ""
|
||||||
|
for member in members
|
||||||
|
)
|
||||||
|
|
||||||
|
def reject_rebuild(*_args: object, **_kwargs: object) -> object:
|
||||||
|
pytest.fail("immutable camera epoch archive must be reused")
|
||||||
|
|
||||||
|
monkeypatch.setattr(tarfile, "open", reject_rebuild)
|
||||||
|
repeated, repeated_path = fixture.service.materialize_camera_epoch_archive(
|
||||||
|
job_id=fixture.job.job_id,
|
||||||
|
claim_token=fixture.claim_token,
|
||||||
|
claim_generation=fixture.job.claim_generation,
|
||||||
|
claimant_id="worker-006",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repeated == archive
|
||||||
|
assert repeated_path == archive_path
|
||||||
|
|
||||||
|
|
||||||
def test_result_upload_is_atomic_resumable_and_required_before_success(
|
def test_result_upload_is_atomic_resumable_and_required_before_success(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import io
|
||||||
import json
|
import json
|
||||||
|
import tarfile
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -9,9 +11,12 @@ import httpx
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from k1link.observatory.portable_artifact_transport import (
|
from k1link.observatory.portable_artifact_transport import (
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_ID_HEADER,
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE,
|
||||||
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
PORTABLE_RESULT_UPLOAD_PLAN_SCHEMA,
|
||||||
PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
PORTABLE_RESULT_UPLOAD_RECEIPT_SCHEMA,
|
||||||
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
PORTABLE_SOURCE_MATERIALIZATION_SCHEMA,
|
||||||
|
portable_camera_epoch_archive_id,
|
||||||
)
|
)
|
||||||
from k1link.observatory.portable_result_contract import (
|
from k1link.observatory.portable_result_contract import (
|
||||||
OBSERVATION_ONLY_AUTHORITY,
|
OBSERVATION_ONLY_AUTHORITY,
|
||||||
@@ -29,6 +34,8 @@ from k1link.observatory.worker_agent import (
|
|||||||
SealedObservatoryRecordedJob,
|
SealedObservatoryRecordedJob,
|
||||||
)
|
)
|
||||||
from k1link.observatory.worker_http_transport import (
|
from k1link.observatory.worker_http_transport import (
|
||||||
|
WORKER_HTTP_CAMERA_ARCHIVE_READ_TIMEOUT_SECONDS,
|
||||||
|
WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS,
|
||||||
ObservatoryWorkerHttpError,
|
ObservatoryWorkerHttpError,
|
||||||
ObservatoryWorkerHttpGateway,
|
ObservatoryWorkerHttpGateway,
|
||||||
)
|
)
|
||||||
@@ -138,7 +145,10 @@ def _source_contract(
|
|||||||
job: SealedObservatoryRecordedJob,
|
job: SealedObservatoryRecordedJob,
|
||||||
*,
|
*,
|
||||||
inject_path: bool = False,
|
inject_path: bool = False,
|
||||||
|
camera_segment_count: int = 1,
|
||||||
) -> tuple[dict[str, object], dict[str, bytes]]:
|
) -> tuple[dict[str, object], dict[str, bytes]]:
|
||||||
|
if camera_segment_count < 1:
|
||||||
|
raise ValueError("camera segment count must be positive")
|
||||||
rows = [
|
rows = [
|
||||||
_source_member(
|
_source_member(
|
||||||
job,
|
job,
|
||||||
@@ -175,16 +185,23 @@ def _source_contract(
|
|||||||
camera_epoch=1,
|
camera_epoch=1,
|
||||||
media_type='video/mp4; codecs="avc1.641028"',
|
media_type='video/mp4; codecs="avc1.641028"',
|
||||||
),
|
),
|
||||||
|
]
|
||||||
|
rows.extend(
|
||||||
_source_member(
|
_source_member(
|
||||||
job,
|
job,
|
||||||
kind="camera-segment",
|
kind="camera-segment",
|
||||||
payload=b"sealed-camera-segment",
|
payload=(
|
||||||
|
b"sealed-camera-segment"
|
||||||
|
if sequence == 1
|
||||||
|
else f"sealed-camera-segment-{sequence}".encode("ascii")
|
||||||
|
),
|
||||||
artifact_id="recorded-camera-right",
|
artifact_id="recorded-camera-right",
|
||||||
camera_epoch=1,
|
camera_epoch=1,
|
||||||
camera_sequence=1,
|
camera_sequence=sequence,
|
||||||
media_type="video/iso.segment",
|
media_type="video/iso.segment",
|
||||||
),
|
)
|
||||||
]
|
for sequence in range(1, camera_segment_count + 1)
|
||||||
|
)
|
||||||
members = [row for row, _payload in rows]
|
members = [row for row, _payload in rows]
|
||||||
members.sort(key=lambda row: str(row["member_id"]))
|
members.sort(key=lambda row: str(row["member_id"]))
|
||||||
if inject_path:
|
if inject_path:
|
||||||
@@ -210,6 +227,86 @@ def _source_contract(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _camera_archive_response(
|
||||||
|
job: SealedObservatoryRecordedJob,
|
||||||
|
manifest: dict[str, object],
|
||||||
|
payloads: dict[str, bytes],
|
||||||
|
*,
|
||||||
|
mutation: str | None = None,
|
||||||
|
) -> tuple[bytes, dict[str, str]]:
|
||||||
|
rows = manifest["members"]
|
||||||
|
assert isinstance(rows, list)
|
||||||
|
camera_rows = [
|
||||||
|
row
|
||||||
|
for row in rows
|
||||||
|
if isinstance(row, dict)
|
||||||
|
and row.get("kind") in {"camera-init", "camera-segment"}
|
||||||
|
]
|
||||||
|
camera_rows.sort(
|
||||||
|
key=lambda row: (
|
||||||
|
0 if row["kind"] == "camera-init" else 1,
|
||||||
|
int(row["camera_sequence"] or 0),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
packed = io.BytesIO()
|
||||||
|
with tarfile.open(
|
||||||
|
fileobj=packed,
|
||||||
|
mode="w",
|
||||||
|
format=tarfile.USTAR_FORMAT,
|
||||||
|
) as archive:
|
||||||
|
for index, row in enumerate(camera_rows):
|
||||||
|
sequence = row["camera_sequence"]
|
||||||
|
name = (
|
||||||
|
"init.mp4"
|
||||||
|
if row["kind"] == "camera-init"
|
||||||
|
else f"segments/{sequence}.m4s"
|
||||||
|
)
|
||||||
|
if mutation == "traversal" and index == len(camera_rows) - 1:
|
||||||
|
name = "../../operator-secret"
|
||||||
|
member_id = str(row["member_id"])
|
||||||
|
payload = payloads[member_id]
|
||||||
|
info = tarfile.TarInfo(name)
|
||||||
|
info.mode = 0o600
|
||||||
|
info.mtime = 0
|
||||||
|
info.uid = 0
|
||||||
|
info.gid = 0
|
||||||
|
info.uname = ""
|
||||||
|
info.gname = ""
|
||||||
|
if mutation == "symlink" and index == len(camera_rows) - 1:
|
||||||
|
info.type = tarfile.SYMTYPE
|
||||||
|
info.linkname = "../../operator-secret"
|
||||||
|
info.size = 0
|
||||||
|
archive.addfile(info)
|
||||||
|
else:
|
||||||
|
info.size = len(payload)
|
||||||
|
archive.addfile(info, io.BytesIO(payload))
|
||||||
|
archive_payload = packed.getvalue()
|
||||||
|
sha256 = hashlib.sha256(archive_payload).hexdigest()
|
||||||
|
init = camera_rows[0]
|
||||||
|
archive_id = portable_camera_epoch_archive_id(
|
||||||
|
job_identity_sha256=job.identity_sha256,
|
||||||
|
source_bundle_sha256=job.source_bundle_sha256,
|
||||||
|
artifact_id=str(init["artifact_id"]),
|
||||||
|
camera_epoch=int(init["camera_epoch"]),
|
||||||
|
member_ids=tuple(str(row["member_id"]) for row in camera_rows),
|
||||||
|
byte_length=len(archive_payload),
|
||||||
|
sha256=sha256,
|
||||||
|
)
|
||||||
|
headers = {
|
||||||
|
"Content-Type": PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE,
|
||||||
|
"Content-Length": str(len(archive_payload)),
|
||||||
|
"X-Mission-Core-Content-Sha256": sha256,
|
||||||
|
PORTABLE_CAMERA_EPOCH_ARCHIVE_ID_HEADER: archive_id,
|
||||||
|
}
|
||||||
|
if mutation == "corruption":
|
||||||
|
changed = bytearray(archive_payload)
|
||||||
|
offset = archive_payload.find(b"sealed-camera-init")
|
||||||
|
assert offset >= 0
|
||||||
|
changed[offset] ^= 1
|
||||||
|
archive_payload = bytes(changed)
|
||||||
|
return archive_payload, headers
|
||||||
|
|
||||||
|
|
||||||
def test_http_gateway_materializes_only_exact_claim_bound_members(
|
def test_http_gateway_materializes_only_exact_claim_bound_members(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -232,6 +329,8 @@ def test_http_gateway_materializes_only_exact_claim_bound_members(
|
|||||||
artifact_requests.append(request)
|
artifact_requests.append(request)
|
||||||
if request.url.path.endswith("/source-materialization"):
|
if request.url.path.endswith("/source-materialization"):
|
||||||
return httpx.Response(200, json=manifest)
|
return httpx.Response(200, json=manifest)
|
||||||
|
if request.url.path.endswith("/source-camera-epoch-archive"):
|
||||||
|
return httpx.Response(404)
|
||||||
member_id = request.url.path.rsplit("/", 1)[-1]
|
member_id = request.url.path.rsplit("/", 1)[-1]
|
||||||
payload = payloads[member_id]
|
payload = payloads[member_id]
|
||||||
return httpx.Response(
|
return httpx.Response(
|
||||||
@@ -266,7 +365,148 @@ def test_http_gateway_materializes_only_exact_claim_bound_members(
|
|||||||
)
|
)
|
||||||
assert persisted == manifest
|
assert persisted == manifest
|
||||||
assert "path" not in json.dumps(persisted, sort_keys=True)
|
assert "path" not in json.dumps(persisted, sort_keys=True)
|
||||||
assert len(artifact_requests) == 1 + len(payloads)
|
assert len(artifact_requests) == 2 + len(payloads)
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_gateway_packed_camera_epoch_bounds_requests_and_header_timeout(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
job = _job(
|
||||||
|
bundle_sha256=hashlib.sha256(b"source-bundle").hexdigest(),
|
||||||
|
capability_sha256=hashlib.sha256(b"source-capability").hexdigest(),
|
||||||
|
)
|
||||||
|
manifest, payloads = _source_contract(job, camera_segment_count=101)
|
||||||
|
archive_payload, archive_headers = _camera_archive_response(
|
||||||
|
job,
|
||||||
|
manifest,
|
||||||
|
payloads,
|
||||||
|
)
|
||||||
|
artifact_requests: list[httpx.Request] = []
|
||||||
|
camera_member_requests: list[str] = []
|
||||||
|
archive_read_timeouts: list[float] = []
|
||||||
|
rows = manifest["members"]
|
||||||
|
assert isinstance(rows, list)
|
||||||
|
camera_member_ids = {
|
||||||
|
str(row["member_id"])
|
||||||
|
for row in rows
|
||||||
|
if isinstance(row, dict)
|
||||||
|
and row.get("kind") in {"camera-init", "camera-segment"}
|
||||||
|
}
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
if request.url.path.endswith("/claims"):
|
||||||
|
return _claim_response()
|
||||||
|
artifact_requests.append(request)
|
||||||
|
if request.url.path.endswith("/source-materialization"):
|
||||||
|
return httpx.Response(200, json=manifest)
|
||||||
|
if request.url.path.endswith("/source-camera-epoch-archive"):
|
||||||
|
timeout = request.extensions.get("timeout")
|
||||||
|
assert isinstance(timeout, dict)
|
||||||
|
read_timeout = timeout.get("read")
|
||||||
|
assert isinstance(read_timeout, float)
|
||||||
|
assert read_timeout > WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS
|
||||||
|
archive_read_timeouts.append(read_timeout)
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
content=archive_payload,
|
||||||
|
headers=archive_headers,
|
||||||
|
)
|
||||||
|
member_id = request.url.path.rsplit("/", 1)[-1]
|
||||||
|
if member_id in camera_member_ids:
|
||||||
|
camera_member_requests.append(member_id)
|
||||||
|
payload = payloads[member_id]
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
content=payload,
|
||||||
|
headers={
|
||||||
|
"X-Mission-Core-Content-Sha256": hashlib.sha256(
|
||||||
|
payload
|
||||||
|
).hexdigest()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
with ObservatoryWorkerHttpGateway(
|
||||||
|
base_url="http://127.0.0.1:18080",
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path / "worker",
|
||||||
|
transport=httpx.MockTransport(handler),
|
||||||
|
) as gateway:
|
||||||
|
_cache_claim(gateway)
|
||||||
|
stage = gateway.materialize(job)
|
||||||
|
|
||||||
|
assert archive_read_timeouts == [
|
||||||
|
WORKER_HTTP_CAMERA_ARCHIVE_READ_TIMEOUT_SECONDS
|
||||||
|
]
|
||||||
|
assert camera_member_requests == []
|
||||||
|
assert len(artifact_requests) == 6
|
||||||
|
assert (
|
||||||
|
stage.root / "camera/epoch-1/segments/101.m4s"
|
||||||
|
).read_bytes() == b"sealed-camera-segment-101"
|
||||||
|
expected_files = {
|
||||||
|
"source-bundle.json",
|
||||||
|
"source-capability.json",
|
||||||
|
"mqtt.raw.k1mqtt",
|
||||||
|
"mqtt.metadata.jsonl",
|
||||||
|
"materialization-manifest.json",
|
||||||
|
"camera/epoch-1/init.mp4",
|
||||||
|
*(f"camera/epoch-1/segments/{sequence}.m4s" for sequence in range(1, 102)),
|
||||||
|
}
|
||||||
|
assert {
|
||||||
|
path.relative_to(stage.root).as_posix()
|
||||||
|
for path in stage.root.rglob("*")
|
||||||
|
if path.is_file()
|
||||||
|
} == expected_files
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("mutation", "message"),
|
||||||
|
[
|
||||||
|
("corruption", "content differs"),
|
||||||
|
("traversal", "unsafe member"),
|
||||||
|
("symlink", "unsafe member"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_http_gateway_rejects_corrupt_or_unsafe_camera_archive(
|
||||||
|
tmp_path: Path,
|
||||||
|
mutation: str,
|
||||||
|
message: str,
|
||||||
|
) -> None:
|
||||||
|
job = _job(
|
||||||
|
bundle_sha256=hashlib.sha256(b"source-bundle").hexdigest(),
|
||||||
|
capability_sha256=hashlib.sha256(b"source-capability").hexdigest(),
|
||||||
|
)
|
||||||
|
manifest, payloads = _source_contract(job)
|
||||||
|
archive_payload, archive_headers = _camera_archive_response(
|
||||||
|
job,
|
||||||
|
manifest,
|
||||||
|
payloads,
|
||||||
|
mutation=mutation,
|
||||||
|
)
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
if request.url.path.endswith("/claims"):
|
||||||
|
return _claim_response()
|
||||||
|
if request.url.path.endswith("/source-materialization"):
|
||||||
|
return httpx.Response(200, json=manifest)
|
||||||
|
if request.url.path.endswith("/source-camera-epoch-archive"):
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
content=archive_payload,
|
||||||
|
headers=archive_headers,
|
||||||
|
)
|
||||||
|
pytest.fail("unsafe camera archive must fail before member fallback")
|
||||||
|
|
||||||
|
with ObservatoryWorkerHttpGateway(
|
||||||
|
base_url="http://127.0.0.1:18080",
|
||||||
|
bearer_token=BEARER_TOKEN,
|
||||||
|
work_root=tmp_path / "worker",
|
||||||
|
transport=httpx.MockTransport(handler),
|
||||||
|
) as gateway:
|
||||||
|
_cache_claim(gateway)
|
||||||
|
with pytest.raises(ObservatoryWorkerHttpError, match=message):
|
||||||
|
gateway.materialize(job)
|
||||||
|
|
||||||
|
assert not list(tmp_path.rglob("operator-secret"))
|
||||||
|
|
||||||
|
|
||||||
def test_http_gateway_rejects_server_selected_source_path(tmp_path: Path) -> None:
|
def test_http_gateway_rejects_server_selected_source_path(tmp_path: Path) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user