feat(simulation): upload real gaussian source bundles

src/k1link/simulation/gaussian_pipeline_gateway.py

@@ -46,23 +46,39 @@ class GaussianPipelineIntegrityError(GaussianPipelineGatewayError):
 
 
 @dataclass(frozen=True, slots=True)
-class GaussianSourceUpload:
+class GaussianSourceMemberUpload:
     upload_id: str
-    filename: str
+    logical_path: str
-    format: str
     sha256: str
     byte_length: int
 
     def to_dict(self) -> dict[str, object]:
         return {
             "upload_id": self.upload_id,
-            "filename": self.filename,
+            "logical_path": self.logical_path,
-            "format": self.format,
             "sha256": self.sha256,
             "byte_length": self.byte_length,
         }
 
 
+@dataclass(frozen=True, slots=True)
+class GaussianSourceBundleUpload:
+    format: str
+    entrypoint: str
+    bundle_sha256: str
+    total_byte_length: int
+    members: tuple[GaussianSourceMemberUpload, ...]
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "format": self.format,
+            "entrypoint": self.entrypoint,
+            "bundle_sha256": self.bundle_sha256,
+            "total_byte_length": self.total_byte_length,
+            "members": [member.to_dict() for member in self.members],
+        }
+
+
 class GaussianPipelineGateway:
     """TUS and JSON client with bounded responses and independent digest checks."""
 
@@ -106,37 +122,90 @@ class GaussianPipelineGateway:
             document.get("schema_version") != CAPABILITIES_SCHEMA
             or document.get("service") != "ndc-gaussian-pipeline"
             or document.get("api_version") != "gaussian-pipeline.api/v1"
+            or document.get("upload_protocol") != "tus/1.0.0"
+            or document.get("source_transport") != "tus-bundle/v1"
         ):
             raise GaussianPipelineGatewayError("Gaussian provider capabilities do not match v1")
         _validate_runtime_provenance(document)
         return document
 
-    def upload_source(
+    def upload_source_bundle(
         self,
-        source_path: Path,
+        bundle_root: Path,
         *,
-        filename: str,
+        entrypoint: str,
         source_format: str,
-        sha256: str,
+    ) -> GaussianSourceBundleUpload:
-    ) -> GaussianSourceUpload:
-        source_candidate = source_path.expanduser().absolute()
-        if source_candidate.is_symlink() or not source_candidate.is_file():
-            raise GaussianPipelineIntegrityError("Gaussian source must be one regular file")
-        source = source_candidate.resolve()
         if source_format not in {"lcc", "lcc2"}:
             raise GaussianPipelineIntegrityError("Gaussian source format must be lcc or lcc2")
-        if Path(filename).name != filename or not filename.lower().endswith(f".{source_format}"):
+        logical_entrypoint = _logical_path(entrypoint, "entrypoint")
-            raise GaussianPipelineIntegrityError("Gaussian source filename is unsafe")
+        if not logical_entrypoint.lower().endswith(f".{source_format}"):
-        if SHA256_PATTERN.fullmatch(sha256) is None:
+            raise GaussianPipelineIntegrityError(
-            raise GaussianPipelineIntegrityError("Gaussian source digest is invalid")
+                "Gaussian source entrypoint extension does not match its format"
+            )
+        root_candidate = bundle_root.expanduser().absolute()
+        if root_candidate.is_symlink() or not root_candidate.is_dir():
+            raise GaussianPipelineIntegrityError(
+                "Gaussian source bundle root must be one regular directory"
+            )
+        root = root_candidate.resolve()
+        logical_paths = _discover_bundle_members(root, logical_entrypoint, source_format)
+        local_members: list[tuple[str, Path, str, int]] = []
+        total_byte_length = 0
+        for logical_path in sorted(logical_paths, key=lambda value: value.encode("utf-8")):
+            source = _bundle_member(root, logical_path)
             byte_length = source.stat().st_size
             if byte_length <= 0:
-            raise GaussianPipelineIntegrityError("Gaussian source is empty")
+                raise GaussianPipelineIntegrityError(
-        if _sha256(source) != sha256:
+                    f"Gaussian source bundle member is empty: {logical_path}"
-            raise GaussianPipelineIntegrityError("Gaussian source digest does not match")
+                )
+            digest = _sha256(source)
+            total_byte_length += byte_length
+            local_members.append((logical_path, source, digest, byte_length))
+
+        capabilities = self.capabilities()
+        max_source_bytes = capabilities.get("max_source_bytes")
+        max_source_files = capabilities.get("max_source_files")
+        if (
+            not isinstance(max_source_bytes, int)
+            or isinstance(max_source_bytes, bool)
+            or total_byte_length > max_source_bytes
+        ):
+            raise GaussianPipelineIntegrityError(
+                "Gaussian source bundle exceeds provider byte admission"
+            )
+        if (
+            not isinstance(max_source_files, int)
+            or isinstance(max_source_files, bool)
+            or len(local_members) > max_source_files
+        ):
+            raise GaussianPipelineIntegrityError(
+                "Gaussian source bundle exceeds provider file admission"
+            )
+
+        members = tuple(
+            self._upload_member(source, logical_path, digest, byte_length)
+            for logical_path, source, digest, byte_length in local_members
+        )
+        bundle_sha256 = _bundle_sha256(source_format, logical_entrypoint, members)
+        return GaussianSourceBundleUpload(
+            format=source_format,
+            entrypoint=logical_entrypoint,
+            bundle_sha256=bundle_sha256,
+            total_byte_length=total_byte_length,
+            members=members,
+        )
+
+    def _upload_member(
+        self,
+        source: Path,
+        logical_path: str,
+        sha256: str,
+        byte_length: int,
+    ) -> GaussianSourceMemberUpload:
 
         metadata = _tus_metadata(
-            {"filename": filename, "format": source_format, "sha256": sha256}
+            {"logical_path": logical_path, "sha256": sha256}
         )
         try:
             response = self._client.post(
@@ -161,10 +230,9 @@ class GaussianPipelineGateway:
         if SAFE_UPLOAD_ID.fullmatch(upload_id) is None:
             raise GaussianPipelineGatewayError("Gaussian upload id is invalid")
         self._send_file(source, upload_url, byte_length)
-        return GaussianSourceUpload(
+        return GaussianSourceMemberUpload(
             upload_id=upload_id,
-            filename=filename,
+            logical_path=logical_path,
-            format=source_format,
             sha256=sha256,
             byte_length=byte_length,
         )
@@ -399,6 +467,134 @@ def _sha256(path: Path) -> str:
     return digest.hexdigest()
 
 
+def _bundle_sha256(
+    source_format: str,
+    entrypoint: str,
+    members: tuple[GaussianSourceMemberUpload, ...],
+) -> str:
+    document = {
+        "format": source_format,
+        "entrypoint": entrypoint,
+        "members": [
+            {
+                "logical_path": member.logical_path,
+                "sha256": member.sha256,
+                "byte_length": member.byte_length,
+            }
+            for member in members
+        ],
+    }
+    canonical = json.dumps(
+        document,
+        ensure_ascii=False,
+        separators=(",", ":"),
+    ).encode("utf-8")
+    return hashlib.sha256(canonical).hexdigest()
+
+
+def _discover_bundle_members(root: Path, entrypoint: str, source_format: str) -> set[str]:
+    descriptor = _bundle_member(root, entrypoint)
+    if descriptor.stat().st_size > 16 * 1024 * 1024:
+        raise GaussianPipelineIntegrityError("Gaussian source descriptor is too large")
+    try:
+        text = descriptor.read_text(encoding="utf-8")
+        try:
+            document = json.loads(text)
+        except json.JSONDecodeError:
+            document = json.loads(re.sub(r",(?=\s*[}\]])", "", text))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise GaussianPipelineIntegrityError(
+            "Gaussian source descriptor is not readable JSON"
+        ) from exc
+    if not isinstance(document, dict):
+        raise GaussianPipelineIntegrityError("Gaussian source descriptor must be an object")
+
+    base = entrypoint.rsplit("/", 1)[0] if "/" in entrypoint else ""
+
+    def related(name: str) -> str:
+        logical = f"{base}/{name}" if base else name
+        return _logical_path(logical, "descriptor member")
+
+    members = {entrypoint}
+    if source_format == "lcc":
+        members.update({related("index.bin"), related("data.bin")})
+        file_type = document.get("fileType")
+        attributes = document.get("attributes")
+        has_sh = file_type == "Quality"
+        if file_type not in {"Portable", "Quality"}:
+            has_sh = isinstance(attributes, list) and any(
+                isinstance(attribute, dict) and attribute.get("name") == "shcoef"
+                for attribute in attributes
+            )
+        if has_sh:
+            members.add(related("shcoef.bin"))
+        environment = related("environment.bin")
+        if (root / Path(*environment.split("/"))).exists():
+            members.add(environment)
+        return members
+
+    root_node = document.get("root")
+    if not isinstance(root_node, dict):
+        raise GaussianPipelineIntegrityError("Gaussian LCC2 descriptor has no root object")
+    splat_files: object
+    if all(key in document for key in ("total_splats", "lod_3dgs_info", "lod_level")):
+        legacy_files = root_node.get("files")
+        if not isinstance(legacy_files, list):
+            raise GaussianPipelineIntegrityError("Gaussian legacy LCC2 descriptor has no files")
+        normalized: list[str] = []
+        for item in legacy_files:
+            if not isinstance(item, str):
+                raise GaussianPipelineIntegrityError("Gaussian LCC2 chunk path is invalid")
+            value = item[1:] if item.startswith("/") else item
+            normalized.append(value if value.endswith(".sog") else f"{value}.sog")
+        splat_files = normalized
+    else:
+        splat_files = root_node.get("splatFiles")
+    if not isinstance(splat_files, list) or not splat_files:
+        raise GaussianPipelineIntegrityError("Gaussian LCC2 descriptor has no splat files")
+    for item in splat_files:
+        if not isinstance(item, str):
+            raise GaussianPipelineIntegrityError("Gaussian LCC2 chunk path is invalid")
+        members.add(related(item))
+    return members
+
+
+def _logical_path(value: str, label: str) -> str:
+    if (
+        not value
+        or len(value) > 1024
+        or value.startswith("/")
+        or "\\" in value
+        or "\x00" in value
+        or any(part in {"", ".", ".."} for part in value.split("/"))
+    ):
+        raise GaussianPipelineIntegrityError(f"Gaussian source {label} is unsafe")
+    return value
+
+
+def _bundle_member(root: Path, logical_path: str) -> Path:
+    logical = _logical_path(logical_path, "bundle member path")
+    candidate = root
+    for part in logical.split("/"):
+        candidate = candidate / part
+        if candidate.is_symlink():
+            raise GaussianPipelineIntegrityError(
+                f"Gaussian source bundle contains a symlink: {logical}"
+            )
+    try:
+        resolved = candidate.resolve(strict=True)
+        resolved.relative_to(root)
+    except (OSError, ValueError) as exc:
+        raise GaussianPipelineIntegrityError(
+            f"Gaussian source bundle member is unavailable: {logical}"
+        ) from exc
+    if not resolved.is_file():
+        raise GaussianPipelineIntegrityError(
+            f"Gaussian source bundle member is not a regular file: {logical}"
+        )
+    return resolved
+
+
 def _offset(value: str | None, byte_length: int) -> int:
     if value is None or not value.isdigit():
         raise GaussianPipelineGatewayError("Gaussian upload offset is invalid")

tests/test_gaussian_pipeline_gateway.py

@@ -13,6 +13,7 @@ from k1link.simulation.gaussian_pipeline_gateway import (
     GaussianPipelineGateway,
     GaussianPipelineGatewayError,
     GaussianPipelineIntegrityError,
+    _discover_bundle_members,
 )
 
 
@@ -22,9 +23,9 @@ def _token_file(tmp_path: Path) -> Path:
     return token
 
 
-def test_gateway_uploads_with_tus_and_reads_provider_contract(tmp_path: Path) -> None:
+def test_gateway_uploads_lcc_bundle_with_tus_and_reads_provider_contract(tmp_path: Path) -> None:
-    uploaded = bytearray()
+    uploads: dict[str, bytearray] = {}
-    metadata: dict[str, str] = {}
+    metadata: dict[str, dict[str, str]] = {}
 
     def handler(request: httpx.Request) -> httpx.Response:
         assert request.headers["authorization"] == f"Bearer {'t' * 64}"
@@ -35,29 +36,47 @@ def test_gateway_uploads_with_tus_and_reads_provider_contract(tmp_path: Path) ->
                     "schema_version": "gaussian-pipeline.capabilities/v1",
                     "service": "ndc-gaussian-pipeline",
                     "api_version": "gaussian-pipeline.api/v1",
+                    "upload_protocol": "tus/1.0.0",
+                    "source_transport": "tus-bundle/v1",
                     "provider": {"id": "playcanvas-splat-transform", "version": "3.3.3"},
                     "runtime": {
                         "source_revision": "a" * 40,
                         "image_digest": f"sha256:{'b' * 64}",
                     },
+                    "max_source_bytes": 1024 * 1024,
+                    "max_source_files": 10_000,
                 },
             )
         if request.method == "POST" and request.url.path == "/v1/uploads":
+            upload_id = f"upload-{len(uploads) + 1:03d}"
+            uploads[upload_id] = bytearray()
+            item_metadata: dict[str, str] = {}
             for item in request.headers["upload-metadata"].split(","):
                 key, encoded = item.split(" ", 1)
-                metadata[key] = base64.b64decode(encoded).decode("utf-8")
+                item_metadata[key] = base64.b64decode(encoded).decode("utf-8")
-            return httpx.Response(201, headers={"Location": "/v1/uploads/upload-001"})
+            metadata[upload_id] = item_metadata
-        if request.method == "HEAD" and request.url.path == "/v1/uploads/upload-001":
+            return httpx.Response(201, headers={"Location": f"/v1/uploads/{upload_id}"})
-            return httpx.Response(200, headers={"Upload-Offset": str(len(uploaded))})
+        upload_id = request.url.path.rsplit("/", 1)[-1]
-        if request.method == "PATCH" and request.url.path == "/v1/uploads/upload-001":
+        if request.method == "HEAD" and upload_id in uploads:
-            assert int(request.headers["upload-offset"]) == len(uploaded)
+            return httpx.Response(200, headers={"Upload-Offset": str(len(uploads[upload_id]))})
-            uploaded.extend(request.content)
+        if request.method == "PATCH" and upload_id in uploads:
-            return httpx.Response(204, headers={"Upload-Offset": str(len(uploaded))})
+            assert int(request.headers["upload-offset"]) == len(uploads[upload_id])
+            uploads[upload_id].extend(request.content)
+            return httpx.Response(
+                204,
+                headers={"Upload-Offset": str(len(uploads[upload_id]))},
+            )
         return httpx.Response(404)
 
-    source = tmp_path / "yard.lcc2"
+    bundle = tmp_path / "bundle"
-    source.write_bytes(b"portable-gaussian-source")
+    bundle.mkdir()
-    digest = hashlib.sha256(source.read_bytes()).hexdigest()
+    (bundle / "yard.lcc").write_text(
+        json.dumps({"fileType": "Quality", "attributes": []}),
+        encoding="utf-8",
+    )
+    (bundle / "index.bin").write_bytes(b"index")
+    (bundle / "data.bin").write_bytes(b"data")
+    (bundle / "shcoef.bin").write_bytes(b"sh")
     with GaussianPipelineGateway(
         "http://gaussian.test",
         _token_file(tmp_path),
@@ -65,49 +84,101 @@ def test_gateway_uploads_with_tus_and_reads_provider_contract(tmp_path: Path) ->
         transport=httpx.MockTransport(handler),
     ) as gateway:
         assert gateway.capabilities()["service"] == "ndc-gaussian-pipeline"
-        descriptor = gateway.upload_source(
+        descriptor = gateway.upload_source_bundle(
-            source,
+            bundle,
-            filename="yard.lcc2",
+            entrypoint="yard.lcc",
-            source_format="lcc2",
+            source_format="lcc",
-            sha256=digest,
         )
 
-    assert bytes(uploaded) == source.read_bytes()
+    paths = [member.logical_path for member in descriptor.members]
-    assert metadata == {"filename": "yard.lcc2", "format": "lcc2", "sha256": digest}
+    assert paths == ["data.bin", "index.bin", "shcoef.bin", "yard.lcc"]
-    assert descriptor.to_dict() == {
+    assert [metadata[f"upload-{index:03d}"]["logical_path"] for index in range(1, 5)] == paths
-        "upload_id": "upload-001",
+    for index, member in enumerate(descriptor.members, start=1):
-        "filename": "yard.lcc2",
+        assert bytes(uploads[f"upload-{index:03d}"]) == (bundle / member.logical_path).read_bytes()
-        "format": "lcc2",
+        assert metadata[f"upload-{index:03d}"]["sha256"] == member.sha256
-        "sha256": digest,
+    identity = {
-        "byte_length": len(uploaded),
+        "format": "lcc",
+        "entrypoint": "yard.lcc",
+        "members": [
+            {
+                "logical_path": member.logical_path,
+                "sha256": member.sha256,
+                "byte_length": member.byte_length,
             }
+            for member in descriptor.members
+        ],
+    }
+    assert descriptor.bundle_sha256 == hashlib.sha256(
+        json.dumps(identity, ensure_ascii=False, separators=(",", ":")).encode()
+    ).hexdigest()
+    assert descriptor.total_byte_length == sum(member.byte_length for member in descriptor.members)
 
 
-def test_gateway_rejects_local_source_digest_mismatch(tmp_path: Path) -> None:
+def test_gateway_rejects_incomplete_lcc_bundle(tmp_path: Path) -> None:
-    source = tmp_path / "yard.lcc"
+    bundle = tmp_path / "bundle"
-    source.write_bytes(b"source")
+    bundle.mkdir()
+    (bundle / "yard.lcc").write_text('{"fileType":"Portable"}', encoding="utf-8")
     with (
         GaussianPipelineGateway(
             "http://gaussian.test",
             _token_file(tmp_path),
             transport=httpx.MockTransport(lambda _request: httpx.Response(500)),
         ) as gateway,
-        pytest.raises(GaussianPipelineIntegrityError, match="digest does not match"),
+        pytest.raises(GaussianPipelineIntegrityError, match="member is unavailable"),
     ):
-        gateway.upload_source(
+        gateway.upload_source_bundle(
-            source,
+            bundle,
-            filename="yard.lcc",
+            entrypoint="yard.lcc",
             source_format="lcc",
-            sha256="a" * 64,
         )
 
 
-def test_gateway_rejects_upload_location_outside_provider(tmp_path: Path) -> None:
+def test_gateway_discovers_nested_lcc2_chunks_with_trailing_commas(tmp_path: Path) -> None:
-    source = tmp_path / "yard.lcc"
+    bundle = tmp_path / "bundle"
-    source.write_bytes(b"source")
+    (bundle / "scene" / "chunks").mkdir(parents=True)
-    digest = hashlib.sha256(source.read_bytes()).hexdigest()
+    (bundle / "scene" / "meta.lcc2").write_text(
+        """
+        {
+          "root": {"splatFiles": ["chunks/0.sog", "chunks/1.spz",],},
+        }
+        """,
+        encoding="utf-8",
+    )
+    (bundle / "scene" / "chunks" / "0.sog").write_bytes(b"sog")
+    (bundle / "scene" / "chunks" / "1.spz").write_bytes(b"spz")
 
-    def handler(_request: httpx.Request) -> httpx.Response:
+    assert _discover_bundle_members(bundle, "scene/meta.lcc2", "lcc2") == {
+        "scene/meta.lcc2",
+        "scene/chunks/0.sog",
+        "scene/chunks/1.spz",
+    }
+
+
+def test_gateway_rejects_upload_location_outside_provider(tmp_path: Path) -> None:
+    bundle = tmp_path / "bundle"
+    bundle.mkdir()
+    (bundle / "yard.lcc").write_text('{"fileType":"Portable"}', encoding="utf-8")
+    (bundle / "data.bin").write_bytes(b"data")
+    (bundle / "index.bin").write_bytes(b"index")
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.method == "GET":
+            return httpx.Response(
+                200,
+                json={
+                    "schema_version": "gaussian-pipeline.capabilities/v1",
+                    "service": "ndc-gaussian-pipeline",
+                    "api_version": "gaussian-pipeline.api/v1",
+                    "upload_protocol": "tus/1.0.0",
+                    "source_transport": "tus-bundle/v1",
+                    "runtime": {
+                        "source_revision": "a" * 40,
+                        "image_digest": f"sha256:{'b' * 64}",
+                    },
+                    "max_source_bytes": 1024,
+                    "max_source_files": 10_000,
+                },
+            )
         return httpx.Response(
             201,
             headers={"Location": "https://attacker.invalid/v1/uploads/stolen"},
@@ -121,19 +192,20 @@ def test_gateway_rejects_upload_location_outside_provider(tmp_path: Path) -> Non
         ) as gateway,
         pytest.raises(GaussianPipelineGatewayError, match="escaped the provider"),
     ):
-        gateway.upload_source(
+        gateway.upload_source_bundle(
-            source,
+            bundle,
-            filename="yard.lcc",
+            entrypoint="yard.lcc",
             source_format="lcc",
-            sha256=digest,
         )
 
 
 def test_gateway_rejects_symlink_source_and_token(tmp_path: Path) -> None:
-    source = tmp_path / "source.lcc"
+    bundle = tmp_path / "bundle"
-    source.write_bytes(b"source")
+    bundle.mkdir()
-    source_link = tmp_path / "source-link.lcc"
+    (bundle / "source.lcc").write_text('{"fileType":"Portable"}', encoding="utf-8")
-    source_link.symlink_to(source)
+    (bundle / "real-data.bin").write_bytes(b"source")
+    (bundle / "data.bin").symlink_to(bundle / "real-data.bin")
+    (bundle / "index.bin").write_bytes(b"index")
     token = _token_file(tmp_path)
     token_link = tmp_path / "token-link"
     token_link.symlink_to(token)
@@ -141,20 +213,18 @@ def test_gateway_rejects_symlink_source_and_token(tmp_path: Path) -> None:
     with pytest.raises(GaussianPipelineGatewayError, match="token must be one regular file"):
         GaussianPipelineGateway("http://gaussian.test", token_link)
 
-    digest = hashlib.sha256(source.read_bytes()).hexdigest()
     with (
         GaussianPipelineGateway(
             "http://gaussian.test",
             token,
             transport=httpx.MockTransport(lambda _request: httpx.Response(500)),
         ) as gateway,
-        pytest.raises(GaussianPipelineIntegrityError, match="one regular file"),
+        pytest.raises(GaussianPipelineIntegrityError, match="contains a symlink"),
     ):
-        gateway.upload_source(
+        gateway.upload_source_bundle(
-            source_link,
+            bundle,
-            filename="source.lcc",
+            entrypoint="source.lcc",
             source_format="lcc",
-            sha256=digest,
         )
 
 
@@ -223,6 +293,8 @@ def test_gateway_rejects_capabilities_without_runtime_provenance(tmp_path: Path)
                 "schema_version": "gaussian-pipeline.capabilities/v1",
                 "service": "ndc-gaussian-pipeline",
                 "api_version": "gaussian-pipeline.api/v1",
+                "upload_protocol": "tus/1.0.0",
+                "source_transport": "tus-bundle/v1",
             },
         )
 

tests/test_simulation_world_provider_api.py

@@ -23,6 +23,8 @@ def _gateway(tmp_path: Path, status: int = 200) -> GaussianPipelineGateway:
                 "schema_version": "gaussian-pipeline.capabilities/v1",
                 "service": "ndc-gaussian-pipeline",
                 "api_version": "gaussian-pipeline.api/v1",
+                "upload_protocol": "tus/1.0.0",
+                "source_transport": "tus-bundle/v1",
                 "provider": {"id": "playcanvas-splat-transform", "version": "3.3.3"},
                 "runtime": {
                     "source_revision": "a" * 40,