ФУНКЦИИ - МЕЖПРОЕКТНАЯ КОММУНИКАЦИЯ: blob-дедупликация файлов

2026-04-27 09:45:16 +03:00
parent b945bc4d31
commit 490aa1bc04
18 changed files with 827 additions and 114 deletions
@@ -0,0 +1,152 @@
+# Copyright (c) 2023-present Plane Software, Inc. and contributors
+# SPDX-License-Identifier: AGPL-3.0-only
+# See the LICENSE file for details.
+
+from unittest.mock import patch
+
+import pytest
+
+from plane.db.models import FileAsset, StoredBlob
+from plane.utils.file_dedup import attach_existing_blob_to_file_asset, finalize_uploaded_file_asset, release_file_asset_blob
+
+
+class FakeStreamingBody:
+    def __init__(self, payload):
+        self.payload = payload
+        self.closed = False
+
+    def iter_chunks(self, chunk_size):
+        for index in range(0, len(self.payload), chunk_size):
+            yield self.payload[index : index + chunk_size]
+
+    def close(self):
+        self.closed = True
+
+
+class FakeS3Client:
+    def __init__(self, objects):
+        self.objects = objects
+
+    def get_object(self, Bucket, Key):
+        return {"Body": FakeStreamingBody(self.objects[Key]["body"])}
+
+
+class FakeStorage:
+    def __init__(self, objects):
+        self.aws_storage_bucket_name = "uploads"
+        self.s3_client = FakeS3Client(objects)
+        self.deleted = []
+        self.objects = objects
+
+    def get_object_metadata(self, object_name):
+        item = self.objects.get(object_name)
+        if item is None:
+            return None
+        return {
+            "ContentType": item["type"],
+            "ContentLength": len(item["body"]),
+            "ETag": item.get("etag", "etag"),
+            "Metadata": {},
+        }
+
+    def delete_files(self, object_names):
+        self.deleted.extend(object_names)
+        for object_name in object_names:
+            self.objects.pop(object_name, None)
+        return True
+
+
+@pytest.fixture
+def project(workspace):
+    from plane.db.models import Project, ProjectMember
+
+    project = Project.objects.create(name="Dedup Project", identifier="DEDUP", workspace=workspace)
+    ProjectMember.objects.create(project=project, member=workspace.owner)
+    return project
+
+
+@pytest.fixture
+def fake_storage():
+    objects = {
+        "workspace/a.txt": {"body": b"same payload", "type": "text/plain"},
+        "workspace/b.txt": {"body": b"same payload", "type": "text/plain"},
+    }
+    return FakeStorage(objects)
+
+
+def create_asset(workspace, project, object_key):
+    return FileAsset.objects.create(
+        workspace=workspace,
+        project=project,
+        asset=object_key,
+        size=12,
+        attributes={"name": object_key.rsplit("/", 1)[-1], "type": "text/plain", "size": 12},
+        entity_type=FileAsset.EntityTypeContext.ISSUE_ATTACHMENT,
+    )
+
+
+@pytest.mark.django_db
+def test_finalize_uploaded_file_asset_reuses_existing_blob(workspace, project, fake_storage):
+    first_asset = create_asset(workspace, project, "workspace/a.txt")
+    duplicate_asset = create_asset(workspace, project, "workspace/b.txt")
+
+    with patch("plane.utils.file_dedup.S3Storage", return_value=fake_storage):
+        first_result = finalize_uploaded_file_asset(first_asset)
+        duplicate_result = finalize_uploaded_file_asset(duplicate_asset)
+
+    first_asset.refresh_from_db()
+    duplicate_asset.refresh_from_db()
+    blob = StoredBlob.objects.get()
+
+    assert first_result.deduplicated is False
+    assert duplicate_result.deduplicated is True
+    assert duplicate_result.deleted_duplicate_object is True
+    assert first_asset.blob_id == blob.id
+    assert duplicate_asset.blob_id == blob.id
+    assert first_asset.asset.name == "workspace/a.txt"
+    assert duplicate_asset.asset.name == "workspace/a.txt"
+    assert blob.ref_count == 2
+    assert fake_storage.deleted == ["workspace/b.txt"]
+
+
+@pytest.mark.django_db
+def test_release_file_asset_blob_deletes_canonical_object_after_last_reference(workspace, project, fake_storage):
+    first_asset = create_asset(workspace, project, "workspace/a.txt")
+    duplicate_asset = create_asset(workspace, project, "workspace/b.txt")
+
+    with patch("plane.utils.file_dedup.S3Storage", return_value=fake_storage):
+        finalize_uploaded_file_asset(first_asset)
+        finalize_uploaded_file_asset(duplicate_asset)
+        first_release = release_file_asset_blob(first_asset)
+        second_release = release_file_asset_blob(duplicate_asset)
+
+    blob = StoredBlob.objects.get()
+
+    assert first_release.had_blob is True
+    assert first_release.deleted_object is False
+    assert second_release.had_blob is True
+    assert second_release.deleted_object is True
+    assert second_release.object_key == "workspace/a.txt"
+    assert blob.ref_count == 0
+    assert blob.status == StoredBlob.Status.ORPHANED
+
+
+@pytest.mark.django_db
+def test_attach_existing_blob_reuses_canonical_object_without_copy(workspace, project, fake_storage):
+    source_asset = create_asset(workspace, project, "workspace/a.txt")
+    target_asset = create_asset(workspace, project, "workspace/c.txt")
+
+    with patch("plane.utils.file_dedup.S3Storage", return_value=fake_storage):
+        finalize_uploaded_file_asset(source_asset)
+        attached = attach_existing_blob_to_file_asset(source_asset, target_asset)
+
+    source_asset.refresh_from_db()
+    target_asset.refresh_from_db()
+    blob = StoredBlob.objects.get()
+
+    assert attached is True
+    assert source_asset.blob_id == blob.id
+    assert target_asset.blob_id == blob.id
+    assert target_asset.asset.name == "workspace/a.txt"
+    assert blob.ref_count == 2
+    assert fake_storage.deleted == []