SILVER/NODEDC_TASKMANAGER
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ФУНКЦИИ - МЕЖПРОЕКТНАЯ КОММУНИКАЦИЯ: blob-дедупликация файлов

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-04-27 09:45:16 +03:00
parent b945bc4d31
commit 490aa1bc04
18 changed files with 827 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plane-src/apps/api/plane/app/serializers/asset.py
+1 -1
View File
@@ -10,4 +10,4 @@ class FileAssetSerializer(BaseSerializer):
class Meta:
model = FileAsset
fields = "__all__"
read_only_fields = ["created_by", "updated_by", "created_at", "updated_at"]
read_only_fields = ["created_by", "updated_by", "created_at", "updated_at", "blob"]
plane-src/apps/api/plane/app/serializers/issue.py
+1
View File
@@ -630,6 +630,7 @@ class IssueAttachmentSerializer(BaseSerializer):
"workspace",
"project",
"issue",
"blob",
]
plane-src/apps/api/plane/app/views/asset/base.py
+23 -4
View File
@@ -11,6 +11,7 @@ from rest_framework.parsers import MultiPartParser, FormParser, JSONParser
from ..base import BaseAPIView, BaseViewSet
from plane.db.models import FileAsset, Workspace
from plane.app.serializers import FileAssetSerializer
from plane.utils.file_dedup import UploadedObjectMissing, confirm_uploaded_file_asset, release_file_asset_blob
class FileAssetEndpoint(BaseAPIView):
@@ -37,15 +38,24 @@ class FileAssetEndpoint(BaseAPIView):
if serializer.is_valid():
# Get the workspace
workspace = Workspace.objects.get(slug=slug)
serializer.save(workspace_id=workspace.id)
asset = serializer.save(workspace_id=workspace.id)
try:
confirm_uploaded_file_asset(asset, request=request)
except UploadedObjectMissing:
return Response(
{"error": "The uploaded asset object was not found.", "status": False},
status=status.HTTP_400_BAD_REQUEST,
)
return Response(serializer.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def delete(self, request, workspace_id, asset_key):
asset_key = str(workspace_id) + "/" + asset_key
file_asset = FileAsset.objects.get(asset=asset_key)
if not file_asset.is_uploaded:
release_file_asset_blob(file_asset, request=request, delete_untracked_object=True)
file_asset.is_deleted = True
file_asset.save(update_fields=["is_deleted"])
file_asset.save(update_fields=["is_deleted", "updated_at"])
return Response(status=status.HTTP_204_NO_CONTENT)
@@ -75,12 +85,21 @@ class UserAssetsEndpoint(BaseAPIView):
def post(self, request):
serializer = FileAssetSerializer(data=request.data)
if serializer.is_valid():
serializer.save()
asset = serializer.save()
try:
confirm_uploaded_file_asset(asset, request=request)
except UploadedObjectMissing:
return Response(
{"error": "The uploaded asset object was not found.", "status": False},
status=status.HTTP_400_BAD_REQUEST,
)
return Response(serializer.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def delete(self, request, asset_key):
file_asset = FileAsset.objects.get(asset=asset_key, created_by=request.user)
if not file_asset.is_uploaded:
release_file_asset_blob(file_asset, request=request, delete_untracked_object=True)
file_asset.is_deleted = True
file_asset.save(update_fields=["is_deleted"])
file_asset.save(update_fields=["is_deleted", "updated_at"])
return Response(status=status.HTTP_204_NO_CONTENT)
plane-src/apps/api/plane/app/views/asset/v2.py
+64 -38
View File
@@ -22,9 +22,14 @@ from plane.db.models import FileAsset, Workspace, Project, User
from plane.settings.storage import S3Storage
from plane.app.permissions import allow_permission, ROLE
from plane.utils.cache import invalidate_cache_directly
from plane.bgtasks.storage_metadata_task import get_asset_object_metadata
from plane.throttles.asset import AssetRateThrottle
from plane.utils.upload_limits import resolve_workspace_upload_size_limit
from plane.utils.file_dedup import (
UploadedObjectMissing,
attach_existing_blob_to_file_asset,
confirm_uploaded_file_asset,
release_file_asset_blob,
)
class UserAssetsV2Endpoint(BaseAPIView):
@@ -34,9 +39,11 @@ class UserAssetsV2Endpoint(BaseAPIView):
asset = FileAsset.objects.filter(id=asset_id).first()
if asset is None:
return
if not asset.is_uploaded:
release_file_asset_blob(asset, delete_untracked_object=True)
asset.is_deleted = True
asset.deleted_at = timezone.now()
asset.save(update_fields=["is_deleted", "deleted_at"])
asset.save(update_fields=["is_deleted", "deleted_at", "updated_at"])
return
def entity_asset_save(self, asset_id, entity_type, asset, request):
@@ -171,11 +178,17 @@ class UserAssetsV2Endpoint(BaseAPIView):
def patch(self, request, asset_id):
# get the asset id
asset = FileAsset.objects.get(id=asset_id, user_id=request.user.id)
# get the storage metadata
asset.is_uploaded = True
# get the storage metadata
if not asset.storage_metadata:
get_asset_object_metadata.delay(asset_id=str(asset_id))
try:
asset = confirm_uploaded_file_asset(
asset,
request=request,
attributes=request.data.get("attributes", asset.attributes),
)
except UploadedObjectMissing:
return Response(
{"error": "The uploaded asset object was not found.", "status": False},
status=status.HTTP_400_BAD_REQUEST,
)
# get the entity and save the asset id for the request field
self.entity_asset_save(
asset_id=asset_id,
@@ -183,19 +196,17 @@ class UserAssetsV2Endpoint(BaseAPIView):
asset=asset,
request=request,
)
# update the attributes
asset.attributes = request.data.get("attributes", asset.attributes)
# save the asset
asset.save(update_fields=["is_uploaded", "attributes"])
return Response(status=status.HTTP_204_NO_CONTENT)
def delete(self, request, asset_id):
asset = FileAsset.objects.get(id=asset_id, user_id=request.user.id)
if not asset.is_uploaded:
release_file_asset_blob(asset, request=request, delete_untracked_object=True)
asset.is_deleted = True
asset.deleted_at = timezone.now()
# get the entity and save the asset id for the request field
self.entity_asset_delete(entity_type=asset.entity_type, asset=asset, request=request)
asset.save(update_fields=["is_deleted", "deleted_at"])
asset.save(update_fields=["is_deleted", "deleted_at", "updated_at"])
return Response(status=status.HTTP_204_NO_CONTENT)
@@ -239,10 +250,12 @@ class WorkspaceFileAssetEndpoint(BaseAPIView):
# Check if the asset exists
if asset is None:
return
if not asset.is_uploaded:
release_file_asset_blob(asset, delete_untracked_object=True)
# Mark the asset as deleted
asset.is_deleted = True
asset.deleted_at = timezone.now()
asset.save(update_fields=["is_deleted", "deleted_at"])
asset.save(update_fields=["is_deleted", "deleted_at", "updated_at"])
return
def entity_asset_save(self, asset_id, entity_type, asset, request):
@@ -380,11 +393,17 @@ class WorkspaceFileAssetEndpoint(BaseAPIView):
def patch(self, request, slug, asset_id):
# get the asset id
asset = FileAsset.objects.get(id=asset_id, workspace__slug=slug)
# get the storage metadata
asset.is_uploaded = True
# get the storage metadata
if not asset.storage_metadata:
get_asset_object_metadata.delay(asset_id=str(asset_id))
try:
asset = confirm_uploaded_file_asset(
asset,
request=request,
attributes=request.data.get("attributes", asset.attributes),
)
except UploadedObjectMissing:
return Response(
{"error": "The uploaded asset object was not found.", "status": False},
status=status.HTTP_400_BAD_REQUEST,
)
# get the entity and save the asset id for the request field
self.entity_asset_save(
asset_id=asset_id,
@@ -392,19 +411,17 @@ class WorkspaceFileAssetEndpoint(BaseAPIView):
asset=asset,
request=request,
)
# update the attributes
asset.attributes = request.data.get("attributes", asset.attributes)
# save the asset
asset.save(update_fields=["is_uploaded", "attributes"])
return Response(status=status.HTTP_204_NO_CONTENT)
def delete(self, request, slug, asset_id):
asset = FileAsset.objects.get(id=asset_id, workspace__slug=slug)
if not asset.is_uploaded:
release_file_asset_blob(asset, request=request, delete_untracked_object=True)
asset.is_deleted = True
asset.deleted_at = timezone.now()
# get the entity and save the asset id for the request field
self.entity_asset_delete(entity_type=asset.entity_type, asset=asset, request=request)
asset.save(update_fields=["is_deleted", "deleted_at"])
asset.save(update_fields=["is_deleted", "deleted_at", "updated_at"])
return Response(status=status.HTTP_204_NO_CONTENT)
def get(self, request, slug, asset_id):
@@ -581,27 +598,30 @@ class ProjectAssetEndpoint(BaseAPIView):
def patch(self, request, slug, project_id, pk):
# get the asset id
asset = FileAsset.objects.get(id=pk, workspace__slug=slug, project_id=project_id)
# get the storage metadata
asset.is_uploaded = True
# get the storage metadata
if not asset.storage_metadata:
get_asset_object_metadata.delay(asset_id=str(pk))
# update the attributes
asset.attributes = request.data.get("attributes", asset.attributes)
# save the asset
asset.save(update_fields=["is_uploaded", "attributes"])
try:
confirm_uploaded_file_asset(
asset,
request=request,
attributes=request.data.get("attributes", asset.attributes),
)
except UploadedObjectMissing:
return Response(
{"error": "The uploaded asset object was not found.", "status": False},
status=status.HTTP_400_BAD_REQUEST,
)
return Response(status=status.HTTP_204_NO_CONTENT)
@allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST])
def delete(self, request, slug, project_id, pk):
# Get the asset
asset = FileAsset.objects.get(id=pk, workspace__slug=slug, project_id=project_id)
if not asset.is_uploaded:
release_file_asset_blob(asset, request=request, delete_untracked_object=True)
# Check deleted assets
asset.is_deleted = True
asset.deleted_at = timezone.now()
# Save the asset
asset.save(update_fields=["is_deleted", "deleted_at"])
asset.save(update_fields=["is_deleted", "deleted_at", "updated_at"])
return Response(status=status.HTTP_204_NO_CONTENT)
@allow_permission([ROLE.ADMIN, ROLE.MEMBER, ROLE.GUEST])
@@ -753,7 +773,7 @@ class DuplicateAssetEndpoint(BaseAPIView):
return Response({"error": "Project not found"}, status=status.HTTP_404_NOT_FOUND)
storage = S3Storage(request=request)
original_asset = FileAsset.objects.filter(id=asset_id, is_uploaded=True).first()
original_asset = FileAsset.objects.filter(id=asset_id, workspace=workspace, is_uploaded=True).first()
if not original_asset:
return Response({"error": "Asset not found"}, status=status.HTTP_404_NOT_FOUND)
@@ -774,9 +794,15 @@ class DuplicateAssetEndpoint(BaseAPIView):
storage_metadata=original_asset.storage_metadata,
**self.get_entity_id_field(entity_type=entity_type, entity_id=entity_id),
)
storage.copy_object(original_asset.asset, destination_key)
# Update the is_uploaded field for all newly created assets
FileAsset.objects.filter(id=duplicated_asset.id).update(is_uploaded=True)
if not attach_existing_blob_to_file_asset(original_asset, duplicated_asset):
storage.copy_object(original_asset.asset, destination_key)
try:
confirm_uploaded_file_asset(duplicated_asset, request=request)
except UploadedObjectMissing:
return Response(
{"error": "The source asset object was not found.", "status": False},
status=status.HTTP_400_BAD_REQUEST,
)
return Response({"asset_id": str(duplicated_asset.id)}, status=status.HTTP_200_OK)
plane-src/apps/api/plane/app/views/issue/attachment.py
+15 -9
View File
@@ -23,10 +23,10 @@ from plane.db.models import FileAsset, Workspace
from plane.bgtasks.issue_activities_task import issue_activity
from plane.app.permissions import allow_permission, ROLE
from plane.settings.storage import S3Storage
from plane.bgtasks.storage_metadata_task import get_asset_object_metadata
from plane.utils.host import base_host
from plane.utils.upload_limits import resolve_workspace_upload_size_limit
from plane.utils.attachment_preview import attachment_object_exists, get_attachment_preview_response
from plane.utils.file_dedup import finalize_uploaded_file_asset, release_file_asset_blob, UploadedObjectMissing
class IssueAttachmentEndpoint(BaseAPIView):
@@ -69,7 +69,9 @@ class IssueAttachmentEndpoint(BaseAPIView):
{"error": "Issue attachment not found."},
status=status.HTTP_404_NOT_FOUND,
)
issue_attachment.asset.delete(save=False)
release_result = release_file_asset_blob(issue_attachment, request=request)
if not release_result.had_blob:
issue_attachment.asset.delete(save=False)
issue_attachment.delete()
issue_activity.delay(
type="attachment.activity.deleted",
@@ -155,9 +157,11 @@ class IssueAttachmentV2Endpoint(BaseAPIView):
@allow_permission([ROLE.ADMIN], creator=True, model=FileAsset)
def delete(self, request, slug, project_id, issue_id, pk):
issue_attachment = FileAsset.objects.get(pk=pk, workspace__slug=slug, project_id=project_id)
if not issue_attachment.is_uploaded:
release_file_asset_blob(issue_attachment, request=request, delete_untracked_object=True)
issue_attachment.is_deleted = True
issue_attachment.deleted_at = timezone.now()
issue_attachment.save()
issue_attachment.save(update_fields=["is_deleted", "deleted_at", "updated_at"])
issue_activity.delay(
type="attachment.activity.deleted",
@@ -225,12 +229,14 @@ class IssueAttachmentV2Endpoint(BaseAPIView):
origin=base_host(request=request, is_app=True),
)
# Update the attachment
issue_attachment.is_uploaded = True
issue_attachment.created_by = request.user
issue_attachment.save(update_fields=["created_by", "updated_at"])
# Get the storage metadata
if not issue_attachment.storage_metadata:
get_asset_object_metadata.delay(str(issue_attachment.id))
issue_attachment.save()
try:
finalize_uploaded_file_asset(issue_attachment, request=request)
except UploadedObjectMissing:
return Response(
{"error": "The uploaded attachment object was not found.", "status": False},
status=status.HTTP_400_BAD_REQUEST,
)
return Response(status=status.HTTP_204_NO_CONTENT)