ФУНКЦИИ - МЕЖПРОЕКТНАЯ КОММУНИКАЦИЯ: очистка хранилища и проектные квоты

plane-src/apps/api/plane/app/views/__init__.py

@@ -83,7 +83,11 @@ from .workspace.module import WorkspaceModulesEndpoint
 from .workspace.cycle import WorkspaceCyclesEndpoint
 from .workspace.quick_link import QuickLinkViewSet
 from .workspace.sticky import WorkspaceStickyViewSet
-from .workspace.storage import WorkspaceStorageSummaryEndpoint
+from .workspace.storage import (
+    WorkspaceStorageMaintenanceEndpoint,
+    WorkspaceStorageProjectQuotaEndpoint,
+    WorkspaceStorageSummaryEndpoint,
+)
 
 from .state.base import StateViewSet, IntakeStateEndpoint
 from .view.base import (

plane-src/apps/api/plane/app/views/asset/v2.py

@@ -23,7 +23,7 @@ from plane.settings.storage import S3Storage
 from plane.app.permissions import allow_permission, ROLE
 from plane.utils.cache import invalidate_cache_directly
 from plane.throttles.asset import AssetRateThrottle
-from plane.utils.upload_limits import resolve_workspace_upload_size_limit
+from plane.utils.upload_limits import get_project_storage_quota_response, resolve_workspace_upload_size_limit
 from plane.utils.file_dedup import (
     UploadedObjectMissing,
     attach_existing_blob_to_file_asset,
@@ -361,6 +361,14 @@ class WorkspaceFileAssetEndpoint(BaseAPIView):
 
         # Get the size limit
         size_limit = resolve_workspace_upload_size_limit(workspace, size)
+        if entity_type == FileAsset.EntityTypeContext.PROJECT_COVER and entity_identifier:
+            quota_project = Project.objects.filter(id=entity_identifier, workspace=workspace).first()
+        else:
+            quota_project = None
+        if quota_project is not None:
+            quota_response = get_project_storage_quota_response(quota_project, requested_size=size)
+            if quota_response is not None:
+                return quota_response
 
         # asset key
         asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
@@ -564,6 +572,10 @@ class ProjectAssetEndpoint(BaseAPIView):
 
         # Get the size limit
         size_limit = resolve_workspace_upload_size_limit(workspace, size)
+        project = Project.objects.get(id=project_id, workspace=workspace)
+        quota_response = get_project_storage_quota_response(project, requested_size=size)
+        if quota_response is not None:
+            return quota_response
 
         # asset key
         asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
@@ -576,7 +588,7 @@ class ProjectAssetEndpoint(BaseAPIView):
             workspace=workspace,
             created_by=request.user,
             entity_type=entity_type,
-            project_id=project_id,
+            project_id=project.id,
             **self.get_entity_id_field(entity_type, entity_identifier),
         )
 
@@ -767,10 +779,6 @@ class DuplicateAssetEndpoint(BaseAPIView):
             )
 
         workspace = Workspace.objects.get(slug=slug)
-        if project_id:
-            # check if project exists in the workspace
-            if not Project.objects.filter(id=project_id, workspace=workspace).exists():
-                return Response({"error": "Project not found"}, status=status.HTTP_404_NOT_FOUND)
 
         storage = S3Storage(request=request)
         original_asset = FileAsset.objects.filter(id=asset_id, workspace=workspace, is_uploaded=True).first()
@@ -778,6 +786,14 @@ class DuplicateAssetEndpoint(BaseAPIView):
         if not original_asset:
             return Response({"error": "Asset not found"}, status=status.HTTP_404_NOT_FOUND)
 
+        if project_id:
+            project = Project.objects.filter(id=project_id, workspace=workspace).first()
+            if not project:
+                return Response({"error": "Project not found"}, status=status.HTTP_404_NOT_FOUND)
+            quota_response = get_project_storage_quota_response(project, requested_size=original_asset.size)
+            if quota_response is not None:
+                return quota_response
+
         destination_key = f"{workspace.id}/{uuid.uuid4().hex}-{original_asset.attributes.get('name')}"
         duplicated_asset = FileAsset.objects.create(
             attributes={

plane-src/apps/api/plane/app/views/issue/attachment.py

@@ -19,12 +19,12 @@ from rest_framework.parsers import MultiPartParser, FormParser
 # Module imports
 from .. import BaseAPIView
 from plane.app.serializers import IssueAttachmentSerializer
-from plane.db.models import FileAsset, Workspace
+from plane.db.models import FileAsset, Project, Workspace
 from plane.bgtasks.issue_activities_task import issue_activity
 from plane.app.permissions import allow_permission, ROLE
 from plane.settings.storage import S3Storage
 from plane.utils.host import base_host
-from plane.utils.upload_limits import resolve_workspace_upload_size_limit
+from plane.utils.upload_limits import get_project_storage_quota_response, resolve_workspace_upload_size_limit
 from plane.utils.attachment_preview import attachment_object_exists, get_attachment_preview_response
 from plane.utils.file_dedup import finalize_uploaded_file_asset, release_file_asset_blob, UploadedObjectMissing
 
@@ -118,12 +118,16 @@ class IssueAttachmentV2Endpoint(BaseAPIView):
 
         # Get the workspace
         workspace = Workspace.objects.get(slug=slug)
+        project = Project.objects.get(id=project_id, workspace=workspace)
 
         # asset key
         asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
 
         # Get the size limit
         size_limit = resolve_workspace_upload_size_limit(workspace, size)
+        quota_response = get_project_storage_quota_response(project, requested_size=size)
+        if quota_response is not None:
+            return quota_response
 
         # Create a File Asset
         asset = FileAsset.objects.create(

plane-src/apps/api/plane/app/views/workspace/storage.py

@@ -2,8 +2,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 # See the LICENSE file for details.
 
+import os
 from datetime import timedelta
 
+from django.conf import settings
 from django.db.models import Sum
 from django.utils import timezone
 from rest_framework import status
@@ -12,6 +14,8 @@ from rest_framework.response import Response
 from plane.app.permissions import ROLE, allow_permission
 from plane.app.views.base import BaseAPIView
 from plane.db.models import FileAsset, Project, StoredBlob, Workspace
+from plane.settings.storage import S3Storage
+from plane.utils.file_dedup import hard_delete_file_asset, release_file_asset_blob
 
 
 def _int_size(value):
@@ -33,24 +37,94 @@ def _dedup_savings(logical_size, physical_size):
     return max(_int_size(logical_size) - _int_size(physical_size), 0)
 
 
+def _retention_days(env_key, default):
+    try:
+        return max(int(os.environ.get(env_key, default)), 0)
+    except (TypeError, ValueError):
+        return int(default)
+
+
+def _storage_cutoffs():
+    now = timezone.now()
+    unuploaded_retention_days = _retention_days("UNUPLOADED_ASSET_DELETE_DAYS", 7)
+    soft_deleted_retention_days = _retention_days("FILE_ASSET_HARD_DELETE_AFTER_DAYS", settings.HARD_DELETE_AFTER_DAYS)
+
+    return {
+        "stale_upload_cutoff": now - timedelta(days=1),
+        "unuploaded_cleanup_cutoff": now - timedelta(days=unuploaded_retention_days),
+        "soft_deleted_cleanup_cutoff": now - timedelta(days=soft_deleted_retention_days),
+        "unuploaded_retention_days": unuploaded_retention_days,
+        "soft_deleted_retention_days": soft_deleted_retention_days,
+    }
+
+
+def _blob_size_queryset(queryset):
+    return _int_size(queryset.aggregate(total=Sum("size")).get("total"))
+
+
+def _blob_size_by_ids(blob_ids):
+    if not blob_ids:
+        return 0
+
+    return _blob_size_queryset(StoredBlob.objects.filter(id__in=blob_ids, status=StoredBlob.Status.ACTIVE))
+
+
+def _cleanup_snapshot(workspace, failed_uploads, soft_deleted_assets, orphaned_blobs):
+    cutoffs = _storage_cutoffs()
+    unuploaded_ready = failed_uploads.filter(created_at__lt=cutoffs["unuploaded_cleanup_cutoff"])
+    soft_deleted_ready = soft_deleted_assets.filter(deleted_at__lt=cutoffs["soft_deleted_cleanup_cutoff"])
+    orphaned_ready = orphaned_blobs.filter(ref_count=0)
+
+    return {
+        "unuploaded_retention_days": cutoffs["unuploaded_retention_days"],
+        "soft_deleted_retention_days": cutoffs["soft_deleted_retention_days"],
+        "unuploaded_ready_count": unuploaded_ready.count(),
+        "unuploaded_ready_size": _sum_asset_size(unuploaded_ready),
+        "soft_deleted_ready_count": soft_deleted_ready.count(),
+        "soft_deleted_ready_size": _sum_asset_size(soft_deleted_ready),
+        "orphaned_blob_ready_count": orphaned_ready.count(),
+        "orphaned_blob_ready_size": _blob_size_queryset(orphaned_ready),
+        "workspace_slug": workspace.slug,
+    }
+
+
+def _project_quota_snapshot(project, used_size):
+    enabled = bool(project.storage_quota_enabled and project.storage_quota > 0)
+    quota = int(project.storage_quota or 0) if enabled else 0
+    used = int(used_size or 0)
+    remaining = max(quota - used, 0) if enabled else None
+    percent = round((used / quota) * 100, 1) if quota > 0 else 0
+
+    return {
+        "quota_enabled": enabled,
+        "quota": quota,
+        "quota_used": used,
+        "quota_remaining": remaining,
+        "quota_percent": percent,
+        "quota_exceeded": enabled and used > quota,
+        "quota_warning": enabled and used <= quota and percent >= 80,
+    }
+
+
 class WorkspaceStorageSummaryEndpoint(BaseAPIView):
     @allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE")
     def get(self, request, slug):
         workspace = Workspace.objects.get(slug=slug)
-        stale_cutoff = timezone.now() - timedelta(days=1)
+        cutoffs = _storage_cutoffs()
 
         active_assets = FileAsset.objects.filter(workspace=workspace, is_uploaded=True)
+        failed_preview_assets = active_assets.filter(attributes__preview_status="failed")
         active_blob_ids = list(active_assets.exclude(blob__isnull=True).values_list("blob_id", flat=True).distinct())
 
         workspace_logical_size = _sum_asset_size(active_assets)
-        workspace_physical_size = _sum_blob_size(active_blob_ids)
+        workspace_physical_size = _blob_size_by_ids(active_blob_ids)
 
         failed_uploads = FileAsset.all_objects.filter(
             workspace=workspace,
             is_uploaded=False,
             deleted_at__isnull=True,
         )
-        stale_unuploaded = failed_uploads.filter(created_at__lt=stale_cutoff)
+        stale_unuploaded = failed_uploads.filter(created_at__lt=cutoffs["stale_upload_cutoff"])
         soft_deleted_assets = FileAsset.all_objects.filter(workspace=workspace, deleted_at__isnull=False)
         orphaned_blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.ORPHANED)
         missing_blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.MISSING)
@@ -64,8 +138,9 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView):
                 project_assets.exclude(blob__isnull=True).values_list("blob_id", flat=True).distinct()
             )
             project_logical_size = _sum_asset_size(project_assets)
-            project_physical_size = _sum_blob_size(project_blob_ids)
+            project_physical_size = _blob_size_by_ids(project_blob_ids)
             project_failed_uploads = failed_uploads.filter(project=project)
+            project_failed_previews = failed_preview_assets.filter(project=project)
             project_soft_deleted = soft_deleted_assets.filter(project=project)
             project_uploaded_without_blob = uploaded_without_blob.filter(project=project)
 
@@ -81,9 +156,15 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView):
                     "dedup_savings": _dedup_savings(project_logical_size, project_physical_size),
                     "failed_upload_count": project_failed_uploads.count(),
                     "failed_upload_size": _sum_asset_size(project_failed_uploads),
+                    "failed_preview_count": project_failed_previews.count(),
+                    "failed_preview_size": _sum_asset_size(project_failed_previews),
+                    "stale_unuploaded_count": project_failed_uploads.filter(
+                        created_at__lt=cutoffs["stale_upload_cutoff"]
+                    ).count(),
                     "soft_deleted_count": project_soft_deleted.count(),
                     "soft_deleted_size": _sum_asset_size(project_soft_deleted),
                     "uploaded_without_blob_count": project_uploaded_without_blob.count(),
+                    **_project_quota_snapshot(project, project_logical_size),
                 }
             )
 
@@ -114,8 +195,160 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView):
                 "orphaned_blob_size": _sum_blob_size(list(orphaned_blobs.values_list("id", flat=True))),
                 "missing_blob_count": missing_blobs.count(),
                 "missing_blob_size": _sum_blob_size(list(missing_blobs.values_list("id", flat=True))),
+                "failed_preview_count": failed_preview_assets.count(),
+                "failed_preview_size": _sum_asset_size(failed_preview_assets),
+                "failed_preview_instrumented": True,
+                "quota_warning_project_count": sum(1 for project in project_rows if project["quota_warning"]),
+                "quota_exceeded_project_count": sum(1 for project in project_rows if project["quota_exceeded"]),
             },
+            "cleanup": _cleanup_snapshot(workspace, failed_uploads, soft_deleted_assets, orphaned_blobs),
             "projects": project_rows,
         }
 
         return Response(data, status=status.HTTP_200_OK)
+
+
+class WorkspaceStorageProjectQuotaEndpoint(BaseAPIView):
+    @allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE")
+    def patch(self, request, slug, project_id):
+        workspace = Workspace.objects.get(slug=slug)
+        project = Project.objects.get(id=project_id, workspace=workspace)
+
+        enabled = bool(request.data.get("quota_enabled", False))
+        try:
+            quota = int(request.data.get("quota", 0) or 0)
+        except (TypeError, ValueError):
+            quota = 0
+
+        project.storage_quota_enabled = enabled and quota > 0
+        project.storage_quota = max(quota, 0)
+        project.save(update_fields=["storage_quota_enabled", "storage_quota", "updated_at"])
+
+        return Response(
+            {
+                "id": str(project.id),
+                **_project_quota_snapshot(
+                    project,
+                    _sum_asset_size(FileAsset.objects.filter(project=project, is_uploaded=True)),
+                ),
+            },
+            status=status.HTTP_200_OK,
+        )
+
+
+class WorkspaceStorageMaintenanceEndpoint(BaseAPIView):
+    @allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE")
+    def post(self, request, slug):
+        workspace = Workspace.objects.get(slug=slug)
+        action = request.data.get("action")
+
+        if action == "scan_missing_blobs":
+            return Response(self._scan_missing_blobs(request, workspace), status=status.HTTP_200_OK)
+
+        if action == "purge_stale_unuploaded":
+            return Response(self._purge_stale_unuploaded(request, workspace), status=status.HTTP_200_OK)
+
+        if action == "purge_expired_deleted":
+            return Response(self._purge_expired_deleted(request, workspace), status=status.HTTP_200_OK)
+
+        if action == "purge_orphaned_blobs":
+            return Response(self._purge_orphaned_blobs(request, workspace), status=status.HTTP_200_OK)
+
+        return Response({"error": "Unsupported storage maintenance action."}, status=status.HTTP_400_BAD_REQUEST)
+
+    def _scan_missing_blobs(self, request, workspace):
+        storage = S3Storage(request=request, is_server=True)
+        blobs = StoredBlob.objects.filter(
+            workspace=workspace,
+            status__in=[StoredBlob.Status.ACTIVE, StoredBlob.Status.MISSING],
+        )
+        scanned = 0
+        missing = 0
+        restored = 0
+
+        for blob in blobs.iterator():
+            scanned += 1
+            metadata = storage.get_object_metadata(object_name=blob.canonical_object_key)
+            if metadata:
+                if blob.status == StoredBlob.Status.MISSING:
+                    restored += 1
+                blob.status = StoredBlob.Status.ACTIVE
+                blob.storage_metadata = metadata
+                blob.save(update_fields=["status", "storage_metadata", "updated_at"])
+                continue
+
+            if blob.status != StoredBlob.Status.MISSING:
+                missing += 1
+            blob.status = StoredBlob.Status.MISSING
+            blob.save(update_fields=["status", "updated_at"])
+
+        return {
+            "action": "scan_missing_blobs",
+            "scanned_count": scanned,
+            "missing_count": missing,
+            "restored_count": restored,
+        }
+
+    def _purge_stale_unuploaded(self, request, workspace):
+        cutoff = _storage_cutoffs()["unuploaded_cleanup_cutoff"]
+        assets = FileAsset.objects.filter(workspace=workspace, is_uploaded=False, created_at__lt=cutoff)
+        deleted_assets = 0
+        deleted_objects = 0
+        released_size = 0
+
+        for asset in assets.iterator():
+            released_size += _int_size(asset.size)
+            release_result = release_file_asset_blob(asset, request=request, delete_untracked_object=True)
+            if release_result.deleted_object:
+                deleted_objects += 1
+            asset.delete(soft=False)
+            deleted_assets += 1
+
+        return {
+            "action": "purge_stale_unuploaded",
+            "deleted_asset_count": deleted_assets,
+            "deleted_object_count": deleted_objects,
+            "released_size": released_size,
+        }
+
+    def _purge_expired_deleted(self, request, workspace):
+        cutoff = _storage_cutoffs()["soft_deleted_cleanup_cutoff"]
+        assets = FileAsset.all_objects.filter(workspace=workspace, deleted_at__lt=cutoff)
+        storage = S3Storage(request=request, is_server=True)
+        deleted_assets = 0
+        deleted_objects = 0
+        released_size = 0
+
+        for asset in assets.iterator():
+            released_size += _int_size(asset.size)
+            if hard_delete_file_asset(asset, request=request, storage=storage):
+                deleted_objects += 1
+            deleted_assets += 1
+
+        return {
+            "action": "purge_expired_deleted",
+            "deleted_asset_count": deleted_assets,
+            "deleted_object_count": deleted_objects,
+            "released_size": released_size,
+        }
+
+    def _purge_orphaned_blobs(self, request, workspace):
+        storage = S3Storage(request=request, is_server=True)
+        blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.ORPHANED, ref_count=0)
+        deleted_blobs = 0
+        deleted_objects = 0
+        released_size = 0
+
+        for blob in blobs.iterator():
+            released_size += _int_size(blob.size)
+            if storage.delete_files(object_names=[blob.canonical_object_key]):
+                deleted_objects += 1
+            blob.delete(soft=False)
+            deleted_blobs += 1
+
+        return {
+            "action": "purge_orphaned_blobs",
+            "deleted_blob_count": deleted_blobs,
+            "deleted_object_count": deleted_objects,
+            "released_size": released_size,
+        }