diff --git a/plane-src/apps/api/plane/app/urls/workspace.py b/plane-src/apps/api/plane/app/urls/workspace.py index bb6b432..437ce14 100644 --- a/plane-src/apps/api/plane/app/urls/workspace.py +++ b/plane-src/apps/api/plane/app/urls/workspace.py @@ -36,6 +36,8 @@ from plane.app.views import ( UserRecentVisitViewSet, WorkspaceHomePreferenceViewSet, WorkspaceStickyViewSet, + WorkspaceStorageMaintenanceEndpoint, + WorkspaceStorageProjectQuotaEndpoint, WorkspaceStorageSummaryEndpoint, WorkspaceUserPreferenceViewSet, ) @@ -263,6 +265,16 @@ urlpatterns = [ WorkspaceStorageSummaryEndpoint.as_view(), name="workspace-storage-summary", ), + path( + "workspaces//storage/maintenance/", + WorkspaceStorageMaintenanceEndpoint.as_view(), + name="workspace-storage-maintenance", + ), + path( + "workspaces//storage/projects//quota/", + WorkspaceStorageProjectQuotaEndpoint.as_view(), + name="workspace-storage-project-quota", + ), # User Preference path( "workspaces//sidebar-preferences/", diff --git a/plane-src/apps/api/plane/app/views/__init__.py b/plane-src/apps/api/plane/app/views/__init__.py index 95e058c..a41bdff 100644 --- a/plane-src/apps/api/plane/app/views/__init__.py +++ b/plane-src/apps/api/plane/app/views/__init__.py @@ -83,7 +83,11 @@ from .workspace.module import WorkspaceModulesEndpoint from .workspace.cycle import WorkspaceCyclesEndpoint from .workspace.quick_link import QuickLinkViewSet from .workspace.sticky import WorkspaceStickyViewSet -from .workspace.storage import WorkspaceStorageSummaryEndpoint +from .workspace.storage import ( + WorkspaceStorageMaintenanceEndpoint, + WorkspaceStorageProjectQuotaEndpoint, + WorkspaceStorageSummaryEndpoint, +) from .state.base import StateViewSet, IntakeStateEndpoint from .view.base import ( diff --git a/plane-src/apps/api/plane/app/views/asset/v2.py b/plane-src/apps/api/plane/app/views/asset/v2.py index 0b55a7a..3cff428 100644 --- a/plane-src/apps/api/plane/app/views/asset/v2.py +++ b/plane-src/apps/api/plane/app/views/asset/v2.py @@ -23,7 +23,7 @@ from plane.settings.storage import S3Storage from plane.app.permissions import allow_permission, ROLE from plane.utils.cache import invalidate_cache_directly from plane.throttles.asset import AssetRateThrottle -from plane.utils.upload_limits import resolve_workspace_upload_size_limit +from plane.utils.upload_limits import get_project_storage_quota_response, resolve_workspace_upload_size_limit from plane.utils.file_dedup import ( UploadedObjectMissing, attach_existing_blob_to_file_asset, @@ -361,6 +361,14 @@ class WorkspaceFileAssetEndpoint(BaseAPIView): # Get the size limit size_limit = resolve_workspace_upload_size_limit(workspace, size) + if entity_type == FileAsset.EntityTypeContext.PROJECT_COVER and entity_identifier: + quota_project = Project.objects.filter(id=entity_identifier, workspace=workspace).first() + else: + quota_project = None + if quota_project is not None: + quota_response = get_project_storage_quota_response(quota_project, requested_size=size) + if quota_response is not None: + return quota_response # asset key asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}" @@ -564,6 +572,10 @@ class ProjectAssetEndpoint(BaseAPIView): # Get the size limit size_limit = resolve_workspace_upload_size_limit(workspace, size) + project = Project.objects.get(id=project_id, workspace=workspace) + quota_response = get_project_storage_quota_response(project, requested_size=size) + if quota_response is not None: + return quota_response # asset key asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}" @@ -576,7 +588,7 @@ class ProjectAssetEndpoint(BaseAPIView): workspace=workspace, created_by=request.user, entity_type=entity_type, - project_id=project_id, + project_id=project.id, **self.get_entity_id_field(entity_type, entity_identifier), ) @@ -767,10 +779,6 @@ class DuplicateAssetEndpoint(BaseAPIView): ) workspace = Workspace.objects.get(slug=slug) - if project_id: - # check if project exists in the workspace - if not Project.objects.filter(id=project_id, workspace=workspace).exists(): - return Response({"error": "Project not found"}, status=status.HTTP_404_NOT_FOUND) storage = S3Storage(request=request) original_asset = FileAsset.objects.filter(id=asset_id, workspace=workspace, is_uploaded=True).first() @@ -778,6 +786,14 @@ class DuplicateAssetEndpoint(BaseAPIView): if not original_asset: return Response({"error": "Asset not found"}, status=status.HTTP_404_NOT_FOUND) + if project_id: + project = Project.objects.filter(id=project_id, workspace=workspace).first() + if not project: + return Response({"error": "Project not found"}, status=status.HTTP_404_NOT_FOUND) + quota_response = get_project_storage_quota_response(project, requested_size=original_asset.size) + if quota_response is not None: + return quota_response + destination_key = f"{workspace.id}/{uuid.uuid4().hex}-{original_asset.attributes.get('name')}" duplicated_asset = FileAsset.objects.create( attributes={ diff --git a/plane-src/apps/api/plane/app/views/issue/attachment.py b/plane-src/apps/api/plane/app/views/issue/attachment.py index f825ac2..928b5c6 100644 --- a/plane-src/apps/api/plane/app/views/issue/attachment.py +++ b/plane-src/apps/api/plane/app/views/issue/attachment.py @@ -19,12 +19,12 @@ from rest_framework.parsers import MultiPartParser, FormParser # Module imports from .. import BaseAPIView from plane.app.serializers import IssueAttachmentSerializer -from plane.db.models import FileAsset, Workspace +from plane.db.models import FileAsset, Project, Workspace from plane.bgtasks.issue_activities_task import issue_activity from plane.app.permissions import allow_permission, ROLE from plane.settings.storage import S3Storage from plane.utils.host import base_host -from plane.utils.upload_limits import resolve_workspace_upload_size_limit +from plane.utils.upload_limits import get_project_storage_quota_response, resolve_workspace_upload_size_limit from plane.utils.attachment_preview import attachment_object_exists, get_attachment_preview_response from plane.utils.file_dedup import finalize_uploaded_file_asset, release_file_asset_blob, UploadedObjectMissing @@ -118,12 +118,16 @@ class IssueAttachmentV2Endpoint(BaseAPIView): # Get the workspace workspace = Workspace.objects.get(slug=slug) + project = Project.objects.get(id=project_id, workspace=workspace) # asset key asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}" # Get the size limit size_limit = resolve_workspace_upload_size_limit(workspace, size) + quota_response = get_project_storage_quota_response(project, requested_size=size) + if quota_response is not None: + return quota_response # Create a File Asset asset = FileAsset.objects.create( diff --git a/plane-src/apps/api/plane/app/views/workspace/storage.py b/plane-src/apps/api/plane/app/views/workspace/storage.py index 45a23d3..9e98c36 100644 --- a/plane-src/apps/api/plane/app/views/workspace/storage.py +++ b/plane-src/apps/api/plane/app/views/workspace/storage.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: AGPL-3.0-only # See the LICENSE file for details. +import os from datetime import timedelta +from django.conf import settings from django.db.models import Sum from django.utils import timezone from rest_framework import status @@ -12,6 +14,8 @@ from rest_framework.response import Response from plane.app.permissions import ROLE, allow_permission from plane.app.views.base import BaseAPIView from plane.db.models import FileAsset, Project, StoredBlob, Workspace +from plane.settings.storage import S3Storage +from plane.utils.file_dedup import hard_delete_file_asset, release_file_asset_blob def _int_size(value): @@ -33,24 +37,94 @@ def _dedup_savings(logical_size, physical_size): return max(_int_size(logical_size) - _int_size(physical_size), 0) +def _retention_days(env_key, default): + try: + return max(int(os.environ.get(env_key, default)), 0) + except (TypeError, ValueError): + return int(default) + + +def _storage_cutoffs(): + now = timezone.now() + unuploaded_retention_days = _retention_days("UNUPLOADED_ASSET_DELETE_DAYS", 7) + soft_deleted_retention_days = _retention_days("FILE_ASSET_HARD_DELETE_AFTER_DAYS", settings.HARD_DELETE_AFTER_DAYS) + + return { + "stale_upload_cutoff": now - timedelta(days=1), + "unuploaded_cleanup_cutoff": now - timedelta(days=unuploaded_retention_days), + "soft_deleted_cleanup_cutoff": now - timedelta(days=soft_deleted_retention_days), + "unuploaded_retention_days": unuploaded_retention_days, + "soft_deleted_retention_days": soft_deleted_retention_days, + } + + +def _blob_size_queryset(queryset): + return _int_size(queryset.aggregate(total=Sum("size")).get("total")) + + +def _blob_size_by_ids(blob_ids): + if not blob_ids: + return 0 + + return _blob_size_queryset(StoredBlob.objects.filter(id__in=blob_ids, status=StoredBlob.Status.ACTIVE)) + + +def _cleanup_snapshot(workspace, failed_uploads, soft_deleted_assets, orphaned_blobs): + cutoffs = _storage_cutoffs() + unuploaded_ready = failed_uploads.filter(created_at__lt=cutoffs["unuploaded_cleanup_cutoff"]) + soft_deleted_ready = soft_deleted_assets.filter(deleted_at__lt=cutoffs["soft_deleted_cleanup_cutoff"]) + orphaned_ready = orphaned_blobs.filter(ref_count=0) + + return { + "unuploaded_retention_days": cutoffs["unuploaded_retention_days"], + "soft_deleted_retention_days": cutoffs["soft_deleted_retention_days"], + "unuploaded_ready_count": unuploaded_ready.count(), + "unuploaded_ready_size": _sum_asset_size(unuploaded_ready), + "soft_deleted_ready_count": soft_deleted_ready.count(), + "soft_deleted_ready_size": _sum_asset_size(soft_deleted_ready), + "orphaned_blob_ready_count": orphaned_ready.count(), + "orphaned_blob_ready_size": _blob_size_queryset(orphaned_ready), + "workspace_slug": workspace.slug, + } + + +def _project_quota_snapshot(project, used_size): + enabled = bool(project.storage_quota_enabled and project.storage_quota > 0) + quota = int(project.storage_quota or 0) if enabled else 0 + used = int(used_size or 0) + remaining = max(quota - used, 0) if enabled else None + percent = round((used / quota) * 100, 1) if quota > 0 else 0 + + return { + "quota_enabled": enabled, + "quota": quota, + "quota_used": used, + "quota_remaining": remaining, + "quota_percent": percent, + "quota_exceeded": enabled and used > quota, + "quota_warning": enabled and used <= quota and percent >= 80, + } + + class WorkspaceStorageSummaryEndpoint(BaseAPIView): @allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE") def get(self, request, slug): workspace = Workspace.objects.get(slug=slug) - stale_cutoff = timezone.now() - timedelta(days=1) + cutoffs = _storage_cutoffs() active_assets = FileAsset.objects.filter(workspace=workspace, is_uploaded=True) + failed_preview_assets = active_assets.filter(attributes__preview_status="failed") active_blob_ids = list(active_assets.exclude(blob__isnull=True).values_list("blob_id", flat=True).distinct()) workspace_logical_size = _sum_asset_size(active_assets) - workspace_physical_size = _sum_blob_size(active_blob_ids) + workspace_physical_size = _blob_size_by_ids(active_blob_ids) failed_uploads = FileAsset.all_objects.filter( workspace=workspace, is_uploaded=False, deleted_at__isnull=True, ) - stale_unuploaded = failed_uploads.filter(created_at__lt=stale_cutoff) + stale_unuploaded = failed_uploads.filter(created_at__lt=cutoffs["stale_upload_cutoff"]) soft_deleted_assets = FileAsset.all_objects.filter(workspace=workspace, deleted_at__isnull=False) orphaned_blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.ORPHANED) missing_blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.MISSING) @@ -64,8 +138,9 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView): project_assets.exclude(blob__isnull=True).values_list("blob_id", flat=True).distinct() ) project_logical_size = _sum_asset_size(project_assets) - project_physical_size = _sum_blob_size(project_blob_ids) + project_physical_size = _blob_size_by_ids(project_blob_ids) project_failed_uploads = failed_uploads.filter(project=project) + project_failed_previews = failed_preview_assets.filter(project=project) project_soft_deleted = soft_deleted_assets.filter(project=project) project_uploaded_without_blob = uploaded_without_blob.filter(project=project) @@ -81,9 +156,15 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView): "dedup_savings": _dedup_savings(project_logical_size, project_physical_size), "failed_upload_count": project_failed_uploads.count(), "failed_upload_size": _sum_asset_size(project_failed_uploads), + "failed_preview_count": project_failed_previews.count(), + "failed_preview_size": _sum_asset_size(project_failed_previews), + "stale_unuploaded_count": project_failed_uploads.filter( + created_at__lt=cutoffs["stale_upload_cutoff"] + ).count(), "soft_deleted_count": project_soft_deleted.count(), "soft_deleted_size": _sum_asset_size(project_soft_deleted), "uploaded_without_blob_count": project_uploaded_without_blob.count(), + **_project_quota_snapshot(project, project_logical_size), } ) @@ -114,8 +195,160 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView): "orphaned_blob_size": _sum_blob_size(list(orphaned_blobs.values_list("id", flat=True))), "missing_blob_count": missing_blobs.count(), "missing_blob_size": _sum_blob_size(list(missing_blobs.values_list("id", flat=True))), + "failed_preview_count": failed_preview_assets.count(), + "failed_preview_size": _sum_asset_size(failed_preview_assets), + "failed_preview_instrumented": True, + "quota_warning_project_count": sum(1 for project in project_rows if project["quota_warning"]), + "quota_exceeded_project_count": sum(1 for project in project_rows if project["quota_exceeded"]), }, + "cleanup": _cleanup_snapshot(workspace, failed_uploads, soft_deleted_assets, orphaned_blobs), "projects": project_rows, } return Response(data, status=status.HTTP_200_OK) + + +class WorkspaceStorageProjectQuotaEndpoint(BaseAPIView): + @allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE") + def patch(self, request, slug, project_id): + workspace = Workspace.objects.get(slug=slug) + project = Project.objects.get(id=project_id, workspace=workspace) + + enabled = bool(request.data.get("quota_enabled", False)) + try: + quota = int(request.data.get("quota", 0) or 0) + except (TypeError, ValueError): + quota = 0 + + project.storage_quota_enabled = enabled and quota > 0 + project.storage_quota = max(quota, 0) + project.save(update_fields=["storage_quota_enabled", "storage_quota", "updated_at"]) + + return Response( + { + "id": str(project.id), + **_project_quota_snapshot( + project, + _sum_asset_size(FileAsset.objects.filter(project=project, is_uploaded=True)), + ), + }, + status=status.HTTP_200_OK, + ) + + +class WorkspaceStorageMaintenanceEndpoint(BaseAPIView): + @allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE") + def post(self, request, slug): + workspace = Workspace.objects.get(slug=slug) + action = request.data.get("action") + + if action == "scan_missing_blobs": + return Response(self._scan_missing_blobs(request, workspace), status=status.HTTP_200_OK) + + if action == "purge_stale_unuploaded": + return Response(self._purge_stale_unuploaded(request, workspace), status=status.HTTP_200_OK) + + if action == "purge_expired_deleted": + return Response(self._purge_expired_deleted(request, workspace), status=status.HTTP_200_OK) + + if action == "purge_orphaned_blobs": + return Response(self._purge_orphaned_blobs(request, workspace), status=status.HTTP_200_OK) + + return Response({"error": "Unsupported storage maintenance action."}, status=status.HTTP_400_BAD_REQUEST) + + def _scan_missing_blobs(self, request, workspace): + storage = S3Storage(request=request, is_server=True) + blobs = StoredBlob.objects.filter( + workspace=workspace, + status__in=[StoredBlob.Status.ACTIVE, StoredBlob.Status.MISSING], + ) + scanned = 0 + missing = 0 + restored = 0 + + for blob in blobs.iterator(): + scanned += 1 + metadata = storage.get_object_metadata(object_name=blob.canonical_object_key) + if metadata: + if blob.status == StoredBlob.Status.MISSING: + restored += 1 + blob.status = StoredBlob.Status.ACTIVE + blob.storage_metadata = metadata + blob.save(update_fields=["status", "storage_metadata", "updated_at"]) + continue + + if blob.status != StoredBlob.Status.MISSING: + missing += 1 + blob.status = StoredBlob.Status.MISSING + blob.save(update_fields=["status", "updated_at"]) + + return { + "action": "scan_missing_blobs", + "scanned_count": scanned, + "missing_count": missing, + "restored_count": restored, + } + + def _purge_stale_unuploaded(self, request, workspace): + cutoff = _storage_cutoffs()["unuploaded_cleanup_cutoff"] + assets = FileAsset.objects.filter(workspace=workspace, is_uploaded=False, created_at__lt=cutoff) + deleted_assets = 0 + deleted_objects = 0 + released_size = 0 + + for asset in assets.iterator(): + released_size += _int_size(asset.size) + release_result = release_file_asset_blob(asset, request=request, delete_untracked_object=True) + if release_result.deleted_object: + deleted_objects += 1 + asset.delete(soft=False) + deleted_assets += 1 + + return { + "action": "purge_stale_unuploaded", + "deleted_asset_count": deleted_assets, + "deleted_object_count": deleted_objects, + "released_size": released_size, + } + + def _purge_expired_deleted(self, request, workspace): + cutoff = _storage_cutoffs()["soft_deleted_cleanup_cutoff"] + assets = FileAsset.all_objects.filter(workspace=workspace, deleted_at__lt=cutoff) + storage = S3Storage(request=request, is_server=True) + deleted_assets = 0 + deleted_objects = 0 + released_size = 0 + + for asset in assets.iterator(): + released_size += _int_size(asset.size) + if hard_delete_file_asset(asset, request=request, storage=storage): + deleted_objects += 1 + deleted_assets += 1 + + return { + "action": "purge_expired_deleted", + "deleted_asset_count": deleted_assets, + "deleted_object_count": deleted_objects, + "released_size": released_size, + } + + def _purge_orphaned_blobs(self, request, workspace): + storage = S3Storage(request=request, is_server=True) + blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.ORPHANED, ref_count=0) + deleted_blobs = 0 + deleted_objects = 0 + released_size = 0 + + for blob in blobs.iterator(): + released_size += _int_size(blob.size) + if storage.delete_files(object_names=[blob.canonical_object_key]): + deleted_objects += 1 + blob.delete(soft=False) + deleted_blobs += 1 + + return { + "action": "purge_orphaned_blobs", + "deleted_blob_count": deleted_blobs, + "deleted_object_count": deleted_objects, + "released_size": released_size, + } diff --git a/plane-src/apps/api/plane/bgtasks/file_asset_task.py b/plane-src/apps/api/plane/bgtasks/file_asset_task.py index 577c7ab..209fd5f 100644 --- a/plane-src/apps/api/plane/bgtasks/file_asset_task.py +++ b/plane-src/apps/api/plane/bgtasks/file_asset_task.py @@ -15,42 +15,9 @@ from django.db.models import Q from celery import shared_task # Module imports -from plane.db.models import FileAsset, StoredBlob +from plane.db.models import FileAsset from plane.settings.storage import S3Storage -from plane.utils.file_dedup import release_file_asset_blob - - -def _asset_object_key(asset): - return str(asset.asset.name or asset.asset) - - -def _delete_legacy_asset_object(asset, storage): - object_key = _asset_object_key(asset) - if not object_key: - return False - - has_other_reference = FileAsset.all_objects.filter(asset=object_key).exclude(pk=asset.pk).exists() - if has_other_reference: - return False - - return storage.delete_files(object_names=[object_key]) - - -def _hard_delete_file_asset(asset, storage=None): - blob_id = asset.blob_id - - if blob_id: - release_file_asset_blob(asset) - StoredBlob.all_objects.filter( - pk=blob_id, - ref_count=0, - status=StoredBlob.Status.ORPHANED, - ).delete() - else: - storage = storage or S3Storage() - _delete_legacy_asset_object(asset, storage) - - asset.delete(soft=False) +from plane.utils.file_dedup import hard_delete_file_asset, release_file_asset_blob @shared_task @@ -74,4 +41,4 @@ def delete_expired_file_asset(): expired_assets = FileAsset.all_objects.filter(deleted_at__lt=cutoff) for asset in expired_assets.iterator(): - _hard_delete_file_asset(asset, storage=storage) + hard_delete_file_asset(asset, storage=storage) diff --git a/plane-src/apps/api/plane/db/migrations/0130_project_storage_quotas.py b/plane-src/apps/api/plane/db/migrations/0130_project_storage_quotas.py new file mode 100644 index 0000000..335a215 --- /dev/null +++ b/plane-src/apps/api/plane/db/migrations/0130_project_storage_quotas.py @@ -0,0 +1,22 @@ +# Generated by Plane on 2026-04-28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("db", "0129_workspace_ai_access_scope"), + ] + + operations = [ + migrations.AddField( + model_name="project", + name="storage_quota_enabled", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="project", + name="storage_quota", + field=models.PositiveBigIntegerField(default=0), + ), + ] diff --git a/plane-src/apps/api/plane/db/models/project.py b/plane-src/apps/api/plane/db/models/project.py index 4039b1d..c1ce66b 100644 --- a/plane-src/apps/api/plane/db/models/project.py +++ b/plane-src/apps/api/plane/db/models/project.py @@ -98,6 +98,8 @@ class Project(BaseModel): is_time_tracking_enabled = models.BooleanField(default=False) is_issue_type_enabled = models.BooleanField(default=False) guest_view_all_features = models.BooleanField(default=False) + storage_quota_enabled = models.BooleanField(default=False) + storage_quota = models.PositiveBigIntegerField(default=0) cover_image = models.TextField(blank=True, null=True) cover_image_asset = models.ForeignKey( "db.FileAsset", diff --git a/plane-src/apps/api/plane/utils/attachment_preview.py b/plane-src/apps/api/plane/utils/attachment_preview.py index feafa90..041b30e 100644 --- a/plane-src/apps/api/plane/utils/attachment_preview.py +++ b/plane-src/apps/api/plane/utils/attachment_preview.py @@ -6,10 +6,28 @@ from urllib.parse import quote from botocore.exceptions import ClientError from django.http import HttpResponse, StreamingHttpResponse +from django.utils import timezone from plane.settings.storage import S3Storage +def _mark_preview_state(asset, status, error=None): + attributes = dict(asset.attributes or {}) + + if status == "failed": + attributes["preview_status"] = "failed" + attributes["preview_failed_at"] = timezone.now().isoformat() + if error: + attributes["preview_error"] = str(error) + else: + attributes.pop("preview_failed_at", None) + attributes.pop("preview_error", None) + attributes["preview_status"] = "ok" + + asset.attributes = attributes + asset.save(update_fields=["attributes", "updated_at"]) + + def attachment_object_exists(asset): storage = S3Storage(request=None) try: @@ -34,7 +52,11 @@ def get_attachment_preview_response(request, asset, disposition="inline"): try: storage_response = storage.s3_client.get_object(**request_kwargs) - except ClientError: + if disposition == "inline" and asset.attributes.get("preview_status") == "failed": + _mark_preview_state(asset, "ok") + except ClientError as exc: + if disposition == "inline": + _mark_preview_state(asset, "failed", error=exc.response.get("Error", {}).get("Code", exc)) return HttpResponse("Attachment object not found.", status=404) content_type = ( diff --git a/plane-src/apps/api/plane/utils/file_dedup.py b/plane-src/apps/api/plane/utils/file_dedup.py index f387ca5..3f28a86 100644 --- a/plane-src/apps/api/plane/utils/file_dedup.py +++ b/plane-src/apps/api/plane/utils/file_dedup.py @@ -277,6 +277,46 @@ def release_file_asset_blob( return FileBlobReleaseResult(had_blob=had_blob, deleted_object=deleted_object, object_key=delete_object_key) +def _delete_legacy_asset_object(asset: FileAsset, storage: S3Storage) -> bool: + object_key = _object_key(asset) + if not object_key: + return False + + has_other_reference = FileAsset.all_objects.filter(asset=object_key).exclude(pk=asset.pk).exists() + if has_other_reference: + return False + + return storage.delete_files(object_names=[object_key]) + + +def hard_delete_file_asset(asset: FileAsset, request=None, storage: S3Storage | None = None) -> bool: + """ + Permanently remove a FileAsset row and release its storage reference. + + Blob-backed assets release the StoredBlob ref-count. Legacy non-blob assets + are physically deleted only when no other FileAsset row still points at the + same object key. + """ + + storage = storage or _server_storage(request=request) + blob_id = asset.blob_id + deleted_object = False + + if blob_id: + release_result = release_file_asset_blob(asset, request=request) + deleted_object = release_result.deleted_object + StoredBlob.all_objects.filter( + pk=blob_id, + ref_count=0, + status=StoredBlob.Status.ORPHANED, + ).delete() + else: + deleted_object = _delete_legacy_asset_object(asset, storage) + + asset.delete(soft=False) + return deleted_object + + def attach_existing_blob_to_file_asset(source: FileAsset, target: FileAsset) -> bool: if not source.blob_id: source.refresh_from_db(fields=["blob"]) diff --git a/plane-src/apps/api/plane/utils/upload_limits.py b/plane-src/apps/api/plane/utils/upload_limits.py index 5f7eb0e..a9b1eb8 100644 --- a/plane-src/apps/api/plane/utils/upload_limits.py +++ b/plane-src/apps/api/plane/utils/upload_limits.py @@ -1,4 +1,7 @@ from django.conf import settings +from django.db.models import Sum +from rest_framework import status +from rest_framework.response import Response def get_workspace_file_size_limit(workspace): @@ -22,3 +25,71 @@ def resolve_workspace_upload_size_limit(workspace, requested_size): return requested_size return min(requested_size, workspace_limit) + + +def get_project_storage_quota(project): + if not getattr(project, "storage_quota_enabled", False): + return None + + quota = getattr(project, "storage_quota", 0) or 0 + return max(int(quota), 0) or None + + +def get_project_storage_usage(project): + from plane.db.models import FileAsset + + total = ( + FileAsset.objects.filter( + workspace_id=project.workspace_id, + project_id=project.id, + is_uploaded=True, + ) + .aggregate(total=Sum("size")) + .get("total") + ) + return int(total or 0) + + +def get_project_storage_quota_state(project, requested_size=0): + quota = get_project_storage_quota(project) + used = get_project_storage_usage(project) + requested = max(int(requested_size or 0), 0) + + if quota is None: + return { + "enabled": False, + "quota": 0, + "used": used, + "remaining": None, + "allowed": True, + "exceeded": False, + } + + remaining = max(quota - used, 0) + return { + "enabled": True, + "quota": quota, + "used": used, + "remaining": remaining, + "allowed": used + requested <= quota, + "exceeded": used > quota, + } + + +def get_project_storage_quota_response(project, requested_size=0): + quota_state = get_project_storage_quota_state(project, requested_size=requested_size) + if quota_state["allowed"]: + return None + + return Response( + { + "error": "Project storage quota exceeded.", + "status": False, + "code": "project_storage_quota_exceeded", + "project_id": str(project.id), + "quota": quota_state["quota"], + "used": quota_state["used"], + "remaining": quota_state["remaining"], + }, + status=status.HTTP_400_BAD_REQUEST, + ) diff --git a/plane-src/apps/web/core/components/issues/issue-detail-widgets/attachments/helper.tsx b/plane-src/apps/web/core/components/issues/issue-detail-widgets/attachments/helper.tsx index e32dfbb..440ac4c 100644 --- a/plane-src/apps/web/core/components/issues/issue-detail-widgets/attachments/helper.tsx +++ b/plane-src/apps/web/core/components/issues/issue-detail-widgets/attachments/helper.tsx @@ -27,6 +27,35 @@ export type TAttachmentHelpers = { snapshot: TAttachmentSnapshot; }; +const formatBytes = (value?: number) => { + const bytes = Number(value || 0); + if (bytes <= 0) return "0 Б"; + + const units = ["Б", "КБ", "МБ", "ГБ", "ТБ"]; + const unitIndex = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1); + const normalizedValue = bytes / 1024 ** unitIndex; + + return `${normalizedValue.toLocaleString("ru-RU", { + maximumFractionDigits: normalizedValue >= 10 ? 0 : 1, + })} ${units[unitIndex]}`; +}; + +const getAttachmentUploadErrorMessage = (error: unknown) => { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "project_storage_quota_exceeded" + ) { + const remaining = "remaining" in error && typeof error.remaining === "number" ? error.remaining : 0; + const quota = "quota" in error && typeof error.quota === "number" ? error.quota : 0; + + return `Квота проекта превышена. Доступно ${formatBytes(remaining)} из лимита ${formatBytes(quota)}. Файл не был загружен.`; + } + + return "Вложение не удалось загрузить."; +}; + export const useAttachmentOperations = ( workspaceSlug: string, projectId: string, @@ -43,14 +72,14 @@ export const useAttachmentOperations = ( if (!workspaceSlug || !projectId || !issueId) throw new Error("Missing required fields"); const attachmentUploadPromise = createAttachment(workspaceSlug, projectId, issueId, file); setPromiseToast(attachmentUploadPromise, { - loading: "Uploading attachment...", + loading: "Загружаем вложение...", success: { - title: "Attachment uploaded", - message: () => "The attachment has been successfully uploaded", + title: "Вложение загружено", + message: () => "Файл успешно добавлен к рабочему элементу.", }, error: { - title: "Attachment not uploaded", - message: () => "The attachment could not be uploaded", + title: "Вложение не загружено", + message: getAttachmentUploadErrorMessage, }, }); diff --git a/plane-src/apps/web/core/components/workspace/settings/storage-settings.tsx b/plane-src/apps/web/core/components/workspace/settings/storage-settings.tsx index c502c16..c4bd922 100644 --- a/plane-src/apps/web/core/components/workspace/settings/storage-settings.tsx +++ b/plane-src/apps/web/core/components/workspace/settings/storage-settings.tsx @@ -4,11 +4,26 @@ * See the LICENSE file for details. */ -import { AlertTriangle, Database, Files, HardDrive, Layers3, Recycle, UploadCloud } from "lucide-react"; +import { + AlertTriangle, + Check, + Database, + Files, + HardDrive, + Layers3, + Recycle, + RotateCw, + SearchCheck, + Trash2, + UploadCloud, +} from "lucide-react"; import type { ElementType } from "react"; +import { useEffect, useState } from "react"; import useSWR from "swr"; // plane imports -import type { IWorkspaceStorageProjectSummary } from "@plane/types"; +import { Button } from "@plane/propel/button"; +import { TOAST_TYPE, setToast } from "@plane/propel/toast"; +import type { IWorkspaceStorageProjectSummary, TWorkspaceStorageMaintenanceAction } from "@plane/types"; import { cn } from "@plane/utils"; // components import { SettingsHeading } from "@/components/settings/heading"; @@ -16,6 +31,7 @@ import { SettingsHeading } from "@/components/settings/heading"; import { WorkspaceService } from "@/services/workspace.service"; const workspaceService = new WorkspaceService(); +const MEGABYTE = 1024 * 1024; const formatBytes = (value: number) => { const bytes = Number(value || 0); @@ -30,6 +46,10 @@ const formatBytes = (value: number) => { const formatCount = (value: number) => new Intl.NumberFormat("ru-RU").format(Number(value || 0)); +const bytesToMegabytes = (value: number) => Math.max(Math.ceil(Number(value || 0) / MEGABYTE), 0); + +const megabytesToBytes = (value: string | number) => Math.max(Math.round(Number(value || 0) * MEGABYTE), 0); + const StatCard = (props: { title: string; value: string; @@ -59,12 +79,17 @@ const StatCard = (props: { ); }; -const ProjectStorageRow = (props: { project: IWorkspaceStorageProjectSummary; maxSize: number }) => { - const { project, maxSize } = props; +const ProjectStorageRow = (props: { + maxSize: number; + onRefresh: () => Promise; + project: IWorkspaceStorageProjectSummary; + workspaceSlug: string; +}) => { + const { maxSize, onRefresh, project, workspaceSlug } = props; const ratio = maxSize > 0 ? Math.max((project.logical_size / maxSize) * 100, project.logical_size > 0 ? 3 : 0) : 0; return ( -
+
{project.name} {project.identifier} @@ -79,8 +104,10 @@ const ProjectStorageRow = (props: { project: IWorkspaceStorageProjectSummary; ma
{formatBytes(project.physical_size)} {formatBytes(project.dedup_savings)} + 0}>{formatCount(project.failed_upload_count)} - {formatCount(project.soft_deleted_count)} + 0}>{formatCount(project.stale_unuploaded_count)} + 0}>{formatCount(project.soft_deleted_count)}
); }; @@ -98,24 +125,177 @@ const StorageValue = (props: { accent?: boolean; children: string; warning?: boo ); const ProjectStorageHeader = () => ( -
+
Проект Файлы Blob Логический объем Физический Дедуп + Квота Ошибки + Зависшие Удалено
); +const ProjectQuotaControl = (props: { + onRefresh: () => Promise; + project: IWorkspaceStorageProjectSummary; + workspaceSlug: string; +}) => { + const { onRefresh, project, workspaceSlug } = props; + const [isSaving, setIsSaving] = useState(false); + const [isEnabled, setIsEnabled] = useState(project.quota_enabled); + const [quotaMb, setQuotaMb] = useState(() => String(bytesToMegabytes(project.quota || project.logical_size))); + + useEffect(() => { + setIsEnabled(project.quota_enabled); + setQuotaMb(String(bytesToMegabytes(project.quota || project.logical_size))); + }, [project.logical_size, project.quota, project.quota_enabled]); + + const currentQuota = megabytesToBytes(quotaMb); + const normalizedQuota = isEnabled ? Math.max(currentQuota, MEGABYTE) : currentQuota; + const hasChanges = + isEnabled !== project.quota_enabled || + (isEnabled && normalizedQuota !== (project.quota_enabled ? Math.max(project.quota || 0, MEGABYTE) : 0)); + + const saveQuota = async () => { + setIsSaving(true); + try { + await workspaceService.updateWorkspaceStorageProjectQuota(workspaceSlug, project.id, { + quota_enabled: isEnabled, + quota: normalizedQuota, + }); + await onRefresh(); + setToast({ + type: TOAST_TYPE.SUCCESS, + title: "Квота обновлена", + message: isEnabled ? "Лимит проекта сохранен." : "Лимит проекта отключен.", + }); + } catch (err) { + setToast({ + type: TOAST_TYPE.ERROR, + title: "Квота не сохранена", + message: "Не удалось обновить лимит проекта.", + }); + } finally { + setIsSaving(false); + } + }; + + return ( +
+ +
+
+ setQuotaMb(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + saveQuota(); + } + }} + type="number" + value={quotaMb} + /> + МБ +
+
+ {project.quota_enabled + ? `${formatBytes(project.quota_used)} из ${formatBytes(project.quota)}` + : "без лимита"} +
+
+ +
+ ); +}; + +type TMaintenanceCardProps = { + action: TWorkspaceStorageMaintenanceAction; + caption: string; + disabled?: boolean; + icon: ElementType; + isRunning: boolean; + onRun: (action: TWorkspaceStorageMaintenanceAction) => void; + title: string; + value: string; +}; + +const MaintenanceCard = ({ + action, + caption, + disabled = false, + icon: Icon, + isRunning, + onRun, + title, + value, +}: TMaintenanceCardProps) => ( +
+
+
+
{title}
+
{value}
+
{caption}
+
+
+ +
+
+ +
+); + type TStorageSettingsContentProps = { workspaceSlug: string; }; export function StorageSettingsContent({ workspaceSlug }: TStorageSettingsContentProps) { - const { data, error, isLoading } = useSWR( + const [runningAction, setRunningAction] = useState(null); + const { data, error, isLoading, mutate } = useSWR( workspaceSlug ? ["workspace-storage-summary", workspaceSlug] : null, ([, slug]) => workspaceService.fetchWorkspaceStorageSummary(slug) ); @@ -123,6 +303,27 @@ export function StorageSettingsContent({ workspaceSlug }: TStorageSettingsConten const projects = [...(data?.projects ?? [])].sort((a, b) => b.logical_size - a.logical_size); const maxProjectSize = Math.max(...projects.map((project) => project.logical_size), 0); + const runMaintenance = async (action: TWorkspaceStorageMaintenanceAction) => { + setRunningAction(action); + try { + const result = await workspaceService.runWorkspaceStorageMaintenance(workspaceSlug, action); + await mutate(); + setToast({ + type: TOAST_TYPE.SUCCESS, + title: "Хранилище обновлено", + message: getMaintenanceResultMessage(result.action, result), + }); + } catch (err) { + setToast({ + type: TOAST_TYPE.ERROR, + title: "Действие не выполнено", + message: "Не удалось выполнить обслуживание хранилища.", + }); + } finally { + setRunningAction(null); + } + }; + return (
-
+
+ 0 ? "warning" : "default"} + /> + 0 ? "warning" : "default"} + />
+
+
+
+

Обслуживание

+

+ Ручные действия для диагностики и очистки файловой помойки без фонового удаления при открытии. +

+
+
+ retention {formatCount(data.cleanup.soft_deleted_retention_days)} дн. +
+
+
+ + + + +
+
+
@@ -209,11 +479,17 @@ export function StorageSettingsContent({ workspaceSlug }: TStorageSettingsConten
-
+
{projects.map((project) => ( - + ))}
@@ -224,3 +500,29 @@ export function StorageSettingsContent({ workspaceSlug }: TStorageSettingsConten
); } + +const getMaintenanceResultMessage = ( + action: TWorkspaceStorageMaintenanceAction, + result: { + deleted_asset_count?: number; + deleted_blob_count?: number; + deleted_object_count?: number; + missing_count?: number; + restored_count?: number; + scanned_count?: number; + } +) => { + if (action === "scan_missing_blobs") + return `Проверено: ${formatCount(result.scanned_count ?? 0)}, потеряно: ${formatCount( + result.missing_count ?? 0 + )}, восстановлено: ${formatCount(result.restored_count ?? 0)}.`; + + if (action === "purge_orphaned_blobs") + return `Удалено blob-записей: ${formatCount(result.deleted_blob_count ?? 0)}, объектов: ${formatCount( + result.deleted_object_count ?? 0 + )}.`; + + return `Удалено файловых записей: ${formatCount(result.deleted_asset_count ?? 0)}, объектов: ${formatCount( + result.deleted_object_count ?? 0 + )}.`; +}; diff --git a/plane-src/apps/web/core/services/workspace.service.ts b/plane-src/apps/web/core/services/workspace.service.ts index 7b25606..5f36163 100644 --- a/plane-src/apps/web/core/services/workspace.service.ts +++ b/plane-src/apps/web/core/services/workspace.service.ts @@ -27,7 +27,10 @@ import type { IWorkspaceSidebarNavigationItem, IWorkspaceSidebarNavigation, IWorkspaceUserPropertiesResponse, + IWorkspaceStorageMaintenanceResponse, + IWorkspaceStorageProjectQuotaResponse, IWorkspaceStorageSummaryResponse, + TWorkspaceStorageMaintenanceAction, } from "@plane/types"; // services import { APIService } from "@/services/api.service"; @@ -61,6 +64,29 @@ export class WorkspaceService extends APIService { }); } + async runWorkspaceStorageMaintenance( + workspaceSlug: string, + action: TWorkspaceStorageMaintenanceAction + ): Promise { + return this.post(`/api/workspaces/${workspaceSlug}/storage/maintenance/`, { action }) + .then((response) => response?.data) + .catch((error) => { + throw error?.response?.data; + }); + } + + async updateWorkspaceStorageProjectQuota( + workspaceSlug: string, + projectId: string, + data: { quota_enabled: boolean; quota: number } + ): Promise { + return this.patch(`/api/workspaces/${workspaceSlug}/storage/projects/${projectId}/quota/`, data) + .then((response) => response?.data) + .catch((error) => { + throw error?.response?.data; + }); + } + async createWorkspace(data: Partial): Promise { return this.post("/api/workspaces/", data) .then((response) => response?.data) diff --git a/plane-src/packages/types/src/workspace.ts b/plane-src/packages/types/src/workspace.ts index 523a8dd..2f9c4e5 100644 --- a/plane-src/packages/types/src/workspace.ts +++ b/plane-src/packages/types/src/workspace.ts @@ -251,9 +251,47 @@ export interface IWorkspaceStorageProjectSummary { dedup_savings: number; failed_upload_count: number; failed_upload_size: number; + failed_preview_count: number; + failed_preview_size: number; + stale_unuploaded_count: number; soft_deleted_count: number; soft_deleted_size: number; uploaded_without_blob_count: number; + quota_enabled: boolean; + quota: number; + quota_used: number; + quota_remaining: number | null; + quota_percent: number; + quota_exceeded: boolean; + quota_warning: boolean; +} + +export interface IWorkspaceStorageProjectQuotaResponse { + id: string; + quota_enabled: boolean; + quota: number; + quota_used: number; + quota_remaining: number | null; + quota_percent: number; + quota_exceeded: boolean; + quota_warning: boolean; +} + +export type TWorkspaceStorageMaintenanceAction = + | "scan_missing_blobs" + | "purge_stale_unuploaded" + | "purge_expired_deleted" + | "purge_orphaned_blobs"; + +export interface IWorkspaceStorageMaintenanceResponse { + action: TWorkspaceStorageMaintenanceAction; + scanned_count?: number; + missing_count?: number; + restored_count?: number; + deleted_asset_count?: number; + deleted_blob_count?: number; + deleted_object_count?: number; + released_size?: number; } export interface IWorkspaceStorageSummaryResponse { @@ -283,6 +321,22 @@ export interface IWorkspaceStorageSummaryResponse { orphaned_blob_size: number; missing_blob_count: number; missing_blob_size: number; + failed_preview_count: number; + failed_preview_size: number; + failed_preview_instrumented: boolean; + quota_warning_project_count: number; + quota_exceeded_project_count: number; + }; + cleanup: { + workspace_slug: string; + unuploaded_retention_days: number; + soft_deleted_retention_days: number; + unuploaded_ready_count: number; + unuploaded_ready_size: number; + soft_deleted_ready_count: number; + soft_deleted_ready_size: number; + orphaned_blob_ready_count: number; + orphaned_blob_ready_size: number; }; projects: IWorkspaceStorageProjectSummary[]; }