SILVER/NODEDC_TASKMANAGER
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ФУНКЦИИ - МЕЖПРОЕКТНАЯ КОММУНИКАЦИЯ: очистка хранилища и проектные квоты

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-04-28 13:25:37 +03:00
parent 02d79da6f9
commit 46e27a326c
15 changed files with 870 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plane-src/apps/api/plane/app/urls/workspace.py
+12
View File
@@ -36,6 +36,8 @@ from plane.app.views import (
UserRecentVisitViewSet,
WorkspaceHomePreferenceViewSet,
WorkspaceStickyViewSet,
WorkspaceStorageMaintenanceEndpoint,
WorkspaceStorageProjectQuotaEndpoint,
WorkspaceStorageSummaryEndpoint,
WorkspaceUserPreferenceViewSet,
)
@@ -263,6 +265,16 @@ urlpatterns = [
WorkspaceStorageSummaryEndpoint.as_view(),
name="workspace-storage-summary",
),
path(
"workspaces/<str:slug>/storage/maintenance/",
WorkspaceStorageMaintenanceEndpoint.as_view(),
name="workspace-storage-maintenance",
),
path(
"workspaces/<str:slug>/storage/projects/<uuid:project_id>/quota/",
WorkspaceStorageProjectQuotaEndpoint.as_view(),
name="workspace-storage-project-quota",
),
# User Preference
path(
"workspaces/<str:slug>/sidebar-preferences/",
plane-src/apps/api/plane/app/views/__init__.py
+5 -1
View File
@@ -83,7 +83,11 @@ from .workspace.module import WorkspaceModulesEndpoint
from .workspace.cycle import WorkspaceCyclesEndpoint
from .workspace.quick_link import QuickLinkViewSet
from .workspace.sticky import WorkspaceStickyViewSet
from .workspace.storage import WorkspaceStorageSummaryEndpoint
from .workspace.storage import (
WorkspaceStorageMaintenanceEndpoint,
WorkspaceStorageProjectQuotaEndpoint,
WorkspaceStorageSummaryEndpoint,
)
from .state.base import StateViewSet, IntakeStateEndpoint
from .view.base import (
plane-src/apps/api/plane/app/views/asset/v2.py
+22 -6
View File
@@ -23,7 +23,7 @@ from plane.settings.storage import S3Storage
from plane.app.permissions import allow_permission, ROLE
from plane.utils.cache import invalidate_cache_directly
from plane.throttles.asset import AssetRateThrottle
from plane.utils.upload_limits import resolve_workspace_upload_size_limit
from plane.utils.upload_limits import get_project_storage_quota_response, resolve_workspace_upload_size_limit
from plane.utils.file_dedup import (
UploadedObjectMissing,
attach_existing_blob_to_file_asset,
@@ -361,6 +361,14 @@ class WorkspaceFileAssetEndpoint(BaseAPIView):
# Get the size limit
size_limit = resolve_workspace_upload_size_limit(workspace, size)
if entity_type == FileAsset.EntityTypeContext.PROJECT_COVER and entity_identifier:
quota_project = Project.objects.filter(id=entity_identifier, workspace=workspace).first()
else:
quota_project = None
if quota_project is not None:
quota_response = get_project_storage_quota_response(quota_project, requested_size=size)
if quota_response is not None:
return quota_response
# asset key
asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
@@ -564,6 +572,10 @@ class ProjectAssetEndpoint(BaseAPIView):
# Get the size limit
size_limit = resolve_workspace_upload_size_limit(workspace, size)
project = Project.objects.get(id=project_id, workspace=workspace)
quota_response = get_project_storage_quota_response(project, requested_size=size)
if quota_response is not None:
return quota_response
# asset key
asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
@@ -576,7 +588,7 @@ class ProjectAssetEndpoint(BaseAPIView):
workspace=workspace,
created_by=request.user,
entity_type=entity_type,
project_id=project_id,
project_id=project.id,
**self.get_entity_id_field(entity_type, entity_identifier),
)
@@ -767,10 +779,6 @@ class DuplicateAssetEndpoint(BaseAPIView):
)
workspace = Workspace.objects.get(slug=slug)
if project_id:
# check if project exists in the workspace
if not Project.objects.filter(id=project_id, workspace=workspace).exists():
return Response({"error": "Project not found"}, status=status.HTTP_404_NOT_FOUND)
storage = S3Storage(request=request)
original_asset = FileAsset.objects.filter(id=asset_id, workspace=workspace, is_uploaded=True).first()
@@ -778,6 +786,14 @@ class DuplicateAssetEndpoint(BaseAPIView):
if not original_asset:
return Response({"error": "Asset not found"}, status=status.HTTP_404_NOT_FOUND)
if project_id:
project = Project.objects.filter(id=project_id, workspace=workspace).first()
if not project:
return Response({"error": "Project not found"}, status=status.HTTP_404_NOT_FOUND)
quota_response = get_project_storage_quota_response(project, requested_size=original_asset.size)
if quota_response is not None:
return quota_response
destination_key = f"{workspace.id}/{uuid.uuid4().hex}-{original_asset.attributes.get('name')}"
duplicated_asset = FileAsset.objects.create(
attributes={
plane-src/apps/api/plane/app/views/issue/attachment.py
+6 -2
View File
@@ -19,12 +19,12 @@ from rest_framework.parsers import MultiPartParser, FormParser
# Module imports
from .. import BaseAPIView
from plane.app.serializers import IssueAttachmentSerializer
from plane.db.models import FileAsset, Workspace
from plane.db.models import FileAsset, Project, Workspace
from plane.bgtasks.issue_activities_task import issue_activity
from plane.app.permissions import allow_permission, ROLE
from plane.settings.storage import S3Storage
from plane.utils.host import base_host
from plane.utils.upload_limits import resolve_workspace_upload_size_limit
from plane.utils.upload_limits import get_project_storage_quota_response, resolve_workspace_upload_size_limit
from plane.utils.attachment_preview import attachment_object_exists, get_attachment_preview_response
from plane.utils.file_dedup import finalize_uploaded_file_asset, release_file_asset_blob, UploadedObjectMissing
@@ -118,12 +118,16 @@ class IssueAttachmentV2Endpoint(BaseAPIView):
# Get the workspace
workspace = Workspace.objects.get(slug=slug)
project = Project.objects.get(id=project_id, workspace=workspace)
# asset key
asset_key = f"{workspace.id}/{uuid.uuid4().hex}-{name}"
# Get the size limit
size_limit = resolve_workspace_upload_size_limit(workspace, size)
quota_response = get_project_storage_quota_response(project, requested_size=size)
if quota_response is not None:
return quota_response
# Create a File Asset
asset = FileAsset.objects.create(
plane-src/apps/api/plane/app/views/workspace/storage.py
+237 -4
View File
@@ -2,8 +2,10 @@
# SPDX-License-Identifier: AGPL-3.0-only
# See the LICENSE file for details.
import os
from datetime import timedelta
from django.conf import settings
from django.db.models import Sum
from django.utils import timezone
from rest_framework import status
@@ -12,6 +14,8 @@ from rest_framework.response import Response
from plane.app.permissions import ROLE, allow_permission
from plane.app.views.base import BaseAPIView
from plane.db.models import FileAsset, Project, StoredBlob, Workspace
from plane.settings.storage import S3Storage
from plane.utils.file_dedup import hard_delete_file_asset, release_file_asset_blob
def _int_size(value):
@@ -33,24 +37,94 @@ def _dedup_savings(logical_size, physical_size):
return max(_int_size(logical_size) - _int_size(physical_size), 0)
def _retention_days(env_key, default):
try:
return max(int(os.environ.get(env_key, default)), 0)
except (TypeError, ValueError):
return int(default)
def _storage_cutoffs():
now = timezone.now()
unuploaded_retention_days = _retention_days("UNUPLOADED_ASSET_DELETE_DAYS", 7)
soft_deleted_retention_days = _retention_days("FILE_ASSET_HARD_DELETE_AFTER_DAYS", settings.HARD_DELETE_AFTER_DAYS)
return {
"stale_upload_cutoff": now - timedelta(days=1),
"unuploaded_cleanup_cutoff": now - timedelta(days=unuploaded_retention_days),
"soft_deleted_cleanup_cutoff": now - timedelta(days=soft_deleted_retention_days),
"unuploaded_retention_days": unuploaded_retention_days,
"soft_deleted_retention_days": soft_deleted_retention_days,
}
def _blob_size_queryset(queryset):
return _int_size(queryset.aggregate(total=Sum("size")).get("total"))
def _blob_size_by_ids(blob_ids):
if not blob_ids:
return 0
return _blob_size_queryset(StoredBlob.objects.filter(id__in=blob_ids, status=StoredBlob.Status.ACTIVE))
def _cleanup_snapshot(workspace, failed_uploads, soft_deleted_assets, orphaned_blobs):
cutoffs = _storage_cutoffs()
unuploaded_ready = failed_uploads.filter(created_at__lt=cutoffs["unuploaded_cleanup_cutoff"])
soft_deleted_ready = soft_deleted_assets.filter(deleted_at__lt=cutoffs["soft_deleted_cleanup_cutoff"])
orphaned_ready = orphaned_blobs.filter(ref_count=0)
return {
"unuploaded_retention_days": cutoffs["unuploaded_retention_days"],
"soft_deleted_retention_days": cutoffs["soft_deleted_retention_days"],
"unuploaded_ready_count": unuploaded_ready.count(),
"unuploaded_ready_size": _sum_asset_size(unuploaded_ready),
"soft_deleted_ready_count": soft_deleted_ready.count(),
"soft_deleted_ready_size": _sum_asset_size(soft_deleted_ready),
"orphaned_blob_ready_count": orphaned_ready.count(),
"orphaned_blob_ready_size": _blob_size_queryset(orphaned_ready),
"workspace_slug": workspace.slug,
}
def _project_quota_snapshot(project, used_size):
enabled = bool(project.storage_quota_enabled and project.storage_quota > 0)
quota = int(project.storage_quota or 0) if enabled else 0
used = int(used_size or 0)
remaining = max(quota - used, 0) if enabled else None
percent = round((used / quota) * 100, 1) if quota > 0 else 0
return {
"quota_enabled": enabled,
"quota": quota,
"quota_used": used,
"quota_remaining": remaining,
"quota_percent": percent,
"quota_exceeded": enabled and used > quota,
"quota_warning": enabled and used <= quota and percent >= 80,
}
class WorkspaceStorageSummaryEndpoint(BaseAPIView):
@allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE")
def get(self, request, slug):
workspace = Workspace.objects.get(slug=slug)
stale_cutoff = timezone.now() - timedelta(days=1)
cutoffs = _storage_cutoffs()
active_assets = FileAsset.objects.filter(workspace=workspace, is_uploaded=True)
failed_preview_assets = active_assets.filter(attributes__preview_status="failed")
active_blob_ids = list(active_assets.exclude(blob__isnull=True).values_list("blob_id", flat=True).distinct())
workspace_logical_size = _sum_asset_size(active_assets)
workspace_physical_size = _sum_blob_size(active_blob_ids)
workspace_physical_size = _blob_size_by_ids(active_blob_ids)
failed_uploads = FileAsset.all_objects.filter(
workspace=workspace,
is_uploaded=False,
deleted_at__isnull=True,
)
stale_unuploaded = failed_uploads.filter(created_at__lt=stale_cutoff)
stale_unuploaded = failed_uploads.filter(created_at__lt=cutoffs["stale_upload_cutoff"])
soft_deleted_assets = FileAsset.all_objects.filter(workspace=workspace, deleted_at__isnull=False)
orphaned_blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.ORPHANED)
missing_blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.MISSING)
@@ -64,8 +138,9 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView):
project_assets.exclude(blob__isnull=True).values_list("blob_id", flat=True).distinct()
)
project_logical_size = _sum_asset_size(project_assets)
project_physical_size = _sum_blob_size(project_blob_ids)
project_physical_size = _blob_size_by_ids(project_blob_ids)
project_failed_uploads = failed_uploads.filter(project=project)
project_failed_previews = failed_preview_assets.filter(project=project)
project_soft_deleted = soft_deleted_assets.filter(project=project)
project_uploaded_without_blob = uploaded_without_blob.filter(project=project)
@@ -81,9 +156,15 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView):
"dedup_savings": _dedup_savings(project_logical_size, project_physical_size),
"failed_upload_count": project_failed_uploads.count(),
"failed_upload_size": _sum_asset_size(project_failed_uploads),
"failed_preview_count": project_failed_previews.count(),
"failed_preview_size": _sum_asset_size(project_failed_previews),
"stale_unuploaded_count": project_failed_uploads.filter(
created_at__lt=cutoffs["stale_upload_cutoff"]
).count(),
"soft_deleted_count": project_soft_deleted.count(),
"soft_deleted_size": _sum_asset_size(project_soft_deleted),
"uploaded_without_blob_count": project_uploaded_without_blob.count(),
**_project_quota_snapshot(project, project_logical_size),
}
)
@@ -114,8 +195,160 @@ class WorkspaceStorageSummaryEndpoint(BaseAPIView):
"orphaned_blob_size": _sum_blob_size(list(orphaned_blobs.values_list("id", flat=True))),
"missing_blob_count": missing_blobs.count(),
"missing_blob_size": _sum_blob_size(list(missing_blobs.values_list("id", flat=True))),
"failed_preview_count": failed_preview_assets.count(),
"failed_preview_size": _sum_asset_size(failed_preview_assets),
"failed_preview_instrumented": True,
"quota_warning_project_count": sum(1 for project in project_rows if project["quota_warning"]),
"quota_exceeded_project_count": sum(1 for project in project_rows if project["quota_exceeded"]),
},
"cleanup": _cleanup_snapshot(workspace, failed_uploads, soft_deleted_assets, orphaned_blobs),
"projects": project_rows,
}
return Response(data, status=status.HTTP_200_OK)
class WorkspaceStorageProjectQuotaEndpoint(BaseAPIView):
@allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE")
def patch(self, request, slug, project_id):
workspace = Workspace.objects.get(slug=slug)
project = Project.objects.get(id=project_id, workspace=workspace)
enabled = bool(request.data.get("quota_enabled", False))
try:
quota = int(request.data.get("quota", 0) or 0)
except (TypeError, ValueError):
quota = 0
project.storage_quota_enabled = enabled and quota > 0
project.storage_quota = max(quota, 0)
project.save(update_fields=["storage_quota_enabled", "storage_quota", "updated_at"])
return Response(
{
"id": str(project.id),
**_project_quota_snapshot(
project,
_sum_asset_size(FileAsset.objects.filter(project=project, is_uploaded=True)),
),
},
status=status.HTTP_200_OK,
)
class WorkspaceStorageMaintenanceEndpoint(BaseAPIView):
@allow_permission(allowed_roles=[ROLE.ADMIN], level="WORKSPACE")
def post(self, request, slug):
workspace = Workspace.objects.get(slug=slug)
action = request.data.get("action")
if action == "scan_missing_blobs":
return Response(self._scan_missing_blobs(request, workspace), status=status.HTTP_200_OK)
if action == "purge_stale_unuploaded":
return Response(self._purge_stale_unuploaded(request, workspace), status=status.HTTP_200_OK)
if action == "purge_expired_deleted":
return Response(self._purge_expired_deleted(request, workspace), status=status.HTTP_200_OK)
if action == "purge_orphaned_blobs":
return Response(self._purge_orphaned_blobs(request, workspace), status=status.HTTP_200_OK)
return Response({"error": "Unsupported storage maintenance action."}, status=status.HTTP_400_BAD_REQUEST)
def _scan_missing_blobs(self, request, workspace):
storage = S3Storage(request=request, is_server=True)
blobs = StoredBlob.objects.filter(
workspace=workspace,
status__in=[StoredBlob.Status.ACTIVE, StoredBlob.Status.MISSING],
)
scanned = 0
missing = 0
restored = 0
for blob in blobs.iterator():
scanned += 1
metadata = storage.get_object_metadata(object_name=blob.canonical_object_key)
if metadata:
if blob.status == StoredBlob.Status.MISSING:
restored += 1
blob.status = StoredBlob.Status.ACTIVE
blob.storage_metadata = metadata
blob.save(update_fields=["status", "storage_metadata", "updated_at"])
continue
if blob.status != StoredBlob.Status.MISSING:
missing += 1
blob.status = StoredBlob.Status.MISSING
blob.save(update_fields=["status", "updated_at"])
return {
"action": "scan_missing_blobs",
"scanned_count": scanned,
"missing_count": missing,
"restored_count": restored,
}
def _purge_stale_unuploaded(self, request, workspace):
cutoff = _storage_cutoffs()["unuploaded_cleanup_cutoff"]
assets = FileAsset.objects.filter(workspace=workspace, is_uploaded=False, created_at__lt=cutoff)
deleted_assets = 0
deleted_objects = 0
released_size = 0
for asset in assets.iterator():
released_size += _int_size(asset.size)
release_result = release_file_asset_blob(asset, request=request, delete_untracked_object=True)
if release_result.deleted_object:
deleted_objects += 1
asset.delete(soft=False)
deleted_assets += 1
return {
"action": "purge_stale_unuploaded",
"deleted_asset_count": deleted_assets,
"deleted_object_count": deleted_objects,
"released_size": released_size,
}
def _purge_expired_deleted(self, request, workspace):
cutoff = _storage_cutoffs()["soft_deleted_cleanup_cutoff"]
assets = FileAsset.all_objects.filter(workspace=workspace, deleted_at__lt=cutoff)
storage = S3Storage(request=request, is_server=True)
deleted_assets = 0
deleted_objects = 0
released_size = 0
for asset in assets.iterator():
released_size += _int_size(asset.size)
if hard_delete_file_asset(asset, request=request, storage=storage):
deleted_objects += 1
deleted_assets += 1
return {
"action": "purge_expired_deleted",
"deleted_asset_count": deleted_assets,
"deleted_object_count": deleted_objects,
"released_size": released_size,
}
def _purge_orphaned_blobs(self, request, workspace):
storage = S3Storage(request=request, is_server=True)
blobs = StoredBlob.objects.filter(workspace=workspace, status=StoredBlob.Status.ORPHANED, ref_count=0)
deleted_blobs = 0
deleted_objects = 0
released_size = 0
for blob in blobs.iterator():
released_size += _int_size(blob.size)
if storage.delete_files(object_names=[blob.canonical_object_key]):
deleted_objects += 1
blob.delete(soft=False)
deleted_blobs += 1
return {
"action": "purge_orphaned_blobs",
"deleted_blob_count": deleted_blobs,
"deleted_object_count": deleted_objects,
"released_size": released_size,
}