diff --git a/plane-src/apps/api/plane/bgtasks/deletion_task.py b/plane-src/apps/api/plane/bgtasks/deletion_task.py index 11d9041..45556e3 100644 --- a/plane-src/apps/api/plane/bgtasks/deletion_task.py +++ b/plane-src/apps/api/plane/bgtasks/deletion_task.py @@ -112,6 +112,7 @@ def restore_related_objects(app_label, model_name, instance_pk, using=None): @shared_task def hard_delete(): + from plane.bgtasks.file_asset_task import delete_expired_file_asset from plane.db.models import ( Workspace, Project, @@ -134,6 +135,9 @@ def hard_delete(): ) days = settings.HARD_DELETE_AFTER_DAYS + + delete_expired_file_asset() + # check delete workspace _ = Workspace.all_objects.filter(deleted_at__lt=timezone.now() - timezone.timedelta(days=days)).delete() @@ -185,6 +189,9 @@ def hard_delete(): # Iterate through all models for model in all_models: + if model._meta.label_lower in {"db.fileasset", "db.storedblob"}: + continue + # Check if the model has a 'deleted_at' field if hasattr(model, "deleted_at"): # Get all instances where 'deleted_at' is greater than 30 days ago diff --git a/plane-src/apps/api/plane/bgtasks/file_asset_task.py b/plane-src/apps/api/plane/bgtasks/file_asset_task.py index bbad46b..577c7ab 100644 --- a/plane-src/apps/api/plane/bgtasks/file_asset_task.py +++ b/plane-src/apps/api/plane/bgtasks/file_asset_task.py @@ -7,6 +7,7 @@ import os from datetime import timedelta # Django imports +from django.conf import settings from django.utils import timezone from django.db.models import Q @@ -14,10 +15,44 @@ from django.db.models import Q from celery import shared_task # Module imports -from plane.db.models import FileAsset +from plane.db.models import FileAsset, StoredBlob +from plane.settings.storage import S3Storage from plane.utils.file_dedup import release_file_asset_blob +def _asset_object_key(asset): + return str(asset.asset.name or asset.asset) + + +def _delete_legacy_asset_object(asset, storage): + object_key = _asset_object_key(asset) + if not object_key: + return False + + has_other_reference = FileAsset.all_objects.filter(asset=object_key).exclude(pk=asset.pk).exists() + if has_other_reference: + return False + + return storage.delete_files(object_names=[object_key]) + + +def _hard_delete_file_asset(asset, storage=None): + blob_id = asset.blob_id + + if blob_id: + release_file_asset_blob(asset) + StoredBlob.all_objects.filter( + pk=blob_id, + ref_count=0, + status=StoredBlob.Status.ORPHANED, + ).delete() + else: + storage = storage or S3Storage() + _delete_legacy_asset_object(asset, storage) + + asset.delete(soft=False) + + @shared_task def delete_unuploaded_file_asset(): """This task deletes unuploaded file assets older than a certain number of days.""" @@ -27,4 +62,16 @@ def delete_unuploaded_file_asset(): ) for asset in stale_assets.iterator(): release_file_asset_blob(asset, delete_untracked_object=True) - asset.delete() + asset.delete(soft=False) + + +@shared_task +def delete_expired_file_asset(): + """Hard delete soft-deleted file assets after the restore retention window.""" + days = int(os.environ.get("FILE_ASSET_HARD_DELETE_AFTER_DAYS", settings.HARD_DELETE_AFTER_DAYS)) + cutoff = timezone.now() - timedelta(days=days) + storage = S3Storage() + expired_assets = FileAsset.all_objects.filter(deleted_at__lt=cutoff) + + for asset in expired_assets.iterator(): + _hard_delete_file_asset(asset, storage=storage) diff --git a/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py b/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py index e8f9b2f..72f9d6c 100644 --- a/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py +++ b/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py @@ -5,8 +5,10 @@ from unittest.mock import patch import pytest +from django.utils import timezone from plane.db.models import FileAsset, StoredBlob +from plane.bgtasks.file_asset_task import delete_expired_file_asset from plane.utils.file_dedup import attach_existing_blob_to_file_asset, finalize_uploaded_file_asset, release_file_asset_blob @@ -150,3 +152,26 @@ def test_attach_existing_blob_reuses_canonical_object_without_copy(workspace, pr assert target_asset.asset.name == "workspace/a.txt" assert blob.ref_count == 2 assert fake_storage.deleted == [] + + +@pytest.mark.django_db +def test_delete_expired_file_asset_releases_blob_and_hard_deletes_rows(workspace, project, fake_storage): + first_asset = create_asset(workspace, project, "workspace/a.txt") + duplicate_asset = create_asset(workspace, project, "workspace/b.txt") + + with patch("plane.utils.file_dedup.S3Storage", return_value=fake_storage): + finalize_uploaded_file_asset(first_asset) + finalize_uploaded_file_asset(duplicate_asset) + + deleted_at = timezone.now() - timezone.timedelta(days=90) + FileAsset.all_objects.filter(pk__in=[first_asset.pk, duplicate_asset.pk]).update(deleted_at=deleted_at) + + with ( + patch("plane.utils.file_dedup.S3Storage", return_value=fake_storage), + patch("plane.bgtasks.file_asset_task.S3Storage", return_value=fake_storage), + ): + delete_expired_file_asset() + + assert FileAsset.all_objects.filter(pk__in=[first_asset.pk, duplicate_asset.pk]).count() == 0 + assert StoredBlob.all_objects.count() == 0 + assert fake_storage.deleted == ["workspace/b.txt", "workspace/a.txt"]