From a7606f2e9a68bbf9ec3d8bd2389903733876af37 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Mon, 27 Apr 2026 13:47:46 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A4=D0=A3=D0=9D=D0=9A=D0=A6=D0=98=D0=98=20-?= =?UTF-8?q?=20=D0=9C=D0=95=D0=96=D0=9F=D0=A0=D0=9E=D0=95=D0=9A=D0=A2=D0=9D?= =?UTF-8?q?=D0=90=D0=AF=20=D0=9A=D0=9E=D0=9C=D0=9C=D0=A3=D0=9D=D0=98=D0=9A?= =?UTF-8?q?=D0=90=D0=A6=D0=98=D0=AF:=20retention=20cleanup=20=D1=84=D0=B0?= =?UTF-8?q?=D0=B9=D0=BB=D0=BE=D0=B2=D0=BE=D0=B3=D0=BE=20=D1=85=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps/api/plane/bgtasks/deletion_task.py | 7 +++ .../apps/api/plane/bgtasks/file_asset_task.py | 51 ++++++++++++++++++- .../plane/tests/unit/utils/test_file_dedup.py | 25 +++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/plane-src/apps/api/plane/bgtasks/deletion_task.py b/plane-src/apps/api/plane/bgtasks/deletion_task.py index 11d9041..45556e3 100644 --- a/plane-src/apps/api/plane/bgtasks/deletion_task.py +++ b/plane-src/apps/api/plane/bgtasks/deletion_task.py @@ -112,6 +112,7 @@ def restore_related_objects(app_label, model_name, instance_pk, using=None): @shared_task def hard_delete(): + from plane.bgtasks.file_asset_task import delete_expired_file_asset from plane.db.models import ( Workspace, Project, @@ -134,6 +135,9 @@ def hard_delete(): ) days = settings.HARD_DELETE_AFTER_DAYS + + delete_expired_file_asset() + # check delete workspace _ = Workspace.all_objects.filter(deleted_at__lt=timezone.now() - timezone.timedelta(days=days)).delete() @@ -185,6 +189,9 @@ def hard_delete(): # Iterate through all models for model in all_models: + if model._meta.label_lower in {"db.fileasset", "db.storedblob"}: + continue + # Check if the model has a 'deleted_at' field if hasattr(model, "deleted_at"): # Get all instances where 'deleted_at' is greater than 30 days ago diff --git a/plane-src/apps/api/plane/bgtasks/file_asset_task.py b/plane-src/apps/api/plane/bgtasks/file_asset_task.py index bbad46b..577c7ab 100644 --- a/plane-src/apps/api/plane/bgtasks/file_asset_task.py +++ b/plane-src/apps/api/plane/bgtasks/file_asset_task.py @@ -7,6 +7,7 @@ import os from datetime import timedelta # Django imports +from django.conf import settings from django.utils import timezone from django.db.models import Q @@ -14,10 +15,44 @@ from django.db.models import Q from celery import shared_task # Module imports -from plane.db.models import FileAsset +from plane.db.models import FileAsset, StoredBlob +from plane.settings.storage import S3Storage from plane.utils.file_dedup import release_file_asset_blob +def _asset_object_key(asset): + return str(asset.asset.name or asset.asset) + + +def _delete_legacy_asset_object(asset, storage): + object_key = _asset_object_key(asset) + if not object_key: + return False + + has_other_reference = FileAsset.all_objects.filter(asset=object_key).exclude(pk=asset.pk).exists() + if has_other_reference: + return False + + return storage.delete_files(object_names=[object_key]) + + +def _hard_delete_file_asset(asset, storage=None): + blob_id = asset.blob_id + + if blob_id: + release_file_asset_blob(asset) + StoredBlob.all_objects.filter( + pk=blob_id, + ref_count=0, + status=StoredBlob.Status.ORPHANED, + ).delete() + else: + storage = storage or S3Storage() + _delete_legacy_asset_object(asset, storage) + + asset.delete(soft=False) + + @shared_task def delete_unuploaded_file_asset(): """This task deletes unuploaded file assets older than a certain number of days.""" @@ -27,4 +62,16 @@ def delete_unuploaded_file_asset(): ) for asset in stale_assets.iterator(): release_file_asset_blob(asset, delete_untracked_object=True) - asset.delete() + asset.delete(soft=False) + + +@shared_task +def delete_expired_file_asset(): + """Hard delete soft-deleted file assets after the restore retention window.""" + days = int(os.environ.get("FILE_ASSET_HARD_DELETE_AFTER_DAYS", settings.HARD_DELETE_AFTER_DAYS)) + cutoff = timezone.now() - timedelta(days=days) + storage = S3Storage() + expired_assets = FileAsset.all_objects.filter(deleted_at__lt=cutoff) + + for asset in expired_assets.iterator(): + _hard_delete_file_asset(asset, storage=storage) diff --git a/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py b/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py index e8f9b2f..72f9d6c 100644 --- a/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py +++ b/plane-src/apps/api/plane/tests/unit/utils/test_file_dedup.py @@ -5,8 +5,10 @@ from unittest.mock import patch import pytest +from django.utils import timezone from plane.db.models import FileAsset, StoredBlob +from plane.bgtasks.file_asset_task import delete_expired_file_asset from plane.utils.file_dedup import attach_existing_blob_to_file_asset, finalize_uploaded_file_asset, release_file_asset_blob @@ -150,3 +152,26 @@ def test_attach_existing_blob_reuses_canonical_object_without_copy(workspace, pr assert target_asset.asset.name == "workspace/a.txt" assert blob.ref_count == 2 assert fake_storage.deleted == [] + + +@pytest.mark.django_db +def test_delete_expired_file_asset_releases_blob_and_hard_deletes_rows(workspace, project, fake_storage): + first_asset = create_asset(workspace, project, "workspace/a.txt") + duplicate_asset = create_asset(workspace, project, "workspace/b.txt") + + with patch("plane.utils.file_dedup.S3Storage", return_value=fake_storage): + finalize_uploaded_file_asset(first_asset) + finalize_uploaded_file_asset(duplicate_asset) + + deleted_at = timezone.now() - timezone.timedelta(days=90) + FileAsset.all_objects.filter(pk__in=[first_asset.pk, duplicate_asset.pk]).update(deleted_at=deleted_at) + + with ( + patch("plane.utils.file_dedup.S3Storage", return_value=fake_storage), + patch("plane.bgtasks.file_asset_task.S3Storage", return_value=fake_storage), + ): + delete_expired_file_asset() + + assert FileAsset.all_objects.filter(pk__in=[first_asset.pk, duplicate_asset.pk]).count() == 0 + assert StoredBlob.all_objects.count() == 0 + assert fake_storage.deleted == ["workspace/b.txt", "workspace/a.txt"]