Add files deletion when destroying attachment, workspace or workspaceMember (#10222)

Solution
- update attachment soft delete logic by destroy (seen with Weiko &
Felix)
- add two jobs for file and workspace folder deletion
- add listener to attachment and workspaceMember destroy event -> add
file deletion job
- update logic in deleteWorkspace method -> add wokspace folder deletion
job

closes https://github.com/twentyhq/core-team-issues/issues/147

To go further
- delete old avatar when workspaceMember replaces its avatar
- same with workspace picture

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>

packages/twenty-front/src/modules/activities/files/components/AttachmentRow.tsx

@@ -5,7 +5,7 @@ import { PREVIEWABLE_EXTENSIONS } from '@/activities/files/components/DocumentVi
 import { Attachment } from '@/activities/files/types/Attachment';
 import { downloadFile } from '@/activities/files/utils/downloadFile';
 import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular';
-import { useDeleteOneRecord } from '@/object-record/hooks/useDeleteOneRecord';
+import { useDestroyOneRecord } from '@/object-record/hooks/useDestroyOneRecord';
 import { useUpdateOneRecord } from '@/object-record/hooks/useUpdateOneRecord';
 import {
   FieldContext,
@@ -95,12 +95,12 @@ export const AttachmentRow = ({
     [attachment?.id],
   );
 
-  const { deleteOneRecord: deleteOneAttachment } = useDeleteOneRecord({
+  const { destroyOneRecord: destroyOneAttachment } = useDestroyOneRecord({
     objectNameSingular: CoreObjectNameSingular.Attachment,
   });
 
   const handleDelete = () => {
-    deleteOneAttachment(attachment.id);
+    destroyOneAttachment(attachment.id);
   };
 
   const { updateOneRecord: updateOneAttachment } = useUpdateOneRecord({

packages/twenty-server/src/engine/core-modules/file/file.module.ts

@@ -1,15 +1,27 @@
 import { Module } from '@nestjs/common';
 
-import { FilePathGuard } from 'src/engine/core-modules/file/guards/file-path-guard';
-import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';
 import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
+import { FilePathGuard } from 'src/engine/core-modules/file/guards/file-path-guard';
+import { FileDeletionJob } from 'src/engine/core-modules/file/jobs/file-deletion.job';
+import { FileWorkspaceFolderDeletionJob } from 'src/engine/core-modules/file/jobs/file-workspace-folder-deletion.job';
+import { FileAttachmentListener } from 'src/engine/core-modules/file/listeners/file-attachment.listener';
+import { FileWorkspaceMemberListener } from 'src/engine/core-modules/file/listeners/file-workspace-member.listener';
+import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';
 
 import { FileController } from './controllers/file.controller';
 import { FileService } from './services/file.service';
 
 @Module({
   imports: [JwtModule],
-  providers: [FileService, EnvironmentService, FilePathGuard],
+  providers: [
+    FileService,
+    EnvironmentService,
+    FilePathGuard,
+    FileAttachmentListener,
+    FileWorkspaceMemberListener,
+    FileWorkspaceFolderDeletionJob,
+    FileDeletionJob,
+  ],
   exports: [FileService],
   controllers: [FileController],
 })

packages/twenty-server/src/engine/core-modules/file/jobs/file-deletion.job.ts

@@ -0,0 +1,42 @@
+import { UnrecoverableError } from 'bullmq';
+
+import { FileService } from 'src/engine/core-modules/file/services/file.service';
+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+
+export type FileDeletionJobData = {
+  workspaceId: string;
+  fullPath: string;
+};
+
+@Processor(MessageQueue.deleteCascadeQueue)
+export class FileDeletionJob {
+  constructor(private readonly fileService: FileService) {}
+
+  @Process(FileDeletionJob.name)
+  async handle(data: FileDeletionJobData): Promise<void> {
+    const { workspaceId, fullPath } = data;
+
+    const folderPath = fullPath.split('/').slice(0, -1).join('/');
+    const filename = fullPath.split('/').pop();
+
+    if (!filename) {
+      throw new UnrecoverableError(
+        `[${FileDeletionJob.name}] Cannot parse filename from full path - ${fullPath}`,
+      );
+    }
+
+    try {
+      await this.fileService.deleteFile({
+        workspaceId,
+        filename,
+        folderPath,
+      });
+    } catch (error) {
+      throw new Error(
+        `[${FileDeletionJob.name}] Cannot delete file - ${fullPath}`,
+      );
+    }
+  }
+}

packages/twenty-server/src/engine/core-modules/file/jobs/file-workspace-folder-deletion.job.ts

@@ -0,0 +1,26 @@
+import { FileService } from 'src/engine/core-modules/file/services/file.service';
+import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
+import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+
+export type FileWorkspaceFolderDeletionJobData = {
+  workspaceId: string;
+};
+
+@Processor(MessageQueue.deleteCascadeQueue)
+export class FileWorkspaceFolderDeletionJob {
+  constructor(private readonly fileService: FileService) {}
+
+  @Process(FileWorkspaceFolderDeletionJob.name)
+  async handle(data: FileWorkspaceFolderDeletionJobData): Promise<void> {
+    const { workspaceId } = data;
+
+    try {
+      await this.fileService.deleteWorkspaceFolder(workspaceId);
+    } catch (error) {
+      throw new Error(
+        `[${FileWorkspaceFolderDeletionJob.name}] Cannot delete workspace folder - ${workspaceId}`,
+      );
+    }
+  }
+}

packages/twenty-server/src/engine/core-modules/file/listeners/file-attachment.listener.ts

@@ -0,0 +1,39 @@
+import { Injectable } from '@nestjs/common';
+
+import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
+import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
+import { ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
+import {
+  FileDeletionJob,
+  FileDeletionJobData,
+} from 'src/engine/core-modules/file/jobs/file-deletion.job';
+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event.type';
+import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity';
+
+@Injectable()
+export class FileAttachmentListener {
+  constructor(
+    @InjectMessageQueue(MessageQueue.deleteCascadeQueue)
+    private readonly messageQueueService: MessageQueueService,
+  ) {}
+
+  @OnDatabaseBatchEvent('attachment', DatabaseEventAction.DESTROYED)
+  async handleDestroyEvent(
+    payload: WorkspaceEventBatch<
+      ObjectRecordDestroyEvent<AttachmentWorkspaceEntity>
+    >,
+  ) {
+    for (const event of payload.events) {
+      await this.messageQueueService.add<FileDeletionJobData>(
+        FileDeletionJob.name,
+        {
+          workspaceId: payload.workspaceId,
+          fullPath: event.properties.before.fullPath,
+        },
+      );
+    }
+  }
+}

packages/twenty-server/src/engine/core-modules/file/listeners/file-workspace-member.listener.ts

@@ -0,0 +1,42 @@
+import { Injectable } from '@nestjs/common';
+
+import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
+import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
+import { ObjectRecordDestroyEvent } from 'src/engine/core-modules/event-emitter/types/object-record-destroy.event';
+import {
+  FileDeletionJob,
+  FileDeletionJobData,
+} from 'src/engine/core-modules/file/jobs/file-deletion.job';
+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event.type';
+import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
+
+@Injectable()
+export class FileWorkspaceMemberListener {
+  constructor(
+    @InjectMessageQueue(MessageQueue.deleteCascadeQueue)
+    private readonly messageQueueService: MessageQueueService,
+  ) {}
+
+  @OnDatabaseBatchEvent('workspaceMember', DatabaseEventAction.DESTROYED)
+  async handleDestroyEvent(
+    payload: WorkspaceEventBatch<
+      ObjectRecordDestroyEvent<WorkspaceMemberWorkspaceEntity>
+    >,
+  ) {
+    for (const event of payload.events) {
+      const avatarUrl = event.properties.before.avatarUrl;
+
+      if (!avatarUrl) {
+        continue;
+      }
+
+      this.messageQueueService.add<FileDeletionJobData>(FileDeletionJob.name, {
+        workspaceId: payload.workspaceId,
+        fullPath: event.properties.before.avatarUrl,
+      });
+    }
+  }
+}

packages/twenty-server/src/engine/core-modules/file/services/file.service.ts

@@ -48,4 +48,29 @@ export class FileService {
 
     return signedPayload;
   }
+
+  async deleteFile({
+    folderPath,
+    filename,
+    workspaceId,
+  }: {
+    folderPath: string;
+    filename: string;
+    workspaceId: string;
+  }) {
+    const workspaceFolderPath = `workspace-${workspaceId}/${folderPath}`;
+
+    return await this.fileStorageService.delete({
+      folderPath: workspaceFolderPath,
+      filename,
+    });
+  }
+
+  async deleteWorkspaceFolder(workspaceId: string) {
+    const workspaceFolderPath = `workspace-${workspaceId}`;
+
+    return await this.fileStorageService.delete({
+      folderPath: workspaceFolderPath,
+    });
+  }
 }

packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.spec.ts

@@ -9,6 +9,8 @@ import { EmailService } from 'src/engine/core-modules/email/email.service';
 import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
 import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
 import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
 import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service';
 import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
 import { UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service';
@@ -101,6 +103,10 @@ describe('WorkspaceService', () => {
           provide: WorkspaceCacheStorageService,
           useValue: {},
         },
+        {
+          provide: getQueueToken(MessageQueue.deleteCascadeQueue),
+          useValue: {},
+        },
       ],
     }).compile();
 

packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts

@@ -20,6 +20,13 @@ import { EnvironmentService } from 'src/engine/core-modules/environment/environm
 import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
 import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
 import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
+import {
+  FileWorkspaceFolderDeletionJob,
+  FileWorkspaceFolderDeletionJobData,
+} from 'src/engine/core-modules/file/jobs/file-workspace-folder-deletion.job';
+import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
 import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
 import { UserWorkspaceService } from 'src/engine/core-modules/user-workspace/user-workspace.service';
 import { User } from 'src/engine/core-modules/user/user.entity';
@@ -63,6 +70,8 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
     private readonly permissionsService: PermissionsService,
     private readonly customDomainService: CustomDomainService,
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
+    @InjectMessageQueue(MessageQueue.deleteCascadeQueue)
+    private readonly messageQueueService: MessageQueueService,
   ) {
     super(workspaceRepository);
   }
@@ -317,6 +326,11 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
 
     await this.deleteMetadataSchemaCacheAndUserWorkspace(workspace);
 
+    await this.messageQueueService.add<FileWorkspaceFolderDeletionJobData>(
+      FileWorkspaceFolderDeletionJob.name,
+      { workspaceId: id },
+    );
+
     return await this.workspaceRepository.delete(id);
   }