5629 update blocklist for messaging v2 (#5756)

Closes #5629 

- Add subdomain support in blocklist (if @example.com is blocked, every
subdomain will be blocked)

packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts

@@ -2,38 +2,68 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u
 
 describe('isEmailBlocklisted', () => {
   it('should return true if email is blocklisted', () => {
-    const email = 'hello@example.com';
+    const channelHandle = 'abc@example.com';
-    const blocklist = ['hello@example.com', 'hey@example.com'];
+    const email = 'hello@twenty.com';
-    const result = isEmailBlocklisted(email, blocklist);
+    const blocklist = ['hello@twenty.com', 'hey@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
 
     expect(result).toBe(true);
   });
   it('should return false if email is not blocklisted', () => {
+    const channelHandle = 'abc@example.com';
     const email = 'hello@twenty.com';
     const blocklist = ['hey@example.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
 
     expect(result).toBe(false);
   });
   it('should return false if email is null', () => {
+    const channelHandle = 'abc@twenty.com';
     const email = null;
     const blocklist = ['@example.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
+
+    expect(result).toBe(false);
+  });
+  it('should return true for subdomains', () => {
+    const channelHandle = 'abc@example.com';
+    const email = 'hello@twenty.twenty.com';
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
+
+    expect(result).toBe(true);
+  });
+  it('should return false for domains which end with blocklisted domain but are not subdomains', () => {
+    const channelHandle = 'abc@example.com';
+    const email = 'hello@twentytwenty.com';
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
 
     expect(result).toBe(false);
   });
   it('should return false if email is undefined', () => {
+    const channelHandle = 'abc@example.com';
     const email = undefined;
-    const blocklist = ['@example.com'];
+    const blocklist = ['@twenty.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
 
     expect(result).toBe(false);
   });
   it('should return true if email ends with blocklisted domain', () => {
-    const email = 'hello@example.com';
+    const channelHandle = 'abc@example.com';
-    const blocklist = ['@example.com'];
+    const email = 'hello@twenty.com';
-    const result = isEmailBlocklisted(email, blocklist);
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
 
     expect(result).toBe(true);
   });
+
+  it('should return false if email is same as channel handle', () => {
+    const channelHandle = 'hello@twenty.com';
+    const email = 'hello@twenty.com';
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
+
+    expect(result).toBe(false);
+  });
 });

packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts

@@ -1,14 +1,17 @@
 export const isEmailBlocklisted = (
+  channelHandle: string,
   email: string | null | undefined,
   blocklist: string[],
 ): boolean => {
-  if (!email) {
+  if (!email || email === channelHandle) {
     return false;
   }
 
   return blocklist.some((item) => {
     if (item.startsWith('@')) {
-      return email.endsWith(item);
+      const domain = email.split('@')[1];
+
+      return domain === item.slice(1) || domain.endsWith(`.${item.slice(1)}`);
     }
 
     return email === item;

packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts

@@ -124,14 +124,13 @@ export class GoogleCalendarSyncService {
 
     const blocklist = await this.getBlocklist(workspaceMemberId, workspaceId);
 
-    let filteredEvents = filterOutBlocklistedEvents(events, blocklist).filter(
+    let filteredEvents = filterOutBlocklistedEvents(
-      (event) => event.status !== 'cancelled',
+      calendarChannel.handle,
-    );
+      events,
+      blocklist,
+    ).filter((event) => event.status !== 'cancelled');
 
     if (emailOrDomainToReimport) {
-      // We still need to filter the events to only keep the ones that have the email or domain we want to reimport
-      // because the q parameter in the list method also filters the events that have the email or domain in their summary, description ...
-      // The q parameter allows us to narrow down the events
       filteredEvents = filteredEvents.filter(
         (event) =>
           event.attendees?.some(

packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts

@@ -3,6 +3,7 @@ import { calendar_v3 as calendarV3 } from 'googleapis';
 import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util';
 
 export const filterOutBlocklistedEvents = (
+  calendarChannelHandle: string,
   events: calendarV3.Schema$Event[],
   blocklist: string[],
 ) => {
@@ -12,7 +13,8 @@ export const filterOutBlocklistedEvents = (
     }
 
     return event.attendees.every(
-      (attendee) => !isEmailBlocklisted(attendee.email, blocklist),
+      (attendee) =>
+        !isEmailBlocklisted(calendarChannelHandle, attendee.email, blocklist),
     );
   });
 };

packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts

@@ -66,12 +66,14 @@ export class BlocklistItemDeleteMessagesJob
 
     const rolesToDelete: ('from' | 'to')[] = ['from', 'to'];
 
-    await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles(
+    for (const messageChannelId of messageChannelIds) {
-      handle,
+      await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdAndRoles(
-      messageChannelIds,
+        handle,
-      rolesToDelete,
+        messageChannelId,
-      workspaceId,
+        rolesToDelete,
-    );
+        workspaceId,
+      );
+    }
 
     await this.threadCleanerService.cleanWorkspaceThreads(workspaceId);
 

packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts

@@ -1,58 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-
-import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
-
-import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
-import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
-import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
-
-export type BlocklistReimportMessagesJobData = {
-  workspaceId: string;
-  workspaceMemberId: string;
-  handle: string;
-};
-
-@Injectable()
-export class BlocklistReimportMessagesJob
-  implements MessageQueueJob<BlocklistReimportMessagesJobData>
-{
-  private readonly logger = new Logger(BlocklistReimportMessagesJob.name);
-
-  constructor(
-    @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
-    private readonly connectedAccountRepository: ConnectedAccountRepository,
-  ) {}
-
-  async handle(data: BlocklistReimportMessagesJobData): Promise<void> {
-    const { workspaceId, workspaceMemberId, handle } = data;
-
-    this.logger.log(
-      `Reimporting messages from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
-    );
-
-    const connectedAccount =
-      await this.connectedAccountRepository.getAllByWorkspaceMemberId(
-        workspaceMemberId,
-        workspaceId,
-      );
-
-    if (!connectedAccount || connectedAccount.length === 0) {
-      this.logger.error(
-        `No connected account found for workspace member ${workspaceMemberId} in workspace ${workspaceId}`,
-      );
-
-      return;
-    }
-
-    // TODO: reimplement that
-    // await this.gmailMessageListFetchJob.fetchConnectedAccountThreads(
-    //   workspaceId,
-    //   connectedAccount[0].id,
-    //   [handle],
-    // );
-
-    this.logger.log(
-      `Reimporting messages from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId} done`,
-    );
-  }
-}

packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts

@@ -6,21 +6,28 @@ import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/t
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
 import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
-import {
-  BlocklistReimportMessagesJob,
-  BlocklistReimportMessagesJobData,
-} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job';
 import {
   BlocklistItemDeleteMessagesJobData,
   BlocklistItemDeleteMessagesJob,
 } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
 import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event';
+import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
+import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
+import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
+import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
 
 @Injectable()
 export class MessagingBlocklistListener {
   constructor(
     @Inject(MessageQueue.messagingQueue)
     private readonly messageQueueService: MessageQueueService,
+    @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
+    private readonly connectedAccountRepository: ConnectedAccountRepository,
+    private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService,
+    @InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
+    private readonly messageChannelRepository: MessageChannelRepository,
   ) {}
 
   @OnEvent('blocklist.created')
@@ -40,13 +47,28 @@ export class MessagingBlocklistListener {
   async handleDeletedEvent(
     payload: ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>,
   ) {
-    await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
+    const workspaceMemberId = payload.properties.before.workspaceMember.id;
-      BlocklistReimportMessagesJob.name,
+    const workspaceId = payload.workspaceId;
-      {
+
-        workspaceId: payload.workspaceId,
+    const connectedAccount =
-        workspaceMemberId: payload.properties.before.workspaceMember.id,
+      await this.connectedAccountRepository.getAllByWorkspaceMemberId(
-        handle: payload.properties.before.handle,
+        workspaceMemberId,
-      },
+        workspaceId,
+      );
+
+    if (!connectedAccount || connectedAccount.length === 0) {
+      return;
+    }
+
+    const messageChannel =
+      await this.messageChannelRepository.getByConnectedAccountId(
+        connectedAccount[0].id,
+        workspaceId,
+      );
+
+    await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
+      messageChannel[0].id,
+      workspaceId,
     );
   }
 
@@ -54,21 +76,36 @@ export class MessagingBlocklistListener {
   async handleUpdatedEvent(
     payload: ObjectRecordUpdateEvent<BlocklistWorkspaceEntity>,
   ) {
+    const workspaceMemberId = payload.properties.before.workspaceMember.id;
+    const workspaceId = payload.workspaceId;
+
     await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
       BlocklistItemDeleteMessagesJob.name,
       {
-        workspaceId: payload.workspaceId,
+        workspaceId,
         blocklistItemId: payload.recordId,
       },
     );
 
-    await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
+    const connectedAccount =
-      BlocklistReimportMessagesJob.name,
+      await this.connectedAccountRepository.getAllByWorkspaceMemberId(
-      {
+        workspaceMemberId,
-        workspaceId: payload.workspaceId,
+        workspaceId,
-        workspaceMemberId: payload.properties.after.workspaceMember.id,
+      );
-        handle: payload.properties.before.handle,
+
-      },
+    if (!connectedAccount || connectedAccount.length === 0) {
+      return;
+    }
+
+    const messageChannel =
+      await this.messageChannelRepository.getByConnectedAccountId(
+        connectedAccount[0].id,
+        workspaceId,
+      );
+
+    await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
+      messageChannel[0].id,
+      workspaceId,
     );
   }
 }

packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts

@@ -1,8 +1,19 @@
 import { Module } from '@nestjs/common';
 
+import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
+import { MessagingBlocklistListener } from 'src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener';
+import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
+import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module';
+
 @Module({
-  imports: [],
+  imports: [MessagingCommonModule, MessagingMessageCleanerModule],
-  providers: [],
+  providers: [
+    MessagingBlocklistListener,
+    {
+      provide: BlocklistItemDeleteMessagesJob.name,
+      useClass: BlocklistItemDeleteMessagesJob,
+    },
+  ],
   exports: [],
 })
 export class MessagingBlocklistManagerModule {}

packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts

@@ -67,9 +67,9 @@ export class MessageChannelMessageAssociationRepository {
     );
   }
 
-  public async deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles(
+  public async deleteByMessageParticipantHandleAndMessageChannelIdAndRoles(
     messageParticipantHandle: string,
-    messageChannelIds: string[],
+    messageChannelId: string,
     rolesToDelete: ('from' | 'to' | 'cc' | 'bcc')[],
     workspaceId: string,
     transactionManager?: EntityManager,
@@ -79,21 +79,35 @@ export class MessageChannelMessageAssociationRepository {
 
     const isHandleDomain = messageParticipantHandle.startsWith('@');
 
+    const messageChannel =
+      await this.workspaceDataSourceService.executeRawQuery(
+        `SELECT * FROM ${dataSourceSchema}."messageChannel"
+        WHERE "id" = $1`,
+        [messageChannelId],
+        workspaceId,
+        transactionManager,
+      );
+
+    const messageChannelHandle = messageChannel[0].handle;
+
     const messageChannelMessageAssociationIdsToDelete =
       await this.workspaceDataSourceService.executeRawQuery(
         `SELECT "messageChannelMessageAssociation".id
       FROM ${dataSourceSchema}."messageChannelMessageAssociation" "messageChannelMessageAssociation"
       JOIN ${dataSourceSchema}."message" ON "messageChannelMessageAssociation"."messageId" = ${dataSourceSchema}."message"."id"
       JOIN ${dataSourceSchema}."messageParticipant" "messageParticipant" ON ${dataSourceSchema}."message"."id" = "messageParticipant"."messageId"
-      WHERE "messageParticipant"."handle" ${
+      WHERE "messageParticipant"."handle" != $1
-        isHandleDomain ? 'ILIKE' : '='
+      AND "messageParticipant"."handle" ${isHandleDomain ? '~*' : '='} $2
-      } $1 AND "messageParticipant"."role" = ANY($2) AND "messageChannelMessageAssociation"."messageChannelId" = ANY($3)`,
+      AND "messageParticipant"."role" = ANY($3)
+      AND "messageChannelMessageAssociation"."messageChannelId" = $4`,
         [
+          messageChannelHandle,
           isHandleDomain
-            ? `%${messageParticipantHandle}`
+            ? // eslint-disable-next-line no-useless-escape
+              `.+@(.+\.)?${messageParticipantHandle.slice(1)}`
             : messageParticipantHandle,
           rolesToDelete,
-          messageChannelIds,
+          messageChannelId,
         ],
         workspaceId,
         transactionManager,

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts

@@ -111,6 +111,7 @@ export class MessagingGmailMessagesImportService {
       );
 
       const messagesToSave = filterEmails(
+        messageChannel.handle,
         allMessages,
         blocklist.map((blocklistItem) => blocklistItem.handle),
       );

packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts

@@ -2,14 +2,20 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u
 import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
 
 // Todo: refactor this into several utils
-export const filterEmails = (messages: GmailMessage[], blocklist: string[]) => {
+export const filterEmails = (
+  messageChannelHandle: string,
+  messages: GmailMessage[],
+  blocklist: string[],
+) => {
   return filterOutBlocklistedMessages(
+    messageChannelHandle,
     filterOutIcsAttachments(filterOutNonPersonalEmails(messages)),
     blocklist,
   );
 };
 
 const filterOutBlocklistedMessages = (
+  messageChannelHandle: string,
   messages: GmailMessage[],
   blocklist: string[],
 ) => {
@@ -19,7 +25,12 @@ const filterOutBlocklistedMessages = (
     }
 
     return message.participants.every(
-      (participant) => !isEmailBlocklisted(participant.handle, blocklist),
+      (participant) =>
+        !isEmailBlocklisted(
+          messageChannelHandle,
+          participant.handle,
+          blocklist,
+        ),
     );
   });
 };