5629 update blocklist for messaging v2 (#5756)

Closes #5629 - Add subdomain support in blocklist (if @example.com is blocked, every subdomain will be blocked)
2024-06-13 07:53:28 +02:00
parent 374237a988
commit f825bea071
11 changed files with 165 additions and 113 deletions
--- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/tests/is-email-blocklisted.util.spec.ts
+++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/tests/is-email-blocklisted.util.spec.ts
@ -2,38 +2,68 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u

 describe('isEmailBlocklisted', () => {
  it('should return true if email is blocklisted', () => {
-    const email = 'hello@example.com';
-    const blocklist = ['hello@example.com', 'hey@example.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const channelHandle = 'abc@example.com';
+    const email = 'hello@twenty.com';
+    const blocklist = ['hello@twenty.com', 'hey@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);

    expect(result).toBe(true);
  });
  it('should return false if email is not blocklisted', () => {
+    const channelHandle = 'abc@example.com';
    const email = 'hello@twenty.com';
    const blocklist = ['hey@example.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);

    expect(result).toBe(false);
  });
  it('should return false if email is null', () => {
+    const channelHandle = 'abc@twenty.com';
    const email = null;
    const blocklist = ['@example.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
+
+    expect(result).toBe(false);
+  });
+  it('should return true for subdomains', () => {
+    const channelHandle = 'abc@example.com';
+    const email = 'hello@twenty.twenty.com';
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
+
+    expect(result).toBe(true);
+  });
+  it('should return false for domains which end with blocklisted domain but are not subdomains', () => {
+    const channelHandle = 'abc@example.com';
+    const email = 'hello@twentytwenty.com';
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);

    expect(result).toBe(false);
  });
  it('should return false if email is undefined', () => {
+    const channelHandle = 'abc@example.com';
    const email = undefined;
-    const blocklist = ['@example.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);

    expect(result).toBe(false);
  });
  it('should return true if email ends with blocklisted domain', () => {
-    const email = 'hello@example.com';
-    const blocklist = ['@example.com'];
-    const result = isEmailBlocklisted(email, blocklist);
+    const channelHandle = 'abc@example.com';
+    const email = 'hello@twenty.com';
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);

    expect(result).toBe(true);
  });
+
+  it('should return false if email is same as channel handle', () => {
+    const channelHandle = 'hello@twenty.com';
+    const email = 'hello@twenty.com';
+    const blocklist = ['@twenty.com'];
+    const result = isEmailBlocklisted(channelHandle, email, blocklist);
+
+    expect(result).toBe(false);
+  });
 });
--- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts
+++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts
@ -1,14 +1,17 @@
 export const isEmailBlocklisted = (
+  channelHandle: string,
  email: string | null | undefined,
  blocklist: string[],
 ): boolean => {
-  if (!email) {
+  if (!email || email === channelHandle) {
    return false;
  }

  return blocklist.some((item) => {
    if (item.startsWith('@')) {
-      return email.endsWith(item);
+      const domain = email.split('@')[1];
+
+      return domain === item.slice(1) || domain.endsWith(`.${item.slice(1)}`);
    }

    return email === item;
--- a/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts
+++ b/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts
@ -124,14 +124,13 @@ export class GoogleCalendarSyncService {

    const blocklist = await this.getBlocklist(workspaceMemberId, workspaceId);

-    let filteredEvents = filterOutBlocklistedEvents(events, blocklist).filter(
-      (event) => event.status !== 'cancelled',
-    );
+    let filteredEvents = filterOutBlocklistedEvents(
+      calendarChannel.handle,
+      events,
+      blocklist,
+    ).filter((event) => event.status !== 'cancelled');

    if (emailOrDomainToReimport) {
-      // We still need to filter the events to only keep the ones that have the email or domain we want to reimport
-      // because the q parameter in the list method also filters the events that have the email or domain in their summary, description ...
-      // The q parameter allows us to narrow down the events
      filteredEvents = filteredEvents.filter(
        (event) =>
          event.attendees?.some(
--- a/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts
+++ b/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts
@ -3,6 +3,7 @@ import { calendar_v3 as calendarV3 } from 'googleapis';
 import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util';

 export const filterOutBlocklistedEvents = (
+  calendarChannelHandle: string,
  events: calendarV3.Schema$Event[],
  blocklist: string[],
 ) => {
@ -12,7 +13,8 @@ export const filterOutBlocklistedEvents = (
    }

    return event.attendees.every(
-      (attendee) => !isEmailBlocklisted(attendee.email, blocklist),
+      (attendee) =>
+        !isEmailBlocklisted(calendarChannelHandle, attendee.email, blocklist),
    );
  });
 };
--- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts
+++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts
@ -66,12 +66,14 @@ export class BlocklistItemDeleteMessagesJob

    const rolesToDelete: ('from' | 'to')[] = ['from', 'to'];

-    await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles(
-      handle,
-      messageChannelIds,
-      rolesToDelete,
-      workspaceId,
-    );
+    for (const messageChannelId of messageChannelIds) {
+      await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdAndRoles(
+        handle,
+        messageChannelId,
+        rolesToDelete,
+        workspaceId,
+      );
+    }

    await this.threadCleanerService.cleanWorkspaceThreads(workspaceId);

--- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts
+++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts
@ -1,58 +0,0 @@
-import { Injectable, Logger } from '@nestjs/common';
-
-import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
-
-import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
-import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
-import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
-
-export type BlocklistReimportMessagesJobData = {
-  workspaceId: string;
-  workspaceMemberId: string;
-  handle: string;
-};
-
-@Injectable()
-export class BlocklistReimportMessagesJob
-  implements MessageQueueJob<BlocklistReimportMessagesJobData>
-{
-  private readonly logger = new Logger(BlocklistReimportMessagesJob.name);
-
-  constructor(
-    @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
-    private readonly connectedAccountRepository: ConnectedAccountRepository,
-  ) {}
-
-  async handle(data: BlocklistReimportMessagesJobData): Promise<void> {
-    const { workspaceId, workspaceMemberId, handle } = data;
-
-    this.logger.log(
-      `Reimporting messages from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
-    );
-
-    const connectedAccount =
-      await this.connectedAccountRepository.getAllByWorkspaceMemberId(
-        workspaceMemberId,
-        workspaceId,
-      );
-
-    if (!connectedAccount || connectedAccount.length === 0) {
-      this.logger.error(
-        `No connected account found for workspace member ${workspaceMemberId} in workspace ${workspaceId}`,
-      );
-
-      return;
-    }
-
-    // TODO: reimplement that
-    // await this.gmailMessageListFetchJob.fetchConnectedAccountThreads(
-    //   workspaceId,
-    //   connectedAccount[0].id,
-    //   [handle],
-    // );
-
-    this.logger.log(
-      `Reimporting messages from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId} done`,
-    );
-  }
-}
--- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts
+++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts
@ -6,21 +6,28 @@ import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/t
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
 import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
-import {
-  BlocklistReimportMessagesJob,
-  BlocklistReimportMessagesJobData,
-} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job';
 import {
  BlocklistItemDeleteMessagesJobData,
  BlocklistItemDeleteMessagesJob,
 } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
 import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event';
+import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
+import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
+import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
+import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';

@Injectable()
 export class MessagingBlocklistListener {
  constructor(
    @Inject(MessageQueue.messagingQueue)
    private readonly messageQueueService: MessageQueueService,
+    @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
+    private readonly connectedAccountRepository: ConnectedAccountRepository,
+    private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService,
+    @InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
+    private readonly messageChannelRepository: MessageChannelRepository,
  ) {}

  @OnEvent('blocklist.created')
@ -40,13 +47,28 @@ export class MessagingBlocklistListener {
  async handleDeletedEvent(
    payload: ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>,
  ) {
-    await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
-      BlocklistReimportMessagesJob.name,
-      {
-        workspaceId: payload.workspaceId,
-        workspaceMemberId: payload.properties.before.workspaceMember.id,
-        handle: payload.properties.before.handle,
-      },
+    const workspaceMemberId = payload.properties.before.workspaceMember.id;
+    const workspaceId = payload.workspaceId;
+
+    const connectedAccount =
+      await this.connectedAccountRepository.getAllByWorkspaceMemberId(
+        workspaceMemberId,
+        workspaceId,
+      );
+
+    if (!connectedAccount || connectedAccount.length === 0) {
+      return;
+    }
+
+    const messageChannel =
+      await this.messageChannelRepository.getByConnectedAccountId(
+        connectedAccount[0].id,
+        workspaceId,
+      );
+
+    await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
+      messageChannel[0].id,
+      workspaceId,
    );
  }

@ -54,21 +76,36 @@ export class MessagingBlocklistListener {
  async handleUpdatedEvent(
    payload: ObjectRecordUpdateEvent<BlocklistWorkspaceEntity>,
  ) {
+    const workspaceMemberId = payload.properties.before.workspaceMember.id;
+    const workspaceId = payload.workspaceId;
+
    await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
      BlocklistItemDeleteMessagesJob.name,
      {
-        workspaceId: payload.workspaceId,
+        workspaceId,
        blocklistItemId: payload.recordId,
      },
    );

-    await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
-      BlocklistReimportMessagesJob.name,
-      {
-        workspaceId: payload.workspaceId,
-        workspaceMemberId: payload.properties.after.workspaceMember.id,
-        handle: payload.properties.before.handle,
-      },
+    const connectedAccount =
+      await this.connectedAccountRepository.getAllByWorkspaceMemberId(
+        workspaceMemberId,
+        workspaceId,
+      );
+
+    if (!connectedAccount || connectedAccount.length === 0) {
+      return;
+    }
+
+    const messageChannel =
+      await this.messageChannelRepository.getByConnectedAccountId(
+        connectedAccount[0].id,
+        workspaceId,
+      );
+
+    await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
+      messageChannel[0].id,
+      workspaceId,
    );
  }
 }
--- a/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts
+++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts
@ -1,8 +1,19 @@
 import { Module } from '@nestjs/common';

+import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
+import { MessagingBlocklistListener } from 'src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener';
+import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
+import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module';
+
@Module({
-  imports: [],
-  providers: [],
+  imports: [MessagingCommonModule, MessagingMessageCleanerModule],
+  providers: [
+    MessagingBlocklistListener,
+    {
+      provide: BlocklistItemDeleteMessagesJob.name,
+      useClass: BlocklistItemDeleteMessagesJob,
+    },
+  ],
  exports: [],
 })
 export class MessagingBlocklistManagerModule {}
--- a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts
+++ b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts
@ -67,9 +67,9 @@ export class MessageChannelMessageAssociationRepository {
    );
  }

-  public async deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles(
+  public async deleteByMessageParticipantHandleAndMessageChannelIdAndRoles(
    messageParticipantHandle: string,
-    messageChannelIds: string[],
+    messageChannelId: string,
    rolesToDelete: ('from' | 'to' | 'cc' | 'bcc')[],
    workspaceId: string,
    transactionManager?: EntityManager,
@ -79,21 +79,35 @@ export class MessageChannelMessageAssociationRepository {

    const isHandleDomain = messageParticipantHandle.startsWith('@');

+    const messageChannel =
+      await this.workspaceDataSourceService.executeRawQuery(
+        `SELECT * FROM ${dataSourceSchema}."messageChannel"
+        WHERE "id" = $1`,
+        [messageChannelId],
+        workspaceId,
+        transactionManager,
+      );
+
+    const messageChannelHandle = messageChannel[0].handle;
+
    const messageChannelMessageAssociationIdsToDelete =
      await this.workspaceDataSourceService.executeRawQuery(
        `SELECT "messageChannelMessageAssociation".id
      FROM ${dataSourceSchema}."messageChannelMessageAssociation" "messageChannelMessageAssociation"
      JOIN ${dataSourceSchema}."message" ON "messageChannelMessageAssociation"."messageId" = ${dataSourceSchema}."message"."id"
      JOIN ${dataSourceSchema}."messageParticipant" "messageParticipant" ON ${dataSourceSchema}."message"."id" = "messageParticipant"."messageId"
-      WHERE "messageParticipant"."handle" ${
-        isHandleDomain ? 'ILIKE' : '='
-      } $1 AND "messageParticipant"."role" = ANY($2) AND "messageChannelMessageAssociation"."messageChannelId" = ANY($3)`,
+      WHERE "messageParticipant"."handle" != $1
+      AND "messageParticipant"."handle" ${isHandleDomain ? '~*' : '='} $2
+      AND "messageParticipant"."role" = ANY($3)
+      AND "messageChannelMessageAssociation"."messageChannelId" = $4`,
        [
+          messageChannelHandle,
          isHandleDomain
-            ? `%${messageParticipantHandle}`
+            ? // eslint-disable-next-line no-useless-escape
+              `.+@(.+\.)?${messageParticipantHandle.slice(1)}`
            : messageParticipantHandle,
          rolesToDelete,
-          messageChannelIds,
+          messageChannelId,
        ],
        workspaceId,
        transactionManager,
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts
@ -111,6 +111,7 @@ export class MessagingGmailMessagesImportService {
      );

      const messagesToSave = filterEmails(
+        messageChannel.handle,
        allMessages,
        blocklist.map((blocklistItem) => blocklistItem.handle),
      );
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts
@ -2,14 +2,20 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u
 import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';

 // Todo: refactor this into several utils
-export const filterEmails = (messages: GmailMessage[], blocklist: string[]) => {
+export const filterEmails = (
+  messageChannelHandle: string,
+  messages: GmailMessage[],
+  blocklist: string[],
+) => {
  return filterOutBlocklistedMessages(
+    messageChannelHandle,
    filterOutIcsAttachments(filterOutNonPersonalEmails(messages)),
    blocklist,
  );
 };

 const filterOutBlocklistedMessages = (
+  messageChannelHandle: string,
  messages: GmailMessage[],
  blocklist: string[],
 ) => {
@ -19,7 +25,12 @@ const filterOutBlocklistedMessages = (
    }

    return message.participants.every(
-      (participant) => !isEmailBlocklisted(participant.handle, blocklist),
+      (participant) =>
+        !isEmailBlocklisted(
+          messageChannelHandle,
+          participant.handle,
+          blocklist,
+        ),
    );
  });
 };