4209 speed up gmail full sync by using search params to query only the relevant emails (#4213)

* create blocklist service

* blocklist is working on email import in full sync

* add log

* add blocklist to partial sync

* define rule for blocklist imports

* gmail filter is working

* correct typo

* fix bugs

* getCompanyNameFromDomainName

* renaming

* remove unused service

* add transaction

packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts

@@ -11,10 +11,12 @@ import {
   GmailFullSyncJobData,
 } from 'src/workspace/messaging/jobs/gmail-full-sync.job';
 import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
-import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
 import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
 import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
 import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
+import { GmailMessage } from 'src/workspace/messaging/types/gmail-message';
+import { isPersonEmail } from 'src/workspace/messaging/utils/is-person-email.util';
+import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
 import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
 
 @Injectable()
@@ -26,10 +28,10 @@ export class GmailPartialSyncService {
     private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
     @Inject(MessageQueue.messagingQueue)
     private readonly messageQueueService: MessageQueueService,
-    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
     private readonly connectedAccountService: ConnectedAccountService,
     private readonly messageChannelService: MessageChannelService,
     private readonly messageService: MessageService,
+    private readonly blocklistService: BlocklistService,
     private readonly saveMessagesAndCreateContactsService: SaveMessagesAndCreateContactsService,
   ) {}
 
@@ -118,7 +120,7 @@ export class GmailPartialSyncService {
 
     const messageQueries = createQueriesFromMessageIds(messagesAdded);
 
-    const { messages: messagesToSave, errors } =
+    const { messages, errors } =
       await this.fetchMessagesByBatchesService.fetchAllMessages(
         messageQueries,
         accessToken,
@@ -127,6 +129,22 @@ export class GmailPartialSyncService {
         connectedAccountId,
       );
 
+    const blocklist = await this.blocklistService.getByWorkspaceMemberId(
+      connectedAccount.accountOwnerId,
+      workspaceId,
+    );
+
+    const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
+
+    const messagesToSave = messages.filter(
+      (message) =>
+        !this.shouldSkipImport(
+          connectedAccount.handle,
+          message,
+          blocklistedEmails,
+        ),
+    );
+
     if (messagesToSave.length !== 0) {
       await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
         messagesToSave,
@@ -292,4 +310,34 @@ export class GmailPartialSyncService {
       },
     );
   }
+
+  private isHandleBlocked = (
+    selfHandle: string,
+    message: GmailMessage,
+    blocklistedEmails: string[],
+  ): boolean => {
+    // If the message is received, check if the sender is in the blocklist
+    // If the message is sent, check if any of the recipients with role 'to' is in the blocklist
+
+    if (message.fromHandle === selfHandle) {
+      return message.participants.some(
+        (participant) =>
+          participant.role === 'to' &&
+          blocklistedEmails.includes(participant.handle),
+      );
+    }
+
+    return blocklistedEmails.includes(message.fromHandle);
+  };
+
+  private shouldSkipImport(
+    selfHandle: string,
+    message: GmailMessage,
+    blocklistedEmails: string[],
+  ): boolean {
+    return (
+      !isPersonEmail(message.fromHandle) ||
+      this.isHandleBlocked(selfHandle, message, blocklistedEmails)
+    );
+  }
 }