4209 speed up gmail full sync by using search params to query only the relevant emails (#4213)

* create blocklist service

* blocklist is working on email import in full sync

* add log

* add blocklist to partial sync

* define rule for blocklist imports

* gmail filter is working

* correct typo

* fix bugs

* getCompanyNameFromDomainName

* renaming

* remove unused service

* add transaction

packages/twenty-server/src/workspace/messaging/services/create-company/create-company.service.ts

@@ -5,7 +5,7 @@ import { v4 } from 'uuid';
 import axios, { AxiosInstance } from 'axios';
 
 import { CompanyService } from 'src/workspace/messaging/repositories/company/company.service';
-import { capitalize } from 'src/utils/capitalize';
+import { getCompanyNameFromDomainName } from 'src/workspace/messaging/utils/get-company-name-from-domain-name.util';
 @Injectable()
 export class CreateCompanyService {
   private readonly httpService: AxiosInstance;
@@ -100,12 +100,12 @@ export class CreateCompanyService {
       const data = response.data;
 
       return {
-        name: data.name,
+        name: data.name ?? getCompanyNameFromDomainName(domainName),
         city: data.city,
       };
     } catch (e) {
       return {
-        name: capitalize(domainName.split('.')[0]),
+        name: getCompanyNameFromDomainName(domainName),
         city: '',
       };
     }

packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts

@@ -12,6 +12,8 @@ import { ConnectedAccountService } from 'src/workspace/messaging/repositories/co
 import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
 import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/repositories/message-channel-message-association/message-channel-message-association.service';
 import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
+import { gmailSearchFilterExcludeEmails } from 'src/workspace/messaging/utils/gmail-search-filter';
+import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
 import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
 
 @Injectable()
@@ -26,6 +28,7 @@ export class GmailFullSyncService {
     private readonly connectedAccountService: ConnectedAccountService,
     private readonly messageChannelService: MessageChannelService,
     private readonly messageChannelMessageAssociationService: MessageChannelMessageAssociationService,
+    private readonly blocklistService: BlocklistService,
     private readonly saveMessagesAndCreateContactsService: SaveMessagesAndCreateContactsService,
   ) {}
 
@@ -41,6 +44,7 @@ export class GmailFullSyncService {
 
     const accessToken = connectedAccount.accessToken;
     const refreshToken = connectedAccount.refreshToken;
+    const workspaceMemberId = connectedAccount.accountOwnerId;
 
     if (!refreshToken) {
       throw new Error('No refresh token found');
@@ -57,12 +61,19 @@ export class GmailFullSyncService {
     const gmailClient =
       await this.gmailClientProvider.getGmailClient(refreshToken);
 
+    const blocklist = await this.blocklistService.getByWorkspaceMemberId(
+      workspaceMemberId,
+      workspaceId,
+    );
+
+    const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
     let startTime = Date.now();
 
     const messages = await gmailClient.users.messages.list({
       userId: 'me',
       maxResults: 500,
       pageToken: nextPageToken,
+      q: gmailSearchFilterExcludeEmails(blocklistedEmails),
     });
 
     let endTime = Date.now();
@@ -80,6 +91,10 @@ export class GmailFullSyncService {
       : [];
 
     if (!messageExternalIds || messageExternalIds?.length === 0) {
+      this.logger.log(
+        `gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
+      );
+
       return;
     }
 

packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts

@@ -11,10 +11,12 @@ import {
   GmailFullSyncJobData,
 } from 'src/workspace/messaging/jobs/gmail-full-sync.job';
 import { ConnectedAccountService } from 'src/workspace/messaging/repositories/connected-account/connected-account.service';
-import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
 import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
 import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
 import { createQueriesFromMessageIds } from 'src/workspace/messaging/utils/create-queries-from-message-ids.util';
+import { GmailMessage } from 'src/workspace/messaging/types/gmail-message';
+import { isPersonEmail } from 'src/workspace/messaging/utils/is-person-email.util';
+import { BlocklistService } from 'src/workspace/messaging/repositories/blocklist/blocklist.service';
 import { SaveMessagesAndCreateContactsService } from 'src/workspace/messaging/services/save-messages-and-create-contacts.service';
 
 @Injectable()
@@ -26,10 +28,10 @@ export class GmailPartialSyncService {
     private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
     @Inject(MessageQueue.messagingQueue)
     private readonly messageQueueService: MessageQueueService,
-    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
     private readonly connectedAccountService: ConnectedAccountService,
     private readonly messageChannelService: MessageChannelService,
     private readonly messageService: MessageService,
+    private readonly blocklistService: BlocklistService,
     private readonly saveMessagesAndCreateContactsService: SaveMessagesAndCreateContactsService,
   ) {}
 
@@ -118,7 +120,7 @@ export class GmailPartialSyncService {
 
     const messageQueries = createQueriesFromMessageIds(messagesAdded);
 
-    const { messages: messagesToSave, errors } =
+    const { messages, errors } =
       await this.fetchMessagesByBatchesService.fetchAllMessages(
         messageQueries,
         accessToken,
@@ -127,6 +129,22 @@ export class GmailPartialSyncService {
         connectedAccountId,
       );
 
+    const blocklist = await this.blocklistService.getByWorkspaceMemberId(
+      connectedAccount.accountOwnerId,
+      workspaceId,
+    );
+
+    const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
+
+    const messagesToSave = messages.filter(
+      (message) =>
+        !this.shouldSkipImport(
+          connectedAccount.handle,
+          message,
+          blocklistedEmails,
+        ),
+    );
+
     if (messagesToSave.length !== 0) {
       await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
         messagesToSave,
@@ -292,4 +310,34 @@ export class GmailPartialSyncService {
       },
     );
   }
+
+  private isHandleBlocked = (
+    selfHandle: string,
+    message: GmailMessage,
+    blocklistedEmails: string[],
+  ): boolean => {
+    // If the message is received, check if the sender is in the blocklist
+    // If the message is sent, check if any of the recipients with role 'to' is in the blocklist
+
+    if (message.fromHandle === selfHandle) {
+      return message.participants.some(
+        (participant) =>
+          participant.role === 'to' &&
+          blocklistedEmails.includes(participant.handle),
+      );
+    }
+
+    return blocklistedEmails.includes(message.fromHandle);
+  };
+
+  private shouldSkipImport(
+    selfHandle: string,
+    message: GmailMessage,
+    blocklistedEmails: string[],
+  ): boolean {
+    return (
+      !isPersonEmail(message.fromHandle) ||
+      this.isHandleBlocked(selfHandle, message, blocklistedEmails)
+    );
+  }
 }

packages/twenty-server/src/workspace/messaging/services/save-messages-and-create-contacts.service.ts

@@ -1,5 +1,7 @@
 import { Injectable, Logger } from '@nestjs/common';
 
+import { EntityManager } from 'typeorm';
+
 import { MessageChannelService } from 'src/workspace/messaging/repositories/message-channel/message-channel.service';
 import { MessageParticipantService } from 'src/workspace/messaging/repositories/message-participant/message-participant.service';
 import { MessageService } from 'src/workspace/messaging/repositories/message/message.service';
@@ -82,10 +84,15 @@ export class SaveMessagesAndCreateContactsService {
     if (isContactAutoCreationEnabled) {
       startTime = Date.now();
 
-      await this.createCompaniesAndContactsService.createCompaniesAndContacts(
-        connectedAccount.handle,
-        contactsToCreate,
-        workspaceId,
+      await workspaceDataSource?.transaction(
+        async (transactionManager: EntityManager) => {
+          await this.createCompaniesAndContactsService.createCompaniesAndContacts(
+            connectedAccount.handle,
+            contactsToCreate,
+            workspaceId,
+            transactionManager,
+          );
+        },
       );
 
       const handles = participantsWithMessageId.map(