3807 auto creation of contact when importing emails (#3888)

* Add CreateContactService to messaging services

* Add logic to create a contact if it doesn't exist

* Add name

* Improvements

* contact creation working

* fix bug

* Add IsPersonEmailService to check if an email is personal or not

* filter is working

* improve filter

* create companies and people

* Refactor createContactFromHandleAndDisplayName to createContactAndCompanyFromHandleAndDisplayName

* improve regex

* reorganizing services

* updates

* reorganize folders

* wip

* use transaction

* wip

* wip

* wip

* batch queries

* almost working

* working

packages/twenty-server/src/workspace/messaging/services/create-company.service.ts

@@ -1,65 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-import { EntityManager } from 'typeorm';
-import axios, { AxiosInstance } from 'axios';
-import { v4 } from 'uuid';
-
-import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity';
-import { capitalize } from 'src/utils/capitalize';
-@Injectable()
-export class CreateCompanyService {
-  private readonly httpService: AxiosInstance;
-
-  constructor() {
-    this.httpService = axios.create({
-      baseURL: 'https://companies.twenty.com',
-    });
-  }
-
-  async createCompanyFromDomainName(
-    domainName: string,
-    dataSourceMetadata: DataSourceEntity,
-    manager: EntityManager,
-  ): Promise<string> {
-    const existingCompany = await manager.query(
-      `SELECT * FROM ${dataSourceMetadata.schema}.company WHERE "domainName" = '${domainName}'`,
-    );
-
-    if (existingCompany.length > 0) {
-      return existingCompany[0].id;
-    }
-
-    const companyId = v4();
-
-    const { name, city } = await this.getCompanyInfoFromDomainName(domainName);
-
-    await manager.query(
-      `INSERT INTO ${dataSourceMetadata.schema}.company (id, name, "domainName", address)
-      VALUES ($1, $2, $3, $4)`,
-      [companyId, name, domainName, city],
-    );
-
-    return companyId;
-  }
-
-  async getCompanyInfoFromDomainName(domainName: string): Promise<{
-    name: string;
-    city: string;
-  }> {
-    try {
-      const response = await this.httpService.get(`/${domainName}`);
-
-      const data = response.data;
-
-      return {
-        name: data.name,
-        city: data.city,
-      };
-    } catch (e) {
-      return {
-        name: capitalize(domainName.split('.')[0]),
-        city: '',
-      };
-    }
-  }
-}

packages/twenty-server/src/workspace/messaging/services/fetch-messages-by-batches.service.ts

@@ -10,12 +10,13 @@ import {
 } from 'src/workspace/messaging/types/gmail-message';
 import { MessageQuery } from 'src/workspace/messaging/types/message-or-thread-query';
 import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmail-message-parsed-response';
+import { IsPersonEmailService } from 'src/workspace/messaging/services/is-person-email.service';
 
 @Injectable()
 export class FetchMessagesByBatchesService {
   private readonly httpService: AxiosInstance;
 
-  constructor() {
+  constructor(private readonly isPersonEmailService: IsPersonEmailService) {
     this.httpService = axios.create({
       baseURL: 'https://www.googleapis.com/batch/gmail/v1',
     });
@@ -189,6 +190,8 @@ export class FetchMessagesByBatchesService {
           } = parsed;
 
           if (!from) throw new Error('From value is missing');
+          if (!this.isPersonEmailService.isPersonEmail(from.value[0].address))
+            return;
           if (!to) throw new Error('To value is missing');
 
           const participants = [

packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts

@@ -2,7 +2,6 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
 
 import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service';
 import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
-import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
 import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
 import {
@@ -13,6 +12,8 @@ import { ConnectedAccountService } from 'src/workspace/messaging/connected-accou
 import { MessageChannelService } from 'src/workspace/messaging/message-channel/message-channel.service';
 import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/message-channel-message-association/message-channel-message-association.service';
 import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
+import { CreateQueriesFromMessageIdsService } from 'src/workspace/messaging/services/utils/create-queries-from-message-ids.service';
+import { MessageService } from 'src/workspace/messaging/message/message.service';
 
 @Injectable()
 export class GmailFullSyncService {
@@ -21,13 +22,14 @@ export class GmailFullSyncService {
   constructor(
     private readonly gmailClientProvider: GmailClientProvider,
     private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
-    private readonly utils: MessagingUtilsService,
     @Inject(MessageQueue.messagingQueue)
     private readonly messageQueueService: MessageQueueService,
     private readonly workspaceDataSourceService: WorkspaceDataSourceService,
     private readonly connectedAccountService: ConnectedAccountService,
     private readonly messageChannelService: MessageChannelService,
     private readonly messageChannelMessageAssociationService: MessageChannelMessageAssociationService,
+    private readonly messageService: MessageService,
+    private readonly createQueriesFromMessageIdsService: CreateQueriesFromMessageIdsService,
   ) {}
 
   public async fetchConnectedAccountThreads(
@@ -100,7 +102,9 @@ export class GmailFullSyncService {
     );
 
     const messageQueries =
-      this.utils.createQueriesFromMessageIds(messagesToFetch);
+      this.createQueriesFromMessageIdsService.createQueriesFromMessageIds(
+        messagesToFetch,
+      );
 
     const { messages: messagesToSave, errors } =
       await this.fetchMessagesByBatchesService.fetchAllMessages(
@@ -116,7 +120,7 @@ export class GmailFullSyncService {
       return;
     }
 
-    await this.utils.saveMessages(
+    await this.messageService.saveMessages(
       messagesToSave,
       dataSourceMetadata,
       workspaceDataSource,

packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts

@@ -13,7 +13,8 @@ import {
 import { ConnectedAccountService } from 'src/workspace/messaging/connected-account/connected-account.service';
 import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
 import { MessageChannelService } from 'src/workspace/messaging/message-channel/message-channel.service';
-import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
+import { MessageService } from 'src/workspace/messaging/message/message.service';
+import { CreateQueriesFromMessageIdsService } from 'src/workspace/messaging/services/utils/create-queries-from-message-ids.service';
 
 @Injectable()
 export class GmailPartialSyncService {
@@ -22,12 +23,13 @@ export class GmailPartialSyncService {
   constructor(
     private readonly gmailClientProvider: GmailClientProvider,
     private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
-    private readonly utils: MessagingUtilsService,
     @Inject(MessageQueue.messagingQueue)
     private readonly messageQueueService: MessageQueueService,
     private readonly workspaceDataSourceService: WorkspaceDataSourceService,
     private readonly connectedAccountService: ConnectedAccountService,
     private readonly messageChannelService: MessageChannelService,
+    private readonly messageService: MessageService,
+    private readonly createQueriesFromMessageIdsService: CreateQueriesFromMessageIdsService,
   ) {}
 
   public async fetchConnectedAccountThreads(
@@ -98,7 +100,9 @@ export class GmailPartialSyncService {
       await this.getMessageIdsFromHistory(history);
 
     const messageQueries =
-      this.utils.createQueriesFromMessageIds(messagesAdded);
+      this.createQueriesFromMessageIdsService.createQueriesFromMessageIds(
+        messagesAdded,
+      );
 
     const { messages: messagesToSave, errors } =
       await this.fetchMessagesByBatchesService.fetchAllMessages(
@@ -107,7 +111,7 @@ export class GmailPartialSyncService {
       );
 
     if (messagesToSave.length !== 0) {
-      await this.utils.saveMessages(
+      await this.messageService.saveMessages(
         messagesToSave,
         dataSourceMetadata,
         workspaceDataSource,
@@ -118,7 +122,7 @@ export class GmailPartialSyncService {
     }
 
     if (messagesDeleted.length !== 0) {
-      await this.utils.deleteMessages(
+      await this.messageService.deleteMessages(
         workspaceDataSource,
         messagesDeleted,
         gmailMessageChannelId,

packages/twenty-server/src/workspace/messaging/services/is-person-email.service.ts

@@ -0,0 +1,15 @@
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class IsPersonEmailService {
+  constructor() {}
+
+  isPersonEmail(email: string | undefined): boolean {
+    if (!email) return false;
+
+    const nonPersonalPattern =
+      /noreply|no-reply|do_not_reply|no\.reply|^(accounts@|info@|admin@|contact@|hello@|support@|sales@|feedback@|service@|help@|mailer-daemon|notifications?|digest|auto|apps|assign|comments|customer-success|enterprise|esign|express|forum|gc@|learn|mailer|marketing|messages|news|notification|payments|receipts|recrutement|security|service|support|team)/;
+
+    return !nonPersonalPattern.test(email);
+  }
+}

packages/twenty-server/src/workspace/messaging/services/messaging-utils.service.ts

@@ -1,310 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-import { EntityManager, DataSource } from 'typeorm';
-import { v4 } from 'uuid';
-
-import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity';
-import {
-  GmailMessage,
-  Participant,
-} from 'src/workspace/messaging/types/gmail-message';
-import { MessageQuery } from 'src/workspace/messaging/types/message-or-thread-query';
-import { MessageChannelMessageAssociationService } from 'src/workspace/messaging/message-channel-message-association/message-channel-message-association.service';
-import { MessageService } from 'src/workspace/messaging/message/message.service';
-import { MessageThreadService } from 'src/workspace/messaging/message-thread/message-thread.service';
-import { ObjectRecord } from 'src/workspace/workspace-sync-metadata/types/object-record';
-import { ConnectedAccountObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/connected-account.object-metadata';
-import { CreateCompanyService } from 'src/workspace/messaging/services/create-company.service';
-
-@Injectable()
-export class MessagingUtilsService {
-  constructor(
-    private readonly messageChannelMessageAssociationService: MessageChannelMessageAssociationService,
-    private readonly messageService: MessageService,
-    private readonly messageThreadService: MessageThreadService,
-    private readonly createCompaniesService: CreateCompanyService,
-  ) {}
-
-  public createQueriesFromMessageIds(
-    messageExternalIds: string[],
-  ): MessageQuery[] {
-    return messageExternalIds.map((messageId) => ({
-      uri: '/gmail/v1/users/me/messages/' + messageId + '?format=RAW',
-    }));
-  }
-
-  public async saveMessages(
-    messages: GmailMessage[],
-    dataSourceMetadata: DataSourceEntity,
-    workspaceDataSource: DataSource,
-    connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
-    gmailMessageChannelId: string,
-    workspaceId: string,
-  ) {
-    for (const message of messages) {
-      await workspaceDataSource?.transaction(async (manager: EntityManager) => {
-        const existingMessageChannelMessageAssociationsCount =
-          await this.messageChannelMessageAssociationService.countByMessageExternalIdsAndMessageChannelId(
-            [message.externalId],
-            gmailMessageChannelId,
-            workspaceId,
-            manager,
-          );
-
-        if (existingMessageChannelMessageAssociationsCount > 0) {
-          return;
-        }
-
-        const savedOrExistingMessageThreadId =
-          await this.saveMessageThreadOrReturnExistingMessageThread(
-            message.messageThreadExternalId,
-            dataSourceMetadata,
-            workspaceId,
-            manager,
-          );
-
-        const savedOrExistingMessageId =
-          await this.saveMessageOrReturnExistingMessage(
-            message,
-            savedOrExistingMessageThreadId,
-            connectedAccount,
-            dataSourceMetadata,
-            workspaceId,
-            manager,
-          );
-
-        await manager.query(
-          `INSERT INTO ${dataSourceMetadata.schema}."messageChannelMessageAssociation" ("messageChannelId", "messageId", "messageExternalId", "messageThreadId", "messageThreadExternalId") VALUES ($1, $2, $3, $4, $5)`,
-          [
-            gmailMessageChannelId,
-            savedOrExistingMessageId,
-            message.externalId,
-            savedOrExistingMessageThreadId,
-            message.messageThreadExternalId,
-          ],
-        );
-      });
-    }
-  }
-
-  private async saveMessageOrReturnExistingMessage(
-    message: GmailMessage,
-    messageThreadId: string,
-    connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
-    dataSourceMetadata: DataSourceEntity,
-    workspaceId: string,
-    manager: EntityManager,
-  ): Promise<string> {
-    const existingMessage = await this.messageService.getFirstByHeaderMessageId(
-      message.headerMessageId,
-      workspaceId,
-    );
-    const existingMessageId = existingMessage?.id;
-
-    if (existingMessageId) {
-      return Promise.resolve(existingMessageId);
-    }
-
-    const newMessageId = v4();
-
-    const messageDirection =
-      connectedAccount.handle === message.fromHandle ? 'outgoing' : 'incoming';
-
-    const receivedAt = new Date(parseInt(message.internalDate));
-
-    await manager.query(
-      `INSERT INTO ${dataSourceMetadata.schema}."message" ("id", "headerMessageId", "subject", "receivedAt", "direction", "messageThreadId", "text", "html") VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
-      [
-        newMessageId,
-        message.headerMessageId,
-        message.subject,
-        receivedAt,
-        messageDirection,
-        messageThreadId,
-        message.text,
-        message.html,
-      ],
-    );
-
-    await this.saveMessageParticipants(
-      message.participants,
-      newMessageId,
-      dataSourceMetadata,
-      manager,
-    );
-
-    return Promise.resolve(newMessageId);
-  }
-
-  private async saveMessageThreadOrReturnExistingMessageThread(
-    messageThreadExternalId: string,
-    dataSourceMetadata: DataSourceEntity,
-    workspaceId: string,
-    manager: EntityManager,
-  ) {
-    const existingMessageChannelMessageAssociationByMessageThreadExternalId =
-      await this.messageChannelMessageAssociationService.getFirstByMessageThreadExternalId(
-        messageThreadExternalId,
-        workspaceId,
-        manager,
-      );
-
-    const existingMessageThread =
-      existingMessageChannelMessageAssociationByMessageThreadExternalId?.messageThreadId;
-
-    if (existingMessageThread) {
-      return Promise.resolve(existingMessageThread);
-    }
-
-    const newMessageThreadId = v4();
-
-    await manager.query(
-      `INSERT INTO ${dataSourceMetadata.schema}."messageThread" ("id") VALUES ($1)`,
-      [newMessageThreadId],
-    );
-
-    return Promise.resolve(newMessageThreadId);
-  }
-
-  private async saveMessageParticipants(
-    participants: Participant[],
-    messageId: string,
-    dataSourceMetadata: DataSourceEntity,
-    manager: EntityManager,
-  ): Promise<void> {
-    if (!participants) return;
-
-    for (const participant of participants) {
-      const participantPerson = await manager.query(
-        `SELECT "person"."id" FROM ${dataSourceMetadata.schema}."person" WHERE "email" = $1 LIMIT 1`,
-        [participant.handle],
-      );
-
-      const participantPersonId = participantPerson[0]?.id;
-
-      const workspaceMember = await manager.query(
-        `SELECT "workspaceMember"."id" FROM ${dataSourceMetadata.schema}."workspaceMember"
-          JOIN ${dataSourceMetadata.schema}."connectedAccount" ON ${dataSourceMetadata.schema}."workspaceMember"."id" = ${dataSourceMetadata.schema}."connectedAccount"."accountOwnerId"
-          WHERE ${dataSourceMetadata.schema}."connectedAccount"."handle" = $1
-          LIMIT 1`,
-        [participant.handle],
-      );
-
-      const participantWorkspaceMemberId = workspaceMember[0]?.id;
-
-      await manager.query(
-        `INSERT INTO ${dataSourceMetadata.schema}."messageParticipant" ("messageId", "role", "handle", "displayName", "personId", "workspaceMemberId") VALUES ($1, $2, $3, $4, $5, $6)`,
-        [
-          messageId,
-          participant.role,
-          participant.handle,
-          participant.displayName,
-          participantPersonId,
-          participantWorkspaceMemberId,
-        ],
-      );
-
-      const companyDomainName = participant.handle
-        .split('@')?.[1]
-        .split('.')
-        .slice(-2)
-        .join('.')
-        .toLowerCase();
-
-      await this.createCompaniesService.createCompanyFromDomainName(
-        companyDomainName,
-        dataSourceMetadata,
-        manager,
-      );
-    }
-  }
-
-  public async deleteMessages(
-    workspaceDataSource: DataSource,
-    messagesDeletedMessageExternalIds: string[],
-    gmailMessageChannelId: string,
-    workspaceId: string,
-  ) {
-    await workspaceDataSource?.transaction(async (manager: EntityManager) => {
-      const messageChannelMessageAssociationsToDelete =
-        await this.messageChannelMessageAssociationService.getByMessageExternalIdsAndMessageChannelId(
-          messagesDeletedMessageExternalIds,
-          gmailMessageChannelId,
-          workspaceId,
-          manager,
-        );
-
-      const messageChannelMessageAssociationIdsToDeleteIds =
-        messageChannelMessageAssociationsToDelete.map(
-          (messageChannelMessageAssociationToDelete) =>
-            messageChannelMessageAssociationToDelete.id,
-        );
-
-      await this.messageChannelMessageAssociationService.deleteByIds(
-        messageChannelMessageAssociationIdsToDeleteIds,
-        workspaceId,
-        manager,
-      );
-
-      const messageIdsFromMessageChannelMessageAssociationsToDelete =
-        messageChannelMessageAssociationsToDelete.map(
-          (messageChannelMessageAssociationToDelete) =>
-            messageChannelMessageAssociationToDelete.messageId,
-        );
-
-      const messageChannelMessageAssociationByMessageIds =
-        await this.messageChannelMessageAssociationService.getByMessageIds(
-          messageIdsFromMessageChannelMessageAssociationsToDelete,
-          workspaceId,
-          manager,
-        );
-
-      const messageIdsFromMessageChannelMessageAssociationByMessageIds =
-        messageChannelMessageAssociationByMessageIds.map(
-          (messageChannelMessageAssociation) =>
-            messageChannelMessageAssociation.messageId,
-        );
-
-      const messageIdsToDelete =
-        messageIdsFromMessageChannelMessageAssociationsToDelete.filter(
-          (messageId) =>
-            !messageIdsFromMessageChannelMessageAssociationByMessageIds.includes(
-              messageId,
-            ),
-        );
-
-      await this.messageService.deleteByIds(
-        messageIdsToDelete,
-        workspaceId,
-        manager,
-      );
-
-      const messageThreadIdsFromMessageChannelMessageAssociationsToDelete =
-        messageChannelMessageAssociationsToDelete.map(
-          (messageChannelMessageAssociationToDelete) =>
-            messageChannelMessageAssociationToDelete.messageThreadId,
-        );
-
-      const messagesByThreadIds =
-        await this.messageService.getByMessageThreadIds(
-          messageThreadIdsFromMessageChannelMessageAssociationsToDelete,
-          workspaceId,
-          manager,
-        );
-
-      const threadIdsToDelete =
-        messageThreadIdsFromMessageChannelMessageAssociationsToDelete.filter(
-          (threadId) =>
-            !messagesByThreadIds.find(
-              (message) => message.messageThreadId === threadId,
-            ),
-        );
-
-      await this.messageThreadService.deleteByIds(
-        threadIdsToDelete,
-        workspaceId,
-        manager,
-      );
-    });
-  }
-}

packages/twenty-server/src/workspace/messaging/services/utils/create-queries-from-message-ids.service.ts

@@ -0,0 +1,16 @@
+import { Injectable } from '@nestjs/common';
+
+import { MessageQuery } from 'src/workspace/messaging/types/message-or-thread-query';
+
+@Injectable()
+export class CreateQueriesFromMessageIdsService {
+  constructor() {}
+
+  public createQueriesFromMessageIds(
+    messageExternalIds: string[],
+  ): MessageQuery[] {
+    return messageExternalIds.map((messageId) => ({
+      uri: '/gmail/v1/users/me/messages/' + messageId + '?format=RAW',
+    }));
+  }
+}