Add message import granulary on non-pro emails, group emails and received contact creation (#6156)

1) Remove featureFlag 2) Base contactCreation on messageChannel.autoContactCreationPolicy 4) add excludeProfessionalEmails + excludeGroupEmails logic
2024-07-08 14:33:48 +02:00
parent ef849d316f
commit 9ba211055a
17 changed files with 135 additions and 147 deletions
--- a/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts
+++ b/packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts
@ -1,30 +1,30 @@
 import { Injectable } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';

-import { EntityManager } from 'typeorm';
-import compact from 'lodash.compact';
 import chunk from 'lodash.chunk';
+import compact from 'lodash.compact';
+import { EntityManager } from 'typeorm';

-import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
-import { CreateCompanyService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company/create-company.service';
-import { CreateContactService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-contact/create-contact.service';
-import { PersonRepository } from 'src/modules/person/repositories/person.repository';
-import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository';
-import { isWorkEmail } from 'src/utils/is-work-email';
 import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
-import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
-import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
-import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
-import { filterOutSelfAndContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util';
-import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
-import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
-import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
 import { CalendarEventParticipantService } from 'src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service';
-import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant';
-import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
 import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
+import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant';
+import { CreateCompanyService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company/create-company.service';
+import { CreateContactService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-contact/create-contact.service';
+import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
+import { filterOutSelfAndContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util';
+import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
+import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
+import { PersonRepository } from 'src/modules/person/repositories/person.repository';
+import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
+import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository';
+import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
+import { isWorkEmail } from 'src/utils/is-work-email';

@Injectable()
 export class CreateCompanyAndContactService {
@ -52,9 +52,6 @@ export class CreateCompanyAndContactService {
      return [];
    }

-    // TODO: This is a feature that may be implemented in the future
-    const isContactAutoCreationForNonWorkEmailsEnabled = false;
-
    const workspaceMembers =
      await this.workspaceMemberRepository.getAllByWorkspaceId(
        workspaceId,
@ -89,9 +86,7 @@ export class CreateCompanyAndContactService {
    const filteredContactsToCreate = uniqueContacts.filter(
      (participant) =>
        !alreadyCreatedContactEmails.includes(participant.handle) &&
-        participant.handle.includes('@') &&
-        (isContactAutoCreationForNonWorkEmailsEnabled ||
-          isWorkEmail(participant.handle)),
+        participant.handle.includes('@'),
    );

    const filteredContactsToCreateWithCompanyDomainNames =
--- a/packages/twenty-server/src/modules/messaging/common/services/messaging-save-messages-and-enqueue-contact-creation.service.ts
+++ b/packages/twenty-server/src/modules/messaging/common/services/messaging-save-messages-and-enqueue-contact-creation.service.ts
@ -1,31 +1,33 @@
 import { Injectable } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
 import { EventEmitter2 } from '@nestjs/event-emitter';
+import { InjectRepository } from '@nestjs/typeorm';

 import { EntityManager, Repository } from 'typeorm';

+import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
+import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
 import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
-import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
 import {
-  CreateCompanyAndContactJobData,
  CreateCompanyAndContactJob,
+  CreateCompanyAndContactJobData,
 } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
+import { MessagingMessageService } from 'src/modules/messaging/common/services/messaging-message.service';
 import {
-  FeatureFlagEntity,
-  FeatureFlagKeys,
-} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
-import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
+  MessageChannelContactAutoCreationPolicy,
+  MessageChannelWorkspaceEntity,
+} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
+import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
 import {
  GmailMessage,
  Participant,
  ParticipantWithMessageId,
 } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
-import { MessagingMessageService } from 'src/modules/messaging/common/services/messaging-message.service';
-import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
-import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
-import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
+import { isGroupEmail } from 'src/utils/is-group-email';
+import { isWorkEmail } from 'src/utils/is-work-email';

@Injectable()
 export class MessagingSaveMessagesAndEnqueueContactCreationService {
@ -51,18 +53,8 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
        workspaceId,
      );

-    const isContactCreationForSentAndReceivedEmailsEnabledFeatureFlag =
-      await this.featureFlagRepository.findOneBy({
-        workspaceId: workspaceId,
-        key: FeatureFlagKeys.IsContactCreationForSentAndReceivedEmailsEnabled,
-        value: true,
-      });
-
    const emailAliases = connectedAccount.emailAliases?.split(',') || [];

-    const isContactCreationForSentAndReceivedEmailsEnabled =
-      isContactCreationForSentAndReceivedEmailsEnabledFeatureFlag?.value;
-
    let savedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];

    const participantsWithMessageId = await workspaceDataSource?.transaction(
@ -87,14 +79,35 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
                  message.participants.find((p) => p.role === 'from')?.handle ||
                  '';

+                const isMessageSentByConnectedAccount =
+                  emailAliases.includes(fromHandle);
+
+                const isParticipantConnectedAccount = emailAliases.includes(
+                  participant.handle,
+                );
+
+                const isExcludedByNonProfessionalEmails =
+                  messageChannel.excludeNonProfessionalEmails &&
+                  !isWorkEmail(participant.handle);
+
+                const isExcludedByGroupEmails =
+                  messageChannel.excludeGroupEmails &&
+                  isGroupEmail(participant.handle);
+
+                const shouldCreateContact =
+                  !isParticipantConnectedAccount &&
+                  !isExcludedByNonProfessionalEmails &&
+                  !isExcludedByGroupEmails &&
+                  (messageChannel.contactAutoCreationPolicy ===
+                    MessageChannelContactAutoCreationPolicy.SENT_AND_RECEIVED ||
+                    (messageChannel.contactAutoCreationPolicy ===
+                      MessageChannelContactAutoCreationPolicy.SENT &&
+                      isMessageSentByConnectedAccount));
+
                return {
                  ...participant,
                  messageId,
-                  shouldCreateContact:
-                    messageChannel.isContactAutoCreationEnabled &&
-                    (isContactCreationForSentAndReceivedEmailsEnabled ||
-                      emailAliases.includes(fromHandle)) &&
-                    !emailAliases.includes(participant.handle),
+                  shouldCreateContact,
                };
              })
            : [];
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts
@ -1,29 +1,29 @@
 import { Injectable, Logger } from '@nestjs/common';

-import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { FeatureFlagKeys } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
+import { IsFeatureEnabledService } from 'src/engine/core-modules/feature-flag/services/is-feature-enabled.service';
+import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
 import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
 import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
-import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
+import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
+import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
+import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
+import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
 import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
 import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
-import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
-import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
-import { MessagingTelemetryService } from 'src/modules/messaging/common/services/messaging-telemetry.service';
-import {
-  MessageChannelWorkspaceEntity,
-  MessageChannelSyncStage,
-} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
-import { filterEmails } from 'src/modules/messaging/message-import-manager/utils/filter-emails.util';
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
 import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
-import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';
-import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-by-batches.service';
 import { MessagingErrorHandlingService } from 'src/modules/messaging/common/services/messaging-error-handling.service';
 import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/common/services/messaging-save-messages-and-enqueue-contact-creation.service';
-import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
-import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
-import { IsFeatureEnabledService } from 'src/engine/core-modules/feature-flag/services/is-feature-enabled.service';
-import { FeatureFlagKeys } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
-import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
+import { MessagingTelemetryService } from 'src/modules/messaging/common/services/messaging-telemetry.service';
+import {
+  MessageChannelSyncStage,
+  MessageChannelWorkspaceEntity,
+} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
+import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-get-batch-size.constant';
+import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-by-batches.service';
+import { filterEmails } from 'src/modules/messaging/message-import-manager/utils/filter-emails.util';

@Injectable()
 export class MessagingGmailMessagesImportService {
--- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts
+++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts
@ -9,7 +9,7 @@ export const filterEmails = (
 ) => {
  return filterOutBlocklistedMessages(
    messageChannelHandle,
-    filterOutIcsAttachments(filterOutNonPersonalEmails(messages)),
+    filterOutIcsAttachments(messages),
    blocklist,
  );
 };
@ -46,22 +46,3 @@ const filterOutIcsAttachments = (messages: GmailMessage[]) => {
    );
  });
 };
-
-const isPersonEmail = (email: string): boolean => {
-  const nonPersonalPattern =
-    /noreply|no-reply|do_not_reply|no\.reply|^(info@|contact@|hello@|support@|feedback@|service@|help@|invites@|invite@|welcome@|alerts@|team@|notifications@|notification@|news@)/;
-
-  return !nonPersonalPattern.test(email);
-};
-
-const filterOutNonPersonalEmails = (messages: GmailMessage[]) => {
-  return messages.filter((message) => {
-    if (!message.participants) {
-      return true;
-    }
-
-    return message.participants.every((participant) =>
-      isPersonEmail(participant.handle),
-    );
-  });
-};
--- a/packages/twenty-server/src/modules/messaging/message-participants-manager/jobs/messaging-create-company-and-contact-after-sync.job.ts
+++ b/packages/twenty-server/src/modules/messaging/message-participants-manager/jobs/messaging-create-company-and-contact-after-sync.job.ts
@ -3,21 +3,21 @@ import { InjectRepository } from '@nestjs/typeorm';

 import { Repository } from 'typeorm';

-import {
-  FeatureFlagEntity,
-  FeatureFlagKeys,
-} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
-import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
-import { CreateCompanyAndContactService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service';
-import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
-import { MessageParticipantRepository } from 'src/modules/messaging/common/repositories/message-participant.repository';
-import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
-import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
+import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
 import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
 import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
+import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
+import { CreateCompanyAndContactService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service';
 import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
+import { MessageParticipantRepository } from 'src/modules/messaging/common/repositories/message-participant.repository';
+import {
+  MessageChannelContactAutoCreationPolicy,
+  MessageChannelWorkspaceEntity,
+} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
+import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';

 export type MessagingCreateCompanyAndContactAfterSyncJobData = {
  workspaceId: string;
@ -55,10 +55,11 @@ export class MessagingCreateCompanyAndContactAfterSyncJob {
      workspaceId,
    );

-    const { isContactAutoCreationEnabled, connectedAccountId } =
-      messageChannel[0];
+    const { contactAutoCreationPolicy, connectedAccountId } = messageChannel[0];

-    if (!isContactAutoCreationEnabled) {
+    if (
+      contactAutoCreationPolicy === MessageChannelContactAutoCreationPolicy.NONE
+    ) {
      return;
    }

@ -73,25 +74,17 @@ export class MessagingCreateCompanyAndContactAfterSyncJob {
      );
    }

-    const isContactCreationForSentAndReceivedEmailsEnabledFeatureFlag =
-      await this.featureFlagRepository.findOneBy({
-        workspaceId: workspaceId,
-        key: FeatureFlagKeys.IsContactCreationForSentAndReceivedEmailsEnabled,
-        value: true,
-      });
-
-    const isContactCreationForSentAndReceivedEmailsEnabled =
-      isContactCreationForSentAndReceivedEmailsEnabledFeatureFlag?.value;
-
-    const contactsToCreate = isContactCreationForSentAndReceivedEmailsEnabled
-      ? await this.messageParticipantRepository.getByMessageChannelIdWithoutPersonIdAndWorkspaceMemberId(
-          messageChannelId,
-          workspaceId,
-        )
-      : await this.messageParticipantRepository.getByMessageChannelIdWithoutPersonIdAndWorkspaceMemberIdAndMessageOutgoing(
-          messageChannelId,
-          workspaceId,
-        );
+    const contactsToCreate =
+      contactAutoCreationPolicy ===
+      MessageChannelContactAutoCreationPolicy.SENT_AND_RECEIVED
+        ? await this.messageParticipantRepository.getByMessageChannelIdWithoutPersonIdAndWorkspaceMemberId(
+            messageChannelId,
+            workspaceId,
+          )
+        : await this.messageParticipantRepository.getByMessageChannelIdWithoutPersonIdAndWorkspaceMemberIdAndMessageOutgoing(
+            messageChannelId,
+            workspaceId,
+          );

    await this.createCompanyAndContactService.createCompaniesAndContactsAndUpdateParticipants(
      connectedAccount,