admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

5748 Create contacts for emails sent and received by email aliases (#5855)

Browse Source 
Closes #5748
- Create feature flag
- Add scope `https://www.googleapis.com/auth/profile.emails.read` when
connecting an account
- Get email aliases with google people API, store them in
connectedAccount and refresh them before each message-import
- Update the contact creation logic accordingly
- Refactor

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
bosiraphael
2024-07-01 14:21:34 +02:00
committed by GitHub
parent a15884ea0a
commit 8c33d91734
52 changed files with 1143 additions and 754 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
packages/twenty-server/src/modules/calendar/jobs/google-calendar-sync.job.ts
View File

@ -2,6 +2,9 @@ import { Logger, Scope } from '@nestjs/common';
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
import { GoogleCalendarSyncService } from 'src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
@ -21,6 +24,8 @@ export class GoogleCalendarSyncJob {
constructor(
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
private readonly googleCalendarSyncService: GoogleCalendarSyncService,
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
private readonly connectedAccountRepository: ConnectedAccountRepository,
) {}
@Process(GoogleCalendarSyncJob.name)
@ -29,9 +34,22 @@ export class GoogleCalendarSyncJob {
`google calendar sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
);
try {
const { connectedAccountId, workspaceId } = data;
const connectedAccount = await this.connectedAccountRepository.getById(
connectedAccountId,
workspaceId,
);
if (!connectedAccount) {
throw new Error(
`No connected account found for ${connectedAccountId} in workspace ${workspaceId}`,
);
}
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
data.workspaceId,
data.connectedAccountId,
connectedAccount,
workspaceId,
);
} catch (e) {
this.logger.error(

18
packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts
View File

@ -108,9 +108,8 @@ export class GoogleCalendarSyncService {
const calendarChannelId = calendarChannel.id;
const { events, nextSyncToken } = await this.getEventsFromGoogleCalendar(
refreshToken,
connectedAccount,
workspaceId,
connectedAccountId,
emailOrDomainToReimport,
syncToken,
);
@ -321,9 +320,8 @@ export class GoogleCalendarSyncService {
}
public async getEventsFromGoogleCalendar(
refreshToken: string,
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
connectedAccountId: string,
emailOrDomainToReimport?: string,
syncToken?: string,
): Promise<{
@ -332,7 +330,7 @@ export class GoogleCalendarSyncService {
}> {
const googleCalendarClient =
await this.googleCalendarClientProvider.getGoogleCalendarClient(
refreshToken,
connectedAccount,
);
const startTime = Date.now();
@ -360,7 +358,7 @@ export class GoogleCalendarSyncService {
await this.calendarChannelRepository.update(
{
id: connectedAccountId,
id: connectedAccount.id,
},
{
syncCursor: '',
@ -368,7 +366,7 @@ export class GoogleCalendarSyncService {
);
this.logger.log(
`Sync token is no longer valid for connected account ${connectedAccountId} in workspace ${workspaceId}, resetting sync cursor.`,
`Sync token is no longer valid for connected account ${connectedAccount.id} in workspace ${workspaceId}, resetting sync cursor.`,
);
return {
@ -399,9 +397,9 @@ export class GoogleCalendarSyncService {
const endTime = Date.now();
this.logger.log(
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} getting events list in ${
endTime - startTime
}ms.`,
`google calendar sync for workspace ${workspaceId} and account ${
connectedAccount.id
} getting events list in ${endTime - startTime}ms.`,
);
return { events, nextSyncToken };

3
packages/twenty-server/src/modules/calendar/services/providers/calendar-providers.module.ts
View File

@ -2,9 +2,10 @@ import { Module } from '@nestjs/common';
import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
import { GoogleCalendarClientProvider } from 'src/modules/calendar/services/providers/google-calendar/google-calendar.provider';
import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
@Module({
imports: [EnvironmentModule],
imports: [EnvironmentModule, OAuth2ClientManagerModule],
providers: [GoogleCalendarClientProvider],
exports: [GoogleCalendarClientProvider],
})

33
packages/twenty-server/src/modules/calendar/services/providers/google-calendar/google-calendar.provider.ts
View File

@ -1,18 +1,21 @@
import { Injectable } from '@nestjs/common';
import { OAuth2Client } from 'google-auth-library';
import { calendar_v3 as calendarV3, google } from 'googleapis';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class GoogleCalendarClientProvider {
constructor(private readonly environmentService: EnvironmentService) {}
constructor(
private readonly oAuth2ClientManagerService: OAuth2ClientManagerService,
) {}
public async getGoogleCalendarClient(
refreshToken: string,
connectedAccount: ConnectedAccountWorkspaceEntity,
): Promise<calendarV3.Calendar> {
const oAuth2Client = await this.getOAuth2Client(refreshToken);
const oAuth2Client =
await this.oAuth2ClientManagerService.getOAuth2Client(connectedAccount);
const googleCalendarClient = google.calendar({
version: 'v3',
@ -21,24 +24,4 @@ export class GoogleCalendarClientProvider {
return googleCalendarClient;
}
private async getOAuth2Client(refreshToken: string): Promise<OAuth2Client> {
const googleCalendarClientId = this.environmentService.get(
'AUTH_GOOGLE_CLIENT_ID',
);
const googleCalendarClientSecret = this.environmentService.get(
'AUTH_GOOGLE_CLIENT_SECRET',
);
const oAuth2Client = new google.auth.OAuth2(
googleCalendarClientId,
googleCalendarClientSecret,
);
oAuth2Client.setCredentials({
refresh_token: refreshToken,
});
return oAuth2Client;
}
}

10
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts
View File

@ -16,7 +16,7 @@ import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/perso
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
import { CalendarEventParticipantService } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.service';
import { filterOutContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util';
import { filterOutSelfAndContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util';
import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
import { MessageParticipantWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-participant.workspace-entity';
import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/standard-objects/calendar-event-participant.workspace-entity';
@ -43,7 +43,7 @@ export class CreateCompanyAndContactService {
) {}
async createCompaniesAndPeople(
connectedAccountHandle: string,
connectedAccount: ConnectedAccountWorkspaceEntity,
contactsToCreate: Contact[],
workspaceId: string,
transactionManager?: EntityManager,
@ -62,9 +62,9 @@ export class CreateCompanyAndContactService {
);
const contactsToCreateFromOtherCompanies =
filterOutContactsFromCompanyOrWorkspace(
filterOutSelfAndContactsFromCompanyOrWorkspace(
contactsToCreate,
connectedAccountHandle,
connectedAccount,
workspaceMembers,
);
@ -150,7 +150,7 @@ export class CreateCompanyAndContactService {
await this.workspaceDataSource?.transaction(
async (transactionManager: EntityManager) => {
const createdPeople = await this.createCompaniesAndPeople(
connectedAccount.handle,
connectedAccount,
contactsBatch,
workspaceId,
transactionManager,

12
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts
View File

@ -1,13 +1,16 @@
import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
export function filterOutContactsFromCompanyOrWorkspace(
export function filterOutSelfAndContactsFromCompanyOrWorkspace(
contacts: Contact[],
selfHandle: string,
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceMembers: WorkspaceMemberWorkspaceEntity[],
): Contact[] {
const selfDomainName = getDomainNameFromHandle(selfHandle);
const selfDomainName = getDomainNameFromHandle(connectedAccount.handle);
const emailAliases = connectedAccount.emailAliases?.split(',') || [];
const workspaceMembersMap = workspaceMembers.reduce(
(map, workspaceMember) => {
@ -21,6 +24,7 @@ export function filterOutContactsFromCompanyOrWorkspace(
return contacts.filter(
(contact) =>
getDomainNameFromHandle(contact.handle) !== selfDomainName &&
!workspaceMembersMap[contact.handle],
!workspaceMembersMap[contact.handle] &&
!emailAliases.includes(contact.handle),
);
}

43
packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/google/google-email-alias-manager.service.ts Normal file
View File

@ -0,0 +1,43 @@
import { Injectable } from '@nestjs/common';
import { google } from 'googleapis';
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class GoogleEmailAliasManagerService {
constructor(
private readonly oAuth2ClientManagerService: OAuth2ClientManagerService,
) {}
public async getEmailAliases(
connectedAccount: ConnectedAccountWorkspaceEntity,
) {
const oAuth2Client =
await this.oAuth2ClientManagerService.getOAuth2Client(connectedAccount);
const people = google.people({
version: 'v1',
auth: oAuth2Client,
});
const emailsResponse = await people.people.get({
resourceName: 'people/me',
personFields: 'emailAddresses',
});
const emailAddresses = emailsResponse.data.emailAddresses;
const emailAliases =
emailAddresses
?.filter((emailAddress) => {
return emailAddress.metadata?.primary !== true;
})
.map((emailAddress) => {
return emailAddress.value || '';
}) || [];
return emailAliases;
}
}

19
packages/twenty-server/src/modules/connected-account/email-alias-manager/email-alias-manager.module.ts Normal file
View File

@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { GoogleEmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/drivers/google/google-email-alias-manager.service';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Module({
imports: [
ObjectMetadataRepositoryModule.forFeature([
ConnectedAccountWorkspaceEntity,
]),
OAuth2ClientManagerModule,
],
providers: [EmailAliasManagerService, GoogleEmailAliasManagerService],
exports: [EmailAliasManagerService],
})
export class EmailAliasManagerModule {}

41
packages/twenty-server/src/modules/connected-account/email-alias-manager/services/email-alias-manager.service.ts Normal file
View File

@ -0,0 +1,41 @@
import { Injectable } from '@nestjs/common';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { GoogleEmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/drivers/google/google-email-alias-manager.service';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class EmailAliasManagerService {
constructor(
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
private readonly connectedAccountRepository: ConnectedAccountRepository,
private readonly googleEmailAliasManagerService: GoogleEmailAliasManagerService,
) {}
public async refreshEmailAliases(
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
) {
let emailAliases: string[];
switch (connectedAccount.provider) {
case 'google':
emailAliases =
await this.googleEmailAliasManagerService.getEmailAliases(
connectedAccount,
);
break;
default:
throw new Error(
`Email alias manager for provider ${connectedAccount.provider} is not implemented`,
);
}
await this.connectedAccountRepository.updateEmailAliases(
emailAliases,
connectedAccount.id,
workspaceId,
);
}
}

29
packages/twenty-server/src/modules/connected-account/oauth2-client-manager/drivers/google/google-oauth2-client-manager.service.ts Normal file
View File

@ -0,0 +1,29 @@
import { Injectable } from '@nestjs/common';
import { OAuth2Client } from 'google-auth-library';
import { google } from 'googleapis';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
@Injectable()
export class GoogleOAuth2ClientManagerService {
constructor(private readonly environmentService: EnvironmentService) {}
public async getOAuth2Client(refreshToken: string): Promise<OAuth2Client> {
const gmailClientId = this.environmentService.get('AUTH_GOOGLE_CLIENT_ID');
const gmailClientSecret = this.environmentService.get(
'AUTH_GOOGLE_CLIENT_SECRET',
);
const oAuth2Client = new google.auth.OAuth2(
gmailClientId,
gmailClientSecret,
);
oAuth2Client.setCredentials({
refresh_token: refreshToken,
});
return oAuth2Client;
}
}

11
packages/twenty-server/src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { GoogleOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/google/google-oauth2-client-manager.service';
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
@Module({
imports: [],
providers: [OAuth2ClientManagerService, GoogleOAuth2ClientManagerService],
exports: [OAuth2ClientManagerService],
})
export class OAuth2ClientManagerModule {}

30
packages/twenty-server/src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service.ts Normal file
View File

@ -0,0 +1,30 @@
import { Injectable } from '@nestjs/common';
import { OAuth2Client } from 'google-auth-library';
import { GoogleOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/google/google-oauth2-client-manager.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class OAuth2ClientManagerService {
constructor(
private readonly googleOAuth2ClientManagerService: GoogleOAuth2ClientManagerService,
) {}
public async getOAuth2Client(
connectedAccount: ConnectedAccountWorkspaceEntity,
): Promise<OAuth2Client> {
const { refreshToken } = connectedAccount;
switch (connectedAccount.provider) {
case 'google':
return this.googleOAuth2ClientManagerService.getOAuth2Client(
refreshToken,
);
default:
throw new Error(
`OAuth2 client manager for provider ${connectedAccount.provider} is not implemented`,
);
}
}
}

18
packages/twenty-server/src/modules/connected-account/repositories/connected-account.repository.ts
View File

@ -306,4 +306,22 @@ export class ConnectedAccountRepository {
return connectedAccount;
}
public async updateEmailAliases(
emailAliases: string[],
connectedAccountId: string,
workspaceId: string,
transactionManager?: EntityManager,
) {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`UPDATE ${dataSourceSchema}."connectedAccount" SET "emailAliases" = $1 WHERE "id" = $2`,
// TODO: modify emailAliases to be of fieldmetadatatype array
[emailAliases.join(','), connectedAccountId],
workspaceId,
transactionManager,
);
}
}

24
packages/twenty-server/src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service.ts
View File

@ -16,33 +16,27 @@ export class GoogleAPIRefreshAccessTokenService {
) {}
async refreshAndSaveAccessToken(
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceId: string,
connectedAccountId: string,
): Promise<string> {
const connectedAccount = await this.connectedAccountRepository.getById(
connectedAccountId,
workspaceId,
);
if (!connectedAccount) {
throw new Error(
`No connected account found for ${connectedAccountId} in workspace ${workspaceId}`,
);
}
const refreshToken = connectedAccount.refreshToken;
if (!refreshToken) {
throw new Error(
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
`No refresh token found for connected account ${connectedAccount.id} in workspace ${workspaceId}`,
);
}
const accessToken = await this.refreshAccessToken(refreshToken);
await this.connectedAccountRepository.updateAccessToken(
accessToken,
connectedAccountId,
connectedAccount.id,
workspaceId,
);
await this.connectedAccountRepository.updateAccessToken(
accessToken,
connectedAccount.id,
workspaceId,
);

14
packages/twenty-server/src/modules/connected-account/standard-objects/connected-account.workspace-entity.ts
View File

@ -17,6 +17,8 @@ import { WorkspaceField } from 'src/engine/twenty-orm/decorators/workspace-field
import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator';
import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace-is-nullable.decorator';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { FeatureFlagKeys } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { WorkspaceGate } from 'src/engine/twenty-orm/decorators/workspace-gate.decorator';
import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace-join-column.decorator';
export enum ConnectedAccountProvider {
@ -89,6 +91,18 @@ export class ConnectedAccountWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
authFailedAt: Date | null;
@WorkspaceField({
standardId: CONNECTED_ACCOUNT_STANDARD_FIELD_IDS.emailAliases,
type: FieldMetadataType.TEXT,
label: 'Email Aliases',
description: 'Email Aliases',
icon: 'IconMail',
})
@WorkspaceGate({
featureFlag: FeatureFlagKeys.IsMessagingAliasFetchingEnabled,
})
emailAliases: string;
@WorkspaceRelation({
standardId: CONNECTED_ACCOUNT_STANDARD_FIELD_IDS.accountOwner,
type: RelationMetadataType.MANY_TO_ONE,

115
packages/twenty-server/src/modules/messaging/common/services/messaging-message.service.ts
View File

@ -1,17 +1,15 @@
import { Injectable, Logger } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { DataSource, EntityManager } from 'typeorm';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/common/repositories/message-channel-message-association.repository';
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
import { MessageThreadRepository } from 'src/modules/messaging/common/repositories/message-thread.repository';
import { MessageRepository } from 'src/modules/messaging/common/repositories/message.repository';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
import { MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message';
@ -19,8 +17,6 @@ import { MessagingMessageThreadService } from 'src/modules/messaging/common/serv
@Injectable()
export class MessagingMessageService {
private readonly logger = new Logger(MessagingMessageService.name);
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
@InjectObjectMetadataRepository(
@ -29,8 +25,6 @@ export class MessagingMessageService {
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
@InjectObjectMetadataRepository(MessageWorkspaceEntity)
private readonly messageRepository: MessageRepository,
@InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
private readonly messageChannelRepository: MessageChannelRepository,
@InjectObjectMetadataRepository(MessageThreadWorkspaceEntity)
private readonly messageThreadRepository: MessageThreadRepository,
private readonly messageThreadService: MessagingMessageThreadService,
@ -101,104 +95,6 @@ export class MessagingMessageService {
return messageExternalIdsAndIdsMap;
}
public async saveMessages(
messages: GmailMessage[],
workspaceDataSource: DataSource,
connectedAccount: ConnectedAccountWorkspaceEntity,
gmailMessageChannelId: string,
workspaceId: string,
): Promise<Map<string, string>> {
const messageExternalIdsAndIdsMap = new Map<string, string>();
try {
let keepImporting = true;
for (const message of messages) {
if (!keepImporting) {
break;
}
await workspaceDataSource?.transaction(
async (manager: EntityManager) => {
const gmailMessageChannel =
await this.messageChannelRepository.getByIds(
[gmailMessageChannelId],
workspaceId,
manager,
);
if (gmailMessageChannel.length === 0) {
this.logger.error(
`No message channel found for connected account ${connectedAccount.id} in workspace ${workspaceId} in saveMessages`,
);
keepImporting = false;
return;
}
const existingMessageChannelMessageAssociationsCount =
await this.messageChannelMessageAssociationRepository.countByMessageExternalIdsAndMessageChannelId(
[message.externalId],
gmailMessageChannelId,
workspaceId,
manager,
);
if (existingMessageChannelMessageAssociationsCount > 0) {
return;
}
// TODO: This does not handle all thread merging use cases and might create orphan threads.
const savedOrExistingMessageThreadId =
await this.messageThreadService.saveMessageThreadOrReturnExistingMessageThread(
message.headerMessageId,
message.messageThreadExternalId,
workspaceId,
manager,
);
if (!savedOrExistingMessageThreadId) {
throw new Error(
`No message thread found for message ${message.headerMessageId} in workspace ${workspaceId} in saveMessages`,
);
}
const savedOrExistingMessageId =
await this.saveMessageOrReturnExistingMessage(
message,
savedOrExistingMessageThreadId,
connectedAccount,
workspaceId,
manager,
);
messageExternalIdsAndIdsMap.set(
message.externalId,
savedOrExistingMessageId,
);
await this.messageChannelMessageAssociationRepository.insert(
gmailMessageChannelId,
savedOrExistingMessageId,
message.externalId,
savedOrExistingMessageThreadId,
message.messageThreadExternalId,
workspaceId,
manager,
);
},
);
}
} catch (error) {
throw new Error(
`Error saving connected account ${connectedAccount.id} messages to workspace ${workspaceId}: ${error.message}`,
);
}
return messageExternalIdsAndIdsMap;
}
private async saveMessageOrReturnExistingMessage(
message: GmailMessage,
messageThreadId: string,
@ -219,8 +115,11 @@ export class MessagingMessageService {
const newMessageId = v4();
const messageDirection =
connectedAccount.handle === message.fromHandle ? 'outgoing' : 'incoming';
const messageDirection = connectedAccount.emailAliases?.includes(
message.fromHandle,
)
? 'outgoing'
: 'incoming';
const receivedAt = new Date(parseInt(message.internalDate));

26
packages/twenty-server/src/modules/messaging/common/services/messaging-save-messages-and-enqueue-contact-creation.service.ts
View File

@ -58,6 +58,8 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
value: true,
});
const emailAliases = connectedAccount.emailAliases?.split(',') || [];
const isContactCreationForSentAndReceivedEmailsEnabled =
isContactCreationForSentAndReceivedEmailsEnabledFeatureFlag?.value;
@ -80,15 +82,21 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
const messageId = messageExternalIdsAndIdsMap.get(message.externalId);
return messageId
? message.participants.map((participant: Participant) => ({
...participant,
messageId,
shouldCreateContact:
messageChannel.isContactAutoCreationEnabled &&
(isContactCreationForSentAndReceivedEmailsEnabled ||
message.participants.find((p) => p.role === 'from')
?.handle === connectedAccount.handle),
}))
? message.participants.map((participant: Participant) => {
const fromHandle =
message.participants.find((p) => p.role === 'from')?.handle ||
'';
return {
...participant,
messageId,
shouldCreateContact:
messageChannel.isContactAutoCreationEnabled &&
(isContactCreationForSentAndReceivedEmailsEnabled ||
emailAliases.includes(fromHandle)) &&
!emailAliases.includes(participant.handle),
};
})
: [];
});

6
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module.ts
View File

@ -2,8 +2,11 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { EmailAliasManagerModule } from 'src/modules/connected-account/email-alias-manager/email-alias-manager.module';
import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
import { GoogleAPIRefreshAccessTokenModule } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.module';
import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@ -30,6 +33,9 @@ import { MessagingGmailPartialMessageListFetchService } from 'src/modules/messag
]),
MessagingCommonModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
OAuth2ClientManagerModule,
EmailAliasManagerModule,
FeatureFlagModule,
],
providers: [
MessagingGmailClientProvider,

33
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/providers/messaging-gmail-client.provider.ts
View File

@ -1,16 +1,21 @@
import { Injectable } from '@nestjs/common';
import { OAuth2Client } from 'google-auth-library';
import { gmail_v1, google } from 'googleapis';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { OAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/services/oauth2-client-manager.service';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@Injectable()
export class MessagingGmailClientProvider {
constructor(private readonly environmentService: EnvironmentService) {}
constructor(
private readonly oAuth2ClientManagerService: OAuth2ClientManagerService,
) {}
public async getGmailClient(refreshToken: string): Promise<gmail_v1.Gmail> {
const oAuth2Client = await this.getOAuth2Client(refreshToken);
public async getGmailClient(
connectedAccount: ConnectedAccountWorkspaceEntity,
): Promise<gmail_v1.Gmail> {
const oAuth2Client =
await this.oAuth2ClientManagerService.getOAuth2Client(connectedAccount);
const gmailClient = google.gmail({
version: 'v1',
@ -19,22 +24,4 @@ export class MessagingGmailClientProvider {
return gmailClient;
}
private async getOAuth2Client(refreshToken: string): Promise<OAuth2Client> {
const gmailClientId = this.environmentService.get('AUTH_GOOGLE_CLIENT_ID');
const gmailClientSecret = this.environmentService.get(
'AUTH_GOOGLE_CLIENT_SECRET',
);
const oAuth2Client = new google.auth.OAuth2(
gmailClientId,
gmailClientSecret,
);
oAuth2Client.setCredentials({
refresh_token: refreshToken,
});
return oAuth2Client;
}
}

4
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-full-message-list-fetch.service.ts
View File

@ -54,9 +54,7 @@ export class MessagingGmailFullMessageListFetchService {
);
const gmailClient: gmail_v1.Gmail =
await this.gmailClientProvider.getGmailClient(
connectedAccount.refreshToken,
);
await this.gmailClientProvider.getGmailClient(connectedAccount);
const { error: gmailError } =
await this.fetchAllMessageIdsFromGmailAndStoreInCache(

31
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts
View File

@ -20,6 +20,9 @@ import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messagi
import { MessagingErrorHandlingService } from 'src/modules/messaging/common/services/messaging-error-handling.service';
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/common/services/messaging-save-messages-and-enqueue-contact-creation.service';
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
import { IsFeatureEnabledService } from 'src/engine/core-modules/feature-flag/services/is-feature-enabled.service';
import { FeatureFlagKeys } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
@Injectable()
@ -41,6 +44,8 @@ export class MessagingGmailMessagesImportService {
private readonly blocklistRepository: BlocklistRepository,
@InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
private readonly messageChannelRepository: MessageChannelRepository,
private readonly emailAliasManagerService: EmailAliasManagerService,
private readonly isFeatureEnabledService: IsFeatureEnabledService,
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
private readonly connectedAccountRepository: ConnectedAccountRepository,
) {}
@ -78,8 +83,8 @@ export class MessagingGmailMessagesImportService {
try {
accessToken =
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
connectedAccount,
workspaceId,
connectedAccount.id,
);
} catch (error) {
await this.messagingTelemetryService.track({
@ -103,6 +108,30 @@ export class MessagingGmailMessagesImportService {
return;
}
if (
await this.isFeatureEnabledService.isFeatureEnabled(
FeatureFlagKeys.IsMessagingAliasFetchingEnabled,
workspaceId,
)
) {
try {
await this.emailAliasManagerService.refreshEmailAliases(
connectedAccount,
workspaceId,
);
} catch (error) {
await this.gmailErrorHandlingService.handleGmailError(
{
code: error.code,
reason: error.message,
},
'messages-import',
messageChannel,
workspaceId,
);
}
}
const messageIdsToFetch =
(await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:gmail:${messageChannel.id}`,

4
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-partial-message-list-fetch.service.ts
View File

@ -52,9 +52,7 @@ export class MessagingGmailPartialMessageListFetchService {
const lastSyncHistoryId = messageChannel.syncCursor;
const gmailClient: gmail_v1.Gmail =
await this.gmailClientProvider.getGmailClient(
connectedAccount.refreshToken,
);
await this.gmailClientProvider.getGmailClient(connectedAccount);
const { history, historyId, error } =
await this.gmailGetHistoryService.getHistory(