[messaing] improve messaging import (#4650)
* [messaging] improve full-sync fetching strategy * fix * rebase * fix * fix * fix rebase * fix * fix * fix * fix * fix * remove deletion * fix setPop with memory storage * fix pgBoss and remove unnecessary job * fix throw * fix * add timeout to ongoing sync
This commit is contained in:
Weiko
@ -19,7 +19,6 @@ export class FetchMessagesByBatchesService {
|
||||
async fetchAllMessages(
|
||||
queries: MessageQuery[],
|
||||
accessToken: string,
|
||||
jobName?: string,
|
||||
workspaceId?: string,
|
||||
connectedAccountId?: string,
|
||||
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
|
||||
@ -32,7 +31,7 @@ export class FetchMessagesByBatchesService {
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
|
||||
queries.length
|
||||
} messages in ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -45,7 +44,7 @@ export class FetchMessagesByBatchesService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
|
||||
queries.length
|
||||
} messages in ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -62,6 +61,10 @@ export class FetchMessagesByBatchesService {
|
||||
|
||||
const errors: any = [];
|
||||
|
||||
const sanitizeString = (str: string) => {
|
||||
return str.replace(/\0/g, '');
|
||||
};
|
||||
|
||||
const formattedResponse = Promise.all(
|
||||
parsedResponses.map(async (message: GmailMessageParsedResponse) => {
|
||||
if (message.error) {
|
||||
@ -119,7 +122,7 @@ export class FetchMessagesByBatchesService {
|
||||
fromHandle: from.value[0].address || '',
|
||||
fromDisplayName: from.value[0].name || '',
|
||||
participants,
|
||||
text: textWithoutReplyQuotations || '',
|
||||
text: sanitizeString(textWithoutReplyQuotations || ''),
|
||||
attachments,
|
||||
};
|
||||
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailFetchMessageContentFromCacheService } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service';
|
||||
import { MessageModule } from 'src/modules/messaging/services/message/message.module';
|
||||
import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
FetchMessagesByBatchesModule,
|
||||
ObjectMetadataRepositoryModule.forFeature([
|
||||
ConnectedAccountObjectMetadata,
|
||||
MessageChannelObjectMetadata,
|
||||
]),
|
||||
SaveMessageAndEmitContactCreationEventModule,
|
||||
MessageModule,
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailFetchMessageContentFromCacheService],
|
||||
exports: [GmailFetchMessageContentFromCacheService],
|
||||
})
|
||||
export class GmailFetchMessageContentFromCacheModule {}
|
||||
@ -0,0 +1,257 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { FetchMessagesByBatchesService } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service';
|
||||
import {
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelSyncStatus,
|
||||
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { createQueriesFromMessageIds } from 'src/modules/messaging/utils/create-queries-from-message-ids.util';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { SaveMessageAndEmitContactCreationEventService } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.service';
|
||||
import {
|
||||
GmailFullSyncV2JobData,
|
||||
GmailFullSyncV2Job,
|
||||
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { GMAIL_ONGOING_SYNC_TIMEOUT } from 'src/modules/messaging/constants/gmail-ongoing-sync-timeout.constant';
|
||||
|
||||
@Injectable()
|
||||
export class GmailFetchMessageContentFromCacheService {
|
||||
private readonly logger = new Logger(
|
||||
GmailFetchMessageContentFromCacheService.name,
|
||||
);
|
||||
|
||||
constructor(
|
||||
private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
private readonly saveMessageAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
@Inject(MessageQueue.messagingQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
) {}
|
||||
|
||||
async fetchMessageContentFromCache(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const accessToken = connectedAccount.accessToken;
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.PENDING) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is not pending.`,
|
||||
);
|
||||
|
||||
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.ONGOING) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ongoingSyncStartedAt = new Date(
|
||||
gmailMessageChannel.ongoingSyncStartedAt,
|
||||
);
|
||||
|
||||
if (
|
||||
ongoingSyncStartedAt < new Date(Date.now() - GMAIL_ONGOING_SYNC_TIMEOUT)
|
||||
) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} failed due to ongoing sync timeout. Restarting full-sync...`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const gmailMessageChannelId = gmailMessageChannel.id;
|
||||
|
||||
const messageIdsToFetch =
|
||||
(await this.cacheStorage.setPop(
|
||||
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannelId}`,
|
||||
GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
|
||||
)) ?? [];
|
||||
|
||||
if (!messageIdsToFetch?.length) {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.SUCCEEDED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import or delete.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.ONGOING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} starting...`,
|
||||
);
|
||||
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await workspaceDataSource
|
||||
?.transaction(async (transactionManager: EntityManager) => {
|
||||
const messageQueries = createQueriesFromMessageIds(messageIdsToFetch);
|
||||
|
||||
const { messages: messagesToSave, errors } =
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
|
||||
if (!messagesToSave.length) {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
const errorsCanBeIgnored = errors.every(
|
||||
(error) => error.code === 404,
|
||||
);
|
||||
|
||||
if (!errorsCanBeIgnored) {
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${JSON.stringify(
|
||||
errors,
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.saveMessageAndEmitContactCreationEventService.saveMessagesAndEmitContactCreationEventWithinTransaction(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannel,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (messageIdsToFetch.length < GMAIL_USERS_MESSAGES_GET_BATCH_SIZE) {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.SUCCEEDED,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with no more messages to import.`,
|
||||
);
|
||||
} else {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with more messages to import.`,
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannelId}`,
|
||||
messageIdsToFetch,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async fallbackToFullSync(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
await this.messageQueueService.add<GmailFullSyncV2JobData>(
|
||||
GmailFullSyncV2Job.name,
|
||||
{ workspaceId, connectedAccountId },
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
|
||||
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
MessagingProvidersModule,
|
||||
FetchMessagesByBatchesModule,
|
||||
ObjectMetadataRepositoryModule.forFeature([
|
||||
ConnectedAccountObjectMetadata,
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelMessageAssociationObjectMetadata,
|
||||
BlocklistObjectMetadata,
|
||||
]),
|
||||
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailFullSyncV2Service],
|
||||
exports: [GmailFullSyncV2Service],
|
||||
})
|
||||
export class GmailFullSynV2Module {}
|
||||
@ -0,0 +1,302 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { EntityManager, Repository } from 'typeorm';
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
|
||||
import {
|
||||
FeatureFlagEntity,
|
||||
FeatureFlagKeys,
|
||||
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/constants/gmail-users-messages-list-max-result.constant';
|
||||
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/repositories/message-channel-message-association.repository';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||
import {
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelSyncStatus,
|
||||
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { gmailSearchFilterExcludeEmails } from 'src/modules/messaging/utils/gmail-search-filter.util';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
|
||||
@Injectable()
|
||||
export class GmailFullSyncV2Service {
|
||||
private readonly logger = new Logger(GmailFullSyncV2Service.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
@InjectObjectMetadataRepository(
|
||||
MessageChannelMessageAssociationObjectMetadata,
|
||||
)
|
||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
public async fetchConnectedAccountThreads(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (gmailMessageChannel.syncStatus === MessageChannelSyncStatus.ONGOING) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is locked, import will be retried later.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.ONGOING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await workspaceDataSource
|
||||
?.transaction(async (transactionManager) => {
|
||||
const gmailClient: gmail_v1.Gmail =
|
||||
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||
|
||||
const blocklistedEmails = await this.fetchBlocklistEmails(
|
||||
connectedAccount.accountOwnerId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||
gmailClient,
|
||||
gmailMessageChannel.id,
|
||||
blocklistedEmails,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
public async fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||
gmailClient: gmail_v1.Gmail,
|
||||
messageChannelId: string,
|
||||
blocklistedEmails: string[],
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) {
|
||||
let pageToken: string | undefined;
|
||||
let hasMoreMessages = true;
|
||||
let messageIdsToFetch = 0;
|
||||
let firstMessageExternalId;
|
||||
|
||||
while (hasMoreMessages) {
|
||||
const response = await gmailClient.users.messages.list({
|
||||
userId: 'me',
|
||||
maxResults: GMAIL_USERS_MESSAGES_LIST_MAX_RESULT,
|
||||
pageToken,
|
||||
q: gmailSearchFilterExcludeEmails(blocklistedEmails),
|
||||
});
|
||||
|
||||
if (response.data?.messages) {
|
||||
const messageExternalIds = response.data.messages
|
||||
.filter((message): message is { id: string } => message.id != null)
|
||||
.map((message) => message.id);
|
||||
|
||||
if (!firstMessageExternalId) {
|
||||
firstMessageExternalId = messageExternalIds[0];
|
||||
}
|
||||
|
||||
const existingMessageChannelMessageAssociations =
|
||||
await this.messageChannelMessageAssociationRepository.getByMessageExternalIdsAndMessageChannelId(
|
||||
messageExternalIds,
|
||||
messageChannelId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const existingMessageChannelMessageAssociationsExternalIds =
|
||||
existingMessageChannelMessageAssociations.map(
|
||||
(messageChannelMessageAssociation) =>
|
||||
messageChannelMessageAssociation.messageExternalId,
|
||||
);
|
||||
|
||||
const messageIdsToImport = messageExternalIds.filter(
|
||||
(messageExternalId) =>
|
||||
!existingMessageChannelMessageAssociationsExternalIds.includes(
|
||||
messageExternalId,
|
||||
),
|
||||
);
|
||||
|
||||
if (messageIdsToImport && messageIdsToImport.length) {
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
|
||||
messageIdsToImport,
|
||||
);
|
||||
|
||||
messageIdsToFetch += messageIdsToImport.length;
|
||||
}
|
||||
}
|
||||
|
||||
pageToken = response.data.nextPageToken ?? undefined;
|
||||
hasMoreMessages = !!pageToken;
|
||||
}
|
||||
|
||||
if (!messageIdsToFetch) {
|
||||
this.logger.log(
|
||||
`No messages found in Gmail for messageChannel ${messageChannelId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Fetched all ${messageIdsToFetch} message ids from Gmail for messageChannel ${messageChannelId} in workspace ${workspaceId} and added to cache for import`,
|
||||
);
|
||||
|
||||
await this.updateLastSyncExternalId(
|
||||
gmailClient,
|
||||
messageChannelId,
|
||||
firstMessageExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
public async fetchBlocklistEmails(
|
||||
workspaceMemberId: string,
|
||||
workspaceId: string,
|
||||
) {
|
||||
const isBlocklistEnabledFeatureFlag =
|
||||
await this.featureFlagRepository.findOneBy({
|
||||
workspaceId,
|
||||
key: FeatureFlagKeys.IsBlocklistEnabled,
|
||||
value: true,
|
||||
});
|
||||
|
||||
const isBlocklistEnabled =
|
||||
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
|
||||
|
||||
const blocklist = isBlocklistEnabled
|
||||
? await this.blocklistRepository.getByWorkspaceMemberId(
|
||||
workspaceMemberId,
|
||||
workspaceId,
|
||||
)
|
||||
: [];
|
||||
|
||||
return blocklist.map((blocklist) => blocklist.handle);
|
||||
}
|
||||
|
||||
private async updateLastSyncExternalId(
|
||||
gmailClient: gmail_v1.Gmail,
|
||||
messageChannelId: string,
|
||||
firstMessageExternalId: string,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) {
|
||||
if (!firstMessageExternalId) {
|
||||
throw new Error(
|
||||
`No first message found for workspace ${workspaceId} and account ${messageChannelId}, can't update sync external id`,
|
||||
);
|
||||
}
|
||||
|
||||
const firstMessageContent = await gmailClient.users.messages.get({
|
||||
userId: 'me',
|
||||
id: firstMessageExternalId,
|
||||
});
|
||||
|
||||
if (!firstMessageContent?.data) {
|
||||
throw new Error(
|
||||
`No first message content found for message ${firstMessageExternalId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const historyId = firstMessageContent?.data?.historyId;
|
||||
|
||||
if (!historyId) {
|
||||
throw new Error(
|
||||
`No historyId found for message ${firstMessageExternalId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Updating last external id: ${historyId} for workspace ${workspaceId} and account ${messageChannelId} succeeded.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateLastSyncExternalIdIfHigher(
|
||||
messageChannelId,
|
||||
historyId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -47,7 +47,7 @@ export class GmailFullSyncService {
|
||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
private readonly saveMessagesAndCreateContactsService: SaveMessageAndEmitContactCreationEventService,
|
||||
private readonly saveMessagesAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
@ -186,7 +186,6 @@ export class GmailFullSyncService {
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
'gmail full-sync',
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
@ -200,12 +199,11 @@ export class GmailFullSyncService {
|
||||
);
|
||||
|
||||
if (messagesToSave.length > 0) {
|
||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
||||
await this.saveMessagesAndEmitContactCreationEventService.saveMessagesAndEmitContactCreation(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannelId,
|
||||
'gmail full-sync',
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
|
||||
import { MessageModule } from 'src/modules/messaging/services/message/message.module';
|
||||
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||
import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
MessagingProvidersModule,
|
||||
FetchMessagesByBatchesModule,
|
||||
ObjectMetadataRepositoryModule.forFeature([
|
||||
ConnectedAccountObjectMetadata,
|
||||
MessageChannelObjectMetadata,
|
||||
BlocklistObjectMetadata,
|
||||
]),
|
||||
MessageModule,
|
||||
SaveMessageAndEmitContactCreationEventModule,
|
||||
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailPartialSyncV2Service],
|
||||
exports: [GmailPartialSyncV2Service],
|
||||
})
|
||||
export class GmailPartialSyncV2Module {}
|
||||
@ -0,0 +1,338 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import {
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelSyncStatus,
|
||||
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { GMAIL_USERS_HISTORY_MAX_RESULT } from 'src/modules/messaging/constants/gmail-users-history-max-result.constant';
|
||||
import { GmailError } from 'src/modules/messaging/types/gmail-error';
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import {
|
||||
GmailFullSyncV2Job,
|
||||
GmailFullSyncV2JobData,
|
||||
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||
|
||||
@Injectable()
|
||||
export class GmailPartialSyncV2Service {
|
||||
private readonly logger = new Logger(GmailPartialSyncV2Service.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
@Inject(MessageQueue.messagingQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
public async fetchConnectedAccountThreads(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
): Promise<void> {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.SUCCEEDED) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is locked, import will be retried later.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.ONGOING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await workspaceDataSource
|
||||
?.transaction(async (transactionManager: EntityManager) => {
|
||||
const lastSyncHistoryId = gmailMessageChannel.syncExternalId;
|
||||
|
||||
if (!lastSyncHistoryId) {
|
||||
this.logger.log(
|
||||
`No lastSyncHistoryId for workspace ${workspaceId} and account ${connectedAccountId}, falling back to full sync.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const gmailClient: gmail_v1.Gmail =
|
||||
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||
|
||||
const { history, historyId, error } = await this.getHistoryFromGmail(
|
||||
gmailClient,
|
||||
lastSyncHistoryId,
|
||||
);
|
||||
|
||||
if (error?.code === 404) {
|
||||
this.logger.log(
|
||||
`404: Invalid lastSyncHistoryId: ${lastSyncHistoryId} for workspace ${workspaceId} and account ${connectedAccountId}, falling back to full sync.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.resetSyncExternalId(
|
||||
gmailMessageChannel.id,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (error?.code === 429) {
|
||||
this.logger.log(
|
||||
`429: rate limit reached for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}, import will be retried later.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!historyId) {
|
||||
throw new Error(
|
||||
`No historyId found for ${connectedAccountId} in workspace ${workspaceId} in gmail history response.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (historyId === lastSyncHistoryId || !history?.length) {
|
||||
this.logger.log(
|
||||
`Messaging import done with history ${historyId} and nothing to update for workspace ${workspaceId} and account ${connectedAccountId}`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { messagesAdded, messagesDeleted } =
|
||||
await this.getMessageIdsFromHistory(history);
|
||||
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannel.id}`,
|
||||
messagesAdded,
|
||||
);
|
||||
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-delete:${workspaceId}:gmail:${gmailMessageChannel.id}`,
|
||||
messagesDeleted,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateLastSyncExternalIdIfHigher(
|
||||
gmailMessageChannel.id,
|
||||
historyId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async getMessageIdsFromHistory(
|
||||
history: gmail_v1.Schema$History[],
|
||||
): Promise<{
|
||||
messagesAdded: string[];
|
||||
messagesDeleted: string[];
|
||||
}> {
|
||||
const { messagesAdded, messagesDeleted } = history.reduce(
|
||||
(
|
||||
acc: {
|
||||
messagesAdded: string[];
|
||||
messagesDeleted: string[];
|
||||
},
|
||||
history,
|
||||
) => {
|
||||
const messagesAdded = history.messagesAdded?.map(
|
||||
(messageAdded) => messageAdded.message?.id || '',
|
||||
);
|
||||
|
||||
const messagesDeleted = history.messagesDeleted?.map(
|
||||
(messageDeleted) => messageDeleted.message?.id || '',
|
||||
);
|
||||
|
||||
if (messagesAdded) acc.messagesAdded.push(...messagesAdded);
|
||||
if (messagesDeleted) acc.messagesDeleted.push(...messagesDeleted);
|
||||
|
||||
return acc;
|
||||
},
|
||||
{ messagesAdded: [], messagesDeleted: [] },
|
||||
);
|
||||
|
||||
const uniqueMessagesAdded = messagesAdded.filter(
|
||||
(messageId) => !messagesDeleted.includes(messageId),
|
||||
);
|
||||
|
||||
const uniqueMessagesDeleted = messagesDeleted.filter(
|
||||
(messageId) => !messagesAdded.includes(messageId),
|
||||
);
|
||||
|
||||
return {
|
||||
messagesAdded: uniqueMessagesAdded,
|
||||
messagesDeleted: uniqueMessagesDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
private async getHistoryFromGmail(
|
||||
gmailClient: gmail_v1.Gmail,
|
||||
lastSyncHistoryId: string,
|
||||
): Promise<{
|
||||
history: gmail_v1.Schema$History[];
|
||||
historyId?: string | null;
|
||||
error?: GmailError;
|
||||
}> {
|
||||
const fullHistory: gmail_v1.Schema$History[] = [];
|
||||
let pageToken: string | undefined;
|
||||
let hasMoreMessages = true;
|
||||
let nextHistoryId: string | undefined;
|
||||
|
||||
while (hasMoreMessages) {
|
||||
try {
|
||||
const response = await gmailClient.users.history.list({
|
||||
userId: 'me',
|
||||
maxResults: GMAIL_USERS_HISTORY_MAX_RESULT,
|
||||
pageToken,
|
||||
startHistoryId: lastSyncHistoryId,
|
||||
historyTypes: ['messageAdded', 'messageDeleted'],
|
||||
});
|
||||
|
||||
nextHistoryId = response?.data?.historyId ?? undefined;
|
||||
|
||||
if (response?.data?.history) {
|
||||
fullHistory.push(...response.data.history);
|
||||
}
|
||||
|
||||
pageToken = response?.data?.nextPageToken ?? undefined;
|
||||
hasMoreMessages = !!pageToken;
|
||||
} catch (error) {
|
||||
const errorData = error?.response?.data?.error;
|
||||
|
||||
if (errorData) {
|
||||
return {
|
||||
history: [],
|
||||
error: errorData,
|
||||
historyId: lastSyncHistoryId,
|
||||
};
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return { history: fullHistory, historyId: nextHistoryId };
|
||||
}
|
||||
|
||||
private async fallbackToFullSync(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
await this.messageQueueService.add<GmailFullSyncV2JobData>(
|
||||
GmailFullSyncV2Job.name,
|
||||
{ workspaceId, connectedAccountId },
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -45,7 +45,7 @@ export class GmailPartialSyncService {
|
||||
private readonly messageService: MessageService,
|
||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
private readonly saveMessagesAndCreateContactsService: SaveMessageAndEmitContactCreationEventService,
|
||||
private readonly saveMessagesAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
@ -174,7 +174,6 @@ export class GmailPartialSyncService {
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
'gmail partial-sync',
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
@ -208,12 +207,11 @@ export class GmailPartialSyncService {
|
||||
);
|
||||
|
||||
if (messagesToSave.length !== 0) {
|
||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
||||
await this.saveMessagesAndEmitContactCreationEventService.saveMessagesAndEmitContactCreation(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannelId,
|
||||
'gmail partial-sync',
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -6,7 +6,6 @@ import { v4 } from 'uuid';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { MessageObjectMetadata } from 'src/modules/messaging/standard-objects/message.object-metadata';
|
||||
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||
import { GmailMessage } from 'src/modules/messaging/types/gmail-message';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
@ -38,9 +37,67 @@ export class MessageService {
|
||||
private readonly messageThreadService: MessageThreadService,
|
||||
) {}
|
||||
|
||||
public async saveMessagesWithinTransaction(
|
||||
messages: GmailMessage[],
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
gmailMessageChannelId: string,
|
||||
workspaceId: string,
|
||||
transactionManager: EntityManager,
|
||||
): Promise<Map<string, string>> {
|
||||
const messageExternalIdsAndIdsMap = new Map<string, string>();
|
||||
|
||||
for (const message of messages) {
|
||||
const existingMessageChannelMessageAssociationsCount =
|
||||
await this.messageChannelMessageAssociationRepository.countByMessageExternalIdsAndMessageChannelId(
|
||||
[message.externalId],
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (existingMessageChannelMessageAssociationsCount > 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: This does not handle all thread merging use cases and might create orphan threads.
|
||||
const savedOrExistingMessageThreadId =
|
||||
await this.messageThreadService.saveMessageThreadOrReturnExistingMessageThread(
|
||||
message.headerMessageId,
|
||||
message.messageThreadExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const savedOrExistingMessageId =
|
||||
await this.saveMessageOrReturnExistingMessage(
|
||||
message,
|
||||
savedOrExistingMessageThreadId,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
messageExternalIdsAndIdsMap.set(
|
||||
message.externalId,
|
||||
savedOrExistingMessageId,
|
||||
);
|
||||
|
||||
await this.messageChannelMessageAssociationRepository.insert(
|
||||
gmailMessageChannelId,
|
||||
savedOrExistingMessageId,
|
||||
message.externalId,
|
||||
savedOrExistingMessageThreadId,
|
||||
message.messageThreadExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
return messageExternalIdsAndIdsMap;
|
||||
}
|
||||
|
||||
public async saveMessages(
|
||||
messages: GmailMessage[],
|
||||
dataSourceMetadata: DataSourceEntity,
|
||||
workspaceDataSource: DataSource,
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
gmailMessageChannelId: string,
|
||||
@ -101,7 +158,6 @@ export class MessageService {
|
||||
message,
|
||||
savedOrExistingMessageThreadId,
|
||||
connectedAccount,
|
||||
dataSourceMetadata,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
@ -136,7 +192,6 @@ export class MessageService {
|
||||
message: GmailMessage,
|
||||
messageThreadId: string,
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
dataSourceMetadata: DataSourceEntity,
|
||||
workspaceId: string,
|
||||
manager: EntityManager,
|
||||
): Promise<string> {
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { MessageParticipantRepository } from 'src/modules/messaging/repositories/message-participant.repository';
|
||||
import {
|
||||
@ -31,14 +33,65 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
async saveMessagesAndCreateContacts(
|
||||
public async saveMessagesAndEmitContactCreationEventWithinTransaction(
|
||||
messagesToSave: GmailMessage[],
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
workspaceId: string,
|
||||
gmailMessageChannel: ObjectRecord<MessageChannelObjectMetadata>,
|
||||
transactionManager: EntityManager,
|
||||
) {
|
||||
const messageExternalIdsAndIdsMap =
|
||||
await this.messageService.saveMessagesWithinTransaction(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
gmailMessageChannel.id,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const participantsWithMessageId: (ParticipantWithMessageId & {
|
||||
shouldCreateContact: boolean;
|
||||
})[] = messagesToSave.flatMap((message) => {
|
||||
const messageId = messageExternalIdsAndIdsMap.get(message.externalId);
|
||||
|
||||
return messageId
|
||||
? message.participants.map((participant) => ({
|
||||
...participant,
|
||||
messageId,
|
||||
shouldCreateContact:
|
||||
gmailMessageChannel.isContactAutoCreationEnabled &&
|
||||
message.participants.find((p) => p.role === 'from')?.handle ===
|
||||
connectedAccount.handle,
|
||||
}))
|
||||
: [];
|
||||
});
|
||||
|
||||
await this.messageParticipantRepository.saveMessageParticipants(
|
||||
participantsWithMessageId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (gmailMessageChannel.isContactAutoCreationEnabled) {
|
||||
const contactsToCreate = participantsWithMessageId.filter(
|
||||
(participant) => participant.shouldCreateContact,
|
||||
);
|
||||
|
||||
this.eventEmitter.emit(`createContacts`, {
|
||||
workspaceId,
|
||||
connectedAccountHandle: connectedAccount.handle,
|
||||
contactsToCreate,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async saveMessagesAndEmitContactCreation(
|
||||
messagesToSave: GmailMessage[],
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
workspaceId: string,
|
||||
gmailMessageChannelId: string,
|
||||
jobName?: string,
|
||||
) {
|
||||
const { dataSource: workspaceDataSource, dataSourceMetadata } =
|
||||
const { dataSource: workspaceDataSource } =
|
||||
await this.workspaceDataSourceService.connectedToWorkspaceDataSourceAndReturnMetadata(
|
||||
workspaceId,
|
||||
);
|
||||
@ -47,7 +100,6 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
|
||||
const messageExternalIdsAndIdsMap = await this.messageService.saveMessages(
|
||||
messagesToSave,
|
||||
dataSourceMetadata,
|
||||
workspaceDataSource,
|
||||
connectedAccount,
|
||||
gmailMessageChannelId,
|
||||
@ -57,7 +109,7 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} saving messages for workspace ${workspaceId} and account ${
|
||||
`Saving messages for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
} in ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -100,13 +152,12 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
gmailMessageChannel,
|
||||
workspaceId,
|
||||
connectedAccount,
|
||||
jobName,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} saving message participants for workspace ${workspaceId} and account in ${
|
||||
`Saving message participants for workspace ${workspaceId} and account in ${
|
||||
connectedAccount.id
|
||||
} ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -119,7 +170,6 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
gmailMessageChannel: ObjectRecord<MessageChannelObjectMetadata>,
|
||||
workspaceId: string,
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
jobName?: string,
|
||||
) {
|
||||
try {
|
||||
await this.messageParticipantRepository.saveMessageParticipants(
|
||||
@ -140,7 +190,7 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`${jobName} error saving message participants for workspace ${workspaceId} and account ${connectedAccount.id}`,
|
||||
`Error saving message participants for workspace ${workspaceId} and account ${connectedAccount.id}`,
|
||||
error,
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user