[messaing] improve messaging import (#4650)
* [messaging] improve full-sync fetching strategy * fix * rebase * fix * fix * fix rebase * fix * fix * fix * fix * fix * remove deletion * fix setPop with memory storage * fix pgBoss and remove unnecessary job * fix throw * fix * add timeout to ongoing sync
This commit is contained in:
Weiko
@ -119,8 +119,9 @@ export class CreateCompanyAndContactService {
|
||||
handle: contact.handle,
|
||||
displayName: contact.displayName,
|
||||
companyId:
|
||||
contact.companyDomainName &&
|
||||
companiesObject[contact.companyDomainName],
|
||||
contact.companyDomainName && contact.companyDomainName !== ''
|
||||
? companiesObject[contact.companyDomainName]
|
||||
: undefined,
|
||||
}));
|
||||
|
||||
await this.createContactService.createContacts(
|
||||
|
||||
@ -0,0 +1,84 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository, In } from 'typeorm';
|
||||
|
||||
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||
|
||||
import {
|
||||
FeatureFlagEntity,
|
||||
FeatureFlagKeys,
|
||||
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { GmailFetchMessageContentFromCacheService } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service';
|
||||
|
||||
@Injectable()
|
||||
export class FetchAllMessagesFromCacheCronJob
|
||||
implements MessageQueueJob<undefined>
|
||||
{
|
||||
private readonly logger = new Logger(FetchAllMessagesFromCacheCronJob.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
private readonly workspaceRepository: Repository<Workspace>,
|
||||
@InjectRepository(DataSourceEntity, 'metadata')
|
||||
private readonly dataSourceRepository: Repository<DataSourceEntity>,
|
||||
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
private readonly gmailFetchMessageContentFromCacheService: GmailFetchMessageContentFromCacheService,
|
||||
) {}
|
||||
|
||||
async handle(): Promise<void> {
|
||||
const workspaceIds = (
|
||||
await this.workspaceRepository.find({
|
||||
where: {
|
||||
subscriptionStatus: 'active',
|
||||
},
|
||||
select: ['id'],
|
||||
})
|
||||
).map((workspace) => workspace.id);
|
||||
|
||||
const workspacesWithFeatureFlagActive =
|
||||
await this.featureFlagRepository.find({
|
||||
where: {
|
||||
workspaceId: In(workspaceIds),
|
||||
key: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||
value: true,
|
||||
},
|
||||
});
|
||||
|
||||
const dataSources = await this.dataSourceRepository.find({
|
||||
where: {
|
||||
workspaceId: In(
|
||||
workspacesWithFeatureFlagActive.map((w) => w.workspaceId),
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
const workspaceIdsWithDataSources = new Set(
|
||||
dataSources.map((dataSource) => dataSource.workspaceId),
|
||||
);
|
||||
|
||||
for (const workspaceId of workspaceIdsWithDataSources) {
|
||||
await this.fetchWorkspaceMessages(workspaceId);
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
|
||||
const messageChannels =
|
||||
await this.messageChannelRepository.getAll(workspaceId);
|
||||
|
||||
for (const messageChannel of messageChannels) {
|
||||
await this.gmailFetchMessageContentFromCacheService.fetchMessageContentFromCache(
|
||||
workspaceId,
|
||||
messageChannel.connectedAccountId,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1 +1 @@
|
||||
export const fetchAllWorkspacesMessagesCronPattern = '*/10 * * * *';
|
||||
export const fetchAllWorkspacesMessagesCronPattern = '*/5 * * * *';
|
||||
|
||||
@ -16,6 +16,14 @@ import {
|
||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import {
|
||||
FeatureFlagEntity,
|
||||
FeatureFlagKeys,
|
||||
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import {
|
||||
GmailPartialSyncV2Job as GmailPartialSyncV2Job,
|
||||
GmailPartialSyncV2JobData as GmailPartialSyncV2JobData,
|
||||
} from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job';
|
||||
|
||||
@Injectable()
|
||||
export class FetchAllWorkspacesMessagesJob
|
||||
@ -32,6 +40,8 @@ export class FetchAllWorkspacesMessagesJob
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
|
||||
async handle(): Promise<void> {
|
||||
@ -61,17 +71,33 @@ export class FetchAllWorkspacesMessagesJob
|
||||
|
||||
private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
|
||||
try {
|
||||
const isFullSyncV2Enabled = await this.featureFlagRepository.findOneBy({
|
||||
workspaceId,
|
||||
key: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||
value: true,
|
||||
});
|
||||
|
||||
const connectedAccounts =
|
||||
await this.connectedAccountRepository.getAll(workspaceId);
|
||||
|
||||
for (const connectedAccount of connectedAccounts) {
|
||||
await this.messageQueueService.add<GmailPartialSyncJobData>(
|
||||
GmailPartialSyncJob.name,
|
||||
{
|
||||
workspaceId,
|
||||
connectedAccountId: connectedAccount.id,
|
||||
},
|
||||
);
|
||||
if (isFullSyncV2Enabled) {
|
||||
await this.messageQueueService.add<GmailPartialSyncV2JobData>(
|
||||
GmailPartialSyncV2Job.name,
|
||||
{
|
||||
workspaceId,
|
||||
connectedAccountId: connectedAccount.id,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
await this.messageQueueService.add<GmailPartialSyncJobData>(
|
||||
GmailPartialSyncJob.name,
|
||||
{
|
||||
workspaceId,
|
||||
connectedAccountId: connectedAccount.id,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
|
||||
@ -4,6 +4,7 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { GmailFullSyncCommand } from 'src/modules/messaging/commands/gmail-full-sync.command';
|
||||
import { GmailPartialSyncCommand } from 'src/modules/messaging/commands/gmail-partial-sync.command';
|
||||
import { StartFetchAllWorkspacesMessagesFromCacheCronCommand } from 'src/modules/messaging/commands/start-fetch-all-workspaces-messages-from-cache.cron.command';
|
||||
import { StartFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging/commands/start-fetch-all-workspaces-messages.cron.command';
|
||||
import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging/commands/stop-fetch-all-workspaces-messages.cron.command';
|
||||
|
||||
@ -16,6 +17,7 @@ import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging
|
||||
GmailPartialSyncCommand,
|
||||
StartFetchAllWorkspacesMessagesCronCommand,
|
||||
StopFetchAllWorkspacesMessagesCronCommand,
|
||||
StartFetchAllWorkspacesMessagesFromCacheCronCommand,
|
||||
],
|
||||
})
|
||||
export class FetchWorkspaceMessagesCommandsModule {}
|
||||
|
||||
@ -0,0 +1,32 @@
|
||||
import { Inject } from '@nestjs/common';
|
||||
|
||||
import { Command, CommandRunner } from 'nest-commander';
|
||||
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/commands/crons/fetch-all-messages-from-cache.cron-job';
|
||||
|
||||
@Command({
|
||||
name: 'fetch-all-workspaces-messages-from-cache:cron:start',
|
||||
description: 'Starts a cron job to fetch all workspaces messages from cache',
|
||||
})
|
||||
export class StartFetchAllWorkspacesMessagesFromCacheCronCommand extends CommandRunner {
|
||||
constructor(
|
||||
@Inject(MessageQueue.cronQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
await this.messageQueueService.addCron<undefined>(
|
||||
FetchAllMessagesFromCacheCronJob.name,
|
||||
undefined,
|
||||
{
|
||||
repeat: {
|
||||
every: 5000,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -23,7 +23,9 @@ export class StartFetchAllWorkspacesMessagesCronCommand extends CommandRunner {
|
||||
await this.messageQueueService.addCron<undefined>(
|
||||
FetchAllWorkspacesMessagesJob.name,
|
||||
undefined,
|
||||
fetchAllWorkspacesMessagesCronPattern,
|
||||
{
|
||||
repeat: { pattern: fetchAllWorkspacesMessagesCronPattern },
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1 @@
|
||||
export const GMAIL_ONGOING_SYNC_TIMEOUT = 1000 * 60 * 60; // 1 hour
|
||||
@ -0,0 +1 @@
|
||||
export const GMAIL_USERS_HISTORY_MAX_RESULT = 500;
|
||||
@ -0,0 +1 @@
|
||||
export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 50;
|
||||
@ -0,0 +1 @@
|
||||
export const GMAIL_USERS_MESSAGES_LIST_MAX_RESULT = 500;
|
||||
@ -0,0 +1 @@
|
||||
export const MESSAGES_TO_DELETE_FROM_CACHE_BATCH_SIZE = 1000;
|
||||
@ -0,0 +1,48 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||
|
||||
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
|
||||
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
|
||||
|
||||
export type GmailFullSyncV2JobData = {
|
||||
workspaceId: string;
|
||||
connectedAccountId: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class GmailFullSyncV2Job
|
||||
implements MessageQueueJob<GmailFullSyncV2JobData>
|
||||
{
|
||||
private readonly logger = new Logger(GmailFullSyncV2Job.name);
|
||||
|
||||
constructor(
|
||||
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
|
||||
private readonly gmailFullSyncV2Service: GmailFullSyncV2Service,
|
||||
) {}
|
||||
|
||||
async handle(data: GmailFullSyncV2JobData): Promise<void> {
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
|
||||
data.workspaceId,
|
||||
data.connectedAccountId,
|
||||
);
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
|
||||
e,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.gmailFullSyncV2Service.fetchConnectedAccountThreads(
|
||||
data.workspaceId,
|
||||
data.connectedAccountId,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -44,7 +44,6 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
|
||||
await this.gmailFullSyncService.fetchConnectedAccountThreads(
|
||||
data.workspaceId,
|
||||
data.connectedAccountId,
|
||||
data.nextPageToken,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,48 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||
|
||||
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
|
||||
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
|
||||
|
||||
export type GmailPartialSyncV2JobData = {
|
||||
workspaceId: string;
|
||||
connectedAccountId: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class GmailPartialSyncV2Job
|
||||
implements MessageQueueJob<GmailPartialSyncV2JobData>
|
||||
{
|
||||
private readonly logger = new Logger(GmailPartialSyncV2Job.name);
|
||||
|
||||
constructor(
|
||||
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
|
||||
private readonly gmailPartialSyncV2Service: GmailPartialSyncV2Service,
|
||||
) {}
|
||||
|
||||
async handle(data: GmailPartialSyncV2JobData): Promise<void> {
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
|
||||
data.workspaceId,
|
||||
data.connectedAccountId,
|
||||
);
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
|
||||
e,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.gmailPartialSyncV2Service.fetchConnectedAccountThreads(
|
||||
data.workspaceId,
|
||||
data.connectedAccountId,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -3,7 +3,10 @@ import { Injectable } from '@nestjs/common';
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import {
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelSyncStatus,
|
||||
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
||||
|
||||
@Injectable()
|
||||
@ -12,6 +15,21 @@ export class MessageChannelRepository {
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
public async getAll(
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
): Promise<ObjectRecord<MessageChannelObjectMetadata>[]> {
|
||||
const dataSourceSchema =
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
return await this.workspaceDataSourceService.executeRawQuery(
|
||||
`SELECT * FROM ${dataSourceSchema}."messageChannel"`,
|
||||
[],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
public async getByConnectedAccountId(
|
||||
connectedAccountId: string,
|
||||
workspaceId: string,
|
||||
@ -49,27 +67,17 @@ export class MessageChannelRepository {
|
||||
public async getFirstByConnectedAccountId(
|
||||
connectedAccountId: string,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
): Promise<ObjectRecord<MessageChannelObjectMetadata> | undefined> {
|
||||
const messageChannels = await this.getByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
return messageChannels[0];
|
||||
}
|
||||
|
||||
public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail(
|
||||
connectedAccountId: string,
|
||||
workspaceId: string,
|
||||
): Promise<boolean> {
|
||||
const messageChannel = await this.getFirstByConnectedAccountIdOrFail(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return messageChannel.isContactAutoCreationEnabled;
|
||||
}
|
||||
|
||||
public async getByIds(
|
||||
ids: string[],
|
||||
workspaceId: string,
|
||||
@ -85,4 +93,69 @@ export class MessageChannelRepository {
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
public async updateSyncStatus(
|
||||
id: string,
|
||||
syncStatus: MessageChannelSyncStatus,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
): Promise<void> {
|
||||
const dataSourceSchema =
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
const needsToUpdateSyncedAt =
|
||||
syncStatus === MessageChannelSyncStatus.SUCCEEDED;
|
||||
|
||||
const needsToUpdateOngoingSyncStartedAt =
|
||||
syncStatus === MessageChannelSyncStatus.ONGOING;
|
||||
|
||||
await this.workspaceDataSourceService.executeRawQuery(
|
||||
`UPDATE ${dataSourceSchema}."messageChannel" SET "syncStatus" = $1 ${
|
||||
needsToUpdateSyncedAt ? `, "syncedAt" = NOW()` : ''
|
||||
} ${
|
||||
needsToUpdateOngoingSyncStartedAt
|
||||
? `, "ongoingSyncStartedAt" = NOW()`
|
||||
: `, "ongoingSyncStartedAt" = NULL`
|
||||
} WHERE "id" = $2`,
|
||||
[syncStatus, id],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
public async updateLastSyncExternalIdIfHigher(
|
||||
id: string,
|
||||
syncExternalId: string,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) {
|
||||
const dataSourceSchema =
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
await this.workspaceDataSourceService.executeRawQuery(
|
||||
`UPDATE ${dataSourceSchema}."messageChannel" SET "syncExternalId" = $1
|
||||
WHERE "id" = $2
|
||||
AND ("syncExternalId" < $1 OR "syncExternalId" = '')`,
|
||||
[syncExternalId, id],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
public async resetSyncExternalId(
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) {
|
||||
const dataSourceSchema =
|
||||
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||
|
||||
await this.workspaceDataSourceService.executeRawQuery(
|
||||
`UPDATE ${dataSourceSchema}."messageChannel" SET "syncExternalId" = ''
|
||||
WHERE "id" = $1`,
|
||||
[id],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,7 +19,6 @@ export class FetchMessagesByBatchesService {
|
||||
async fetchAllMessages(
|
||||
queries: MessageQuery[],
|
||||
accessToken: string,
|
||||
jobName?: string,
|
||||
workspaceId?: string,
|
||||
connectedAccountId?: string,
|
||||
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
|
||||
@ -32,7 +31,7 @@ export class FetchMessagesByBatchesService {
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
|
||||
queries.length
|
||||
} messages in ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -45,7 +44,7 @@ export class FetchMessagesByBatchesService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
|
||||
queries.length
|
||||
} messages in ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -62,6 +61,10 @@ export class FetchMessagesByBatchesService {
|
||||
|
||||
const errors: any = [];
|
||||
|
||||
const sanitizeString = (str: string) => {
|
||||
return str.replace(/\0/g, '');
|
||||
};
|
||||
|
||||
const formattedResponse = Promise.all(
|
||||
parsedResponses.map(async (message: GmailMessageParsedResponse) => {
|
||||
if (message.error) {
|
||||
@ -119,7 +122,7 @@ export class FetchMessagesByBatchesService {
|
||||
fromHandle: from.value[0].address || '',
|
||||
fromDisplayName: from.value[0].name || '',
|
||||
participants,
|
||||
text: textWithoutReplyQuotations || '',
|
||||
text: sanitizeString(textWithoutReplyQuotations || ''),
|
||||
attachments,
|
||||
};
|
||||
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailFetchMessageContentFromCacheService } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service';
|
||||
import { MessageModule } from 'src/modules/messaging/services/message/message.module';
|
||||
import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
FetchMessagesByBatchesModule,
|
||||
ObjectMetadataRepositoryModule.forFeature([
|
||||
ConnectedAccountObjectMetadata,
|
||||
MessageChannelObjectMetadata,
|
||||
]),
|
||||
SaveMessageAndEmitContactCreationEventModule,
|
||||
MessageModule,
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailFetchMessageContentFromCacheService],
|
||||
exports: [GmailFetchMessageContentFromCacheService],
|
||||
})
|
||||
export class GmailFetchMessageContentFromCacheModule {}
|
||||
@ -0,0 +1,257 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { FetchMessagesByBatchesService } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service';
|
||||
import {
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelSyncStatus,
|
||||
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { createQueriesFromMessageIds } from 'src/modules/messaging/utils/create-queries-from-message-ids.util';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { SaveMessageAndEmitContactCreationEventService } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.service';
|
||||
import {
|
||||
GmailFullSyncV2JobData,
|
||||
GmailFullSyncV2Job,
|
||||
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { GMAIL_ONGOING_SYNC_TIMEOUT } from 'src/modules/messaging/constants/gmail-ongoing-sync-timeout.constant';
|
||||
|
||||
@Injectable()
|
||||
export class GmailFetchMessageContentFromCacheService {
|
||||
private readonly logger = new Logger(
|
||||
GmailFetchMessageContentFromCacheService.name,
|
||||
);
|
||||
|
||||
constructor(
|
||||
private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
private readonly saveMessageAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
@Inject(MessageQueue.messagingQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
) {}
|
||||
|
||||
async fetchMessageContentFromCache(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const accessToken = connectedAccount.accessToken;
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.PENDING) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is not pending.`,
|
||||
);
|
||||
|
||||
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.ONGOING) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ongoingSyncStartedAt = new Date(
|
||||
gmailMessageChannel.ongoingSyncStartedAt,
|
||||
);
|
||||
|
||||
if (
|
||||
ongoingSyncStartedAt < new Date(Date.now() - GMAIL_ONGOING_SYNC_TIMEOUT)
|
||||
) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} failed due to ongoing sync timeout. Restarting full-sync...`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const gmailMessageChannelId = gmailMessageChannel.id;
|
||||
|
||||
const messageIdsToFetch =
|
||||
(await this.cacheStorage.setPop(
|
||||
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannelId}`,
|
||||
GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
|
||||
)) ?? [];
|
||||
|
||||
if (!messageIdsToFetch?.length) {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.SUCCEEDED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import or delete.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.ONGOING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} starting...`,
|
||||
);
|
||||
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await workspaceDataSource
|
||||
?.transaction(async (transactionManager: EntityManager) => {
|
||||
const messageQueries = createQueriesFromMessageIds(messageIdsToFetch);
|
||||
|
||||
const { messages: messagesToSave, errors } =
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
|
||||
if (!messagesToSave.length) {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
const errorsCanBeIgnored = errors.every(
|
||||
(error) => error.code === 404,
|
||||
);
|
||||
|
||||
if (!errorsCanBeIgnored) {
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${JSON.stringify(
|
||||
errors,
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.saveMessageAndEmitContactCreationEventService.saveMessagesAndEmitContactCreationEventWithinTransaction(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannel,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (messageIdsToFetch.length < GMAIL_USERS_MESSAGES_GET_BATCH_SIZE) {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.SUCCEEDED,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with no more messages to import.`,
|
||||
);
|
||||
} else {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with more messages to import.`,
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannelId}`,
|
||||
messageIdsToFetch,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannelId,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async fallbackToFullSync(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
await this.messageQueueService.add<GmailFullSyncV2JobData>(
|
||||
GmailFullSyncV2Job.name,
|
||||
{ workspaceId, connectedAccountId },
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
|
||||
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
MessagingProvidersModule,
|
||||
FetchMessagesByBatchesModule,
|
||||
ObjectMetadataRepositoryModule.forFeature([
|
||||
ConnectedAccountObjectMetadata,
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelMessageAssociationObjectMetadata,
|
||||
BlocklistObjectMetadata,
|
||||
]),
|
||||
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailFullSyncV2Service],
|
||||
exports: [GmailFullSyncV2Service],
|
||||
})
|
||||
export class GmailFullSynV2Module {}
|
||||
@ -0,0 +1,302 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { EntityManager, Repository } from 'typeorm';
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
|
||||
import {
|
||||
FeatureFlagEntity,
|
||||
FeatureFlagKeys,
|
||||
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/constants/gmail-users-messages-list-max-result.constant';
|
||||
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/repositories/message-channel-message-association.repository';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||
import {
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelSyncStatus,
|
||||
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { gmailSearchFilterExcludeEmails } from 'src/modules/messaging/utils/gmail-search-filter.util';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
|
||||
@Injectable()
|
||||
export class GmailFullSyncV2Service {
|
||||
private readonly logger = new Logger(GmailFullSyncV2Service.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
@InjectObjectMetadataRepository(
|
||||
MessageChannelMessageAssociationObjectMetadata,
|
||||
)
|
||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
public async fetchConnectedAccountThreads(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (gmailMessageChannel.syncStatus === MessageChannelSyncStatus.ONGOING) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is locked, import will be retried later.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.ONGOING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await workspaceDataSource
|
||||
?.transaction(async (transactionManager) => {
|
||||
const gmailClient: gmail_v1.Gmail =
|
||||
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||
|
||||
const blocklistedEmails = await this.fetchBlocklistEmails(
|
||||
connectedAccount.accountOwnerId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||
gmailClient,
|
||||
gmailMessageChannel.id,
|
||||
blocklistedEmails,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
public async fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||
gmailClient: gmail_v1.Gmail,
|
||||
messageChannelId: string,
|
||||
blocklistedEmails: string[],
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) {
|
||||
let pageToken: string | undefined;
|
||||
let hasMoreMessages = true;
|
||||
let messageIdsToFetch = 0;
|
||||
let firstMessageExternalId;
|
||||
|
||||
while (hasMoreMessages) {
|
||||
const response = await gmailClient.users.messages.list({
|
||||
userId: 'me',
|
||||
maxResults: GMAIL_USERS_MESSAGES_LIST_MAX_RESULT,
|
||||
pageToken,
|
||||
q: gmailSearchFilterExcludeEmails(blocklistedEmails),
|
||||
});
|
||||
|
||||
if (response.data?.messages) {
|
||||
const messageExternalIds = response.data.messages
|
||||
.filter((message): message is { id: string } => message.id != null)
|
||||
.map((message) => message.id);
|
||||
|
||||
if (!firstMessageExternalId) {
|
||||
firstMessageExternalId = messageExternalIds[0];
|
||||
}
|
||||
|
||||
const existingMessageChannelMessageAssociations =
|
||||
await this.messageChannelMessageAssociationRepository.getByMessageExternalIdsAndMessageChannelId(
|
||||
messageExternalIds,
|
||||
messageChannelId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const existingMessageChannelMessageAssociationsExternalIds =
|
||||
existingMessageChannelMessageAssociations.map(
|
||||
(messageChannelMessageAssociation) =>
|
||||
messageChannelMessageAssociation.messageExternalId,
|
||||
);
|
||||
|
||||
const messageIdsToImport = messageExternalIds.filter(
|
||||
(messageExternalId) =>
|
||||
!existingMessageChannelMessageAssociationsExternalIds.includes(
|
||||
messageExternalId,
|
||||
),
|
||||
);
|
||||
|
||||
if (messageIdsToImport && messageIdsToImport.length) {
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
|
||||
messageIdsToImport,
|
||||
);
|
||||
|
||||
messageIdsToFetch += messageIdsToImport.length;
|
||||
}
|
||||
}
|
||||
|
||||
pageToken = response.data.nextPageToken ?? undefined;
|
||||
hasMoreMessages = !!pageToken;
|
||||
}
|
||||
|
||||
if (!messageIdsToFetch) {
|
||||
this.logger.log(
|
||||
`No messages found in Gmail for messageChannel ${messageChannelId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Fetched all ${messageIdsToFetch} message ids from Gmail for messageChannel ${messageChannelId} in workspace ${workspaceId} and added to cache for import`,
|
||||
);
|
||||
|
||||
await this.updateLastSyncExternalId(
|
||||
gmailClient,
|
||||
messageChannelId,
|
||||
firstMessageExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
public async fetchBlocklistEmails(
|
||||
workspaceMemberId: string,
|
||||
workspaceId: string,
|
||||
) {
|
||||
const isBlocklistEnabledFeatureFlag =
|
||||
await this.featureFlagRepository.findOneBy({
|
||||
workspaceId,
|
||||
key: FeatureFlagKeys.IsBlocklistEnabled,
|
||||
value: true,
|
||||
});
|
||||
|
||||
const isBlocklistEnabled =
|
||||
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
|
||||
|
||||
const blocklist = isBlocklistEnabled
|
||||
? await this.blocklistRepository.getByWorkspaceMemberId(
|
||||
workspaceMemberId,
|
||||
workspaceId,
|
||||
)
|
||||
: [];
|
||||
|
||||
return blocklist.map((blocklist) => blocklist.handle);
|
||||
}
|
||||
|
||||
private async updateLastSyncExternalId(
|
||||
gmailClient: gmail_v1.Gmail,
|
||||
messageChannelId: string,
|
||||
firstMessageExternalId: string,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) {
|
||||
if (!firstMessageExternalId) {
|
||||
throw new Error(
|
||||
`No first message found for workspace ${workspaceId} and account ${messageChannelId}, can't update sync external id`,
|
||||
);
|
||||
}
|
||||
|
||||
const firstMessageContent = await gmailClient.users.messages.get({
|
||||
userId: 'me',
|
||||
id: firstMessageExternalId,
|
||||
});
|
||||
|
||||
if (!firstMessageContent?.data) {
|
||||
throw new Error(
|
||||
`No first message content found for message ${firstMessageExternalId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const historyId = firstMessageContent?.data?.historyId;
|
||||
|
||||
if (!historyId) {
|
||||
throw new Error(
|
||||
`No historyId found for message ${firstMessageExternalId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Updating last external id: ${historyId} for workspace ${workspaceId} and account ${messageChannelId} succeeded.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateLastSyncExternalIdIfHigher(
|
||||
messageChannelId,
|
||||
historyId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -47,7 +47,7 @@ export class GmailFullSyncService {
|
||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
private readonly saveMessagesAndCreateContactsService: SaveMessageAndEmitContactCreationEventService,
|
||||
private readonly saveMessagesAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
@ -186,7 +186,6 @@ export class GmailFullSyncService {
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
'gmail full-sync',
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
@ -200,12 +199,11 @@ export class GmailFullSyncService {
|
||||
);
|
||||
|
||||
if (messagesToSave.length > 0) {
|
||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
||||
await this.saveMessagesAndEmitContactCreationEventService.saveMessagesAndEmitContactCreation(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannelId,
|
||||
'gmail full-sync',
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
|
||||
import { MessageModule } from 'src/modules/messaging/services/message/message.module';
|
||||
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||
import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
MessagingProvidersModule,
|
||||
FetchMessagesByBatchesModule,
|
||||
ObjectMetadataRepositoryModule.forFeature([
|
||||
ConnectedAccountObjectMetadata,
|
||||
MessageChannelObjectMetadata,
|
||||
BlocklistObjectMetadata,
|
||||
]),
|
||||
MessageModule,
|
||||
SaveMessageAndEmitContactCreationEventModule,
|
||||
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
],
|
||||
providers: [GmailPartialSyncV2Service],
|
||||
exports: [GmailPartialSyncV2Service],
|
||||
})
|
||||
export class GmailPartialSyncV2Module {}
|
||||
@ -0,0 +1,338 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import {
|
||||
MessageChannelObjectMetadata,
|
||||
MessageChannelSyncStatus,
|
||||
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { GMAIL_USERS_HISTORY_MAX_RESULT } from 'src/modules/messaging/constants/gmail-users-history-max-result.constant';
|
||||
import { GmailError } from 'src/modules/messaging/types/gmail-error';
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import {
|
||||
GmailFullSyncV2Job,
|
||||
GmailFullSyncV2JobData,
|
||||
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||
|
||||
@Injectable()
|
||||
export class GmailPartialSyncV2Service {
|
||||
private readonly logger = new Logger(GmailPartialSyncV2Service.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
@Inject(MessageQueue.messagingQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||
private readonly messageChannelRepository: MessageChannelRepository,
|
||||
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
public async fetchConnectedAccountThreads(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
): Promise<void> {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.SUCCEEDED) {
|
||||
this.logger.log(
|
||||
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is locked, import will be retried later.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.ONGOING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const workspaceDataSource =
|
||||
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await workspaceDataSource
|
||||
?.transaction(async (transactionManager: EntityManager) => {
|
||||
const lastSyncHistoryId = gmailMessageChannel.syncExternalId;
|
||||
|
||||
if (!lastSyncHistoryId) {
|
||||
this.logger.log(
|
||||
`No lastSyncHistoryId for workspace ${workspaceId} and account ${connectedAccountId}, falling back to full sync.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const gmailClient: gmail_v1.Gmail =
|
||||
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||
|
||||
const { history, historyId, error } = await this.getHistoryFromGmail(
|
||||
gmailClient,
|
||||
lastSyncHistoryId,
|
||||
);
|
||||
|
||||
if (error?.code === 404) {
|
||||
this.logger.log(
|
||||
`404: Invalid lastSyncHistoryId: ${lastSyncHistoryId} for workspace ${workspaceId} and account ${connectedAccountId}, falling back to full sync.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.resetSyncExternalId(
|
||||
gmailMessageChannel.id,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (error?.code === 429) {
|
||||
this.logger.log(
|
||||
`429: rate limit reached for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}, import will be retried later.`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!historyId) {
|
||||
throw new Error(
|
||||
`No historyId found for ${connectedAccountId} in workspace ${workspaceId} in gmail history response.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (historyId === lastSyncHistoryId || !history?.length) {
|
||||
this.logger.log(
|
||||
`Messaging import done with history ${historyId} and nothing to update for workspace ${workspaceId} and account ${connectedAccountId}`,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { messagesAdded, messagesDeleted } =
|
||||
await this.getMessageIdsFromHistory(history);
|
||||
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannel.id}`,
|
||||
messagesAdded,
|
||||
);
|
||||
|
||||
await this.cacheStorage.setAdd(
|
||||
`messages-to-delete:${workspaceId}:gmail:${gmailMessageChannel.id}`,
|
||||
messagesDeleted,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateLastSyncExternalIdIfHigher(
|
||||
gmailMessageChannel.id,
|
||||
historyId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.PENDING,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await this.messageChannelRepository.updateSyncStatus(
|
||||
gmailMessageChannel.id,
|
||||
MessageChannelSyncStatus.FAILED,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async getMessageIdsFromHistory(
|
||||
history: gmail_v1.Schema$History[],
|
||||
): Promise<{
|
||||
messagesAdded: string[];
|
||||
messagesDeleted: string[];
|
||||
}> {
|
||||
const { messagesAdded, messagesDeleted } = history.reduce(
|
||||
(
|
||||
acc: {
|
||||
messagesAdded: string[];
|
||||
messagesDeleted: string[];
|
||||
},
|
||||
history,
|
||||
) => {
|
||||
const messagesAdded = history.messagesAdded?.map(
|
||||
(messageAdded) => messageAdded.message?.id || '',
|
||||
);
|
||||
|
||||
const messagesDeleted = history.messagesDeleted?.map(
|
||||
(messageDeleted) => messageDeleted.message?.id || '',
|
||||
);
|
||||
|
||||
if (messagesAdded) acc.messagesAdded.push(...messagesAdded);
|
||||
if (messagesDeleted) acc.messagesDeleted.push(...messagesDeleted);
|
||||
|
||||
return acc;
|
||||
},
|
||||
{ messagesAdded: [], messagesDeleted: [] },
|
||||
);
|
||||
|
||||
const uniqueMessagesAdded = messagesAdded.filter(
|
||||
(messageId) => !messagesDeleted.includes(messageId),
|
||||
);
|
||||
|
||||
const uniqueMessagesDeleted = messagesDeleted.filter(
|
||||
(messageId) => !messagesAdded.includes(messageId),
|
||||
);
|
||||
|
||||
return {
|
||||
messagesAdded: uniqueMessagesAdded,
|
||||
messagesDeleted: uniqueMessagesDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
private async getHistoryFromGmail(
|
||||
gmailClient: gmail_v1.Gmail,
|
||||
lastSyncHistoryId: string,
|
||||
): Promise<{
|
||||
history: gmail_v1.Schema$History[];
|
||||
historyId?: string | null;
|
||||
error?: GmailError;
|
||||
}> {
|
||||
const fullHistory: gmail_v1.Schema$History[] = [];
|
||||
let pageToken: string | undefined;
|
||||
let hasMoreMessages = true;
|
||||
let nextHistoryId: string | undefined;
|
||||
|
||||
while (hasMoreMessages) {
|
||||
try {
|
||||
const response = await gmailClient.users.history.list({
|
||||
userId: 'me',
|
||||
maxResults: GMAIL_USERS_HISTORY_MAX_RESULT,
|
||||
pageToken,
|
||||
startHistoryId: lastSyncHistoryId,
|
||||
historyTypes: ['messageAdded', 'messageDeleted'],
|
||||
});
|
||||
|
||||
nextHistoryId = response?.data?.historyId ?? undefined;
|
||||
|
||||
if (response?.data?.history) {
|
||||
fullHistory.push(...response.data.history);
|
||||
}
|
||||
|
||||
pageToken = response?.data?.nextPageToken ?? undefined;
|
||||
hasMoreMessages = !!pageToken;
|
||||
} catch (error) {
|
||||
const errorData = error?.response?.data?.error;
|
||||
|
||||
if (errorData) {
|
||||
return {
|
||||
history: [],
|
||||
error: errorData,
|
||||
historyId: lastSyncHistoryId,
|
||||
};
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return { history: fullHistory, historyId: nextHistoryId };
|
||||
}
|
||||
|
||||
private async fallbackToFullSync(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
await this.messageQueueService.add<GmailFullSyncV2JobData>(
|
||||
GmailFullSyncV2Job.name,
|
||||
{ workspaceId, connectedAccountId },
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -45,7 +45,7 @@ export class GmailPartialSyncService {
|
||||
private readonly messageService: MessageService,
|
||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
private readonly saveMessagesAndCreateContactsService: SaveMessageAndEmitContactCreationEventService,
|
||||
private readonly saveMessagesAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
@ -174,7 +174,6 @@ export class GmailPartialSyncService {
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
'gmail partial-sync',
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
@ -208,12 +207,11 @@ export class GmailPartialSyncService {
|
||||
);
|
||||
|
||||
if (messagesToSave.length !== 0) {
|
||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
||||
await this.saveMessagesAndEmitContactCreationEventService.saveMessagesAndEmitContactCreation(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannelId,
|
||||
'gmail partial-sync',
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -6,7 +6,6 @@ import { v4 } from 'uuid';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { MessageObjectMetadata } from 'src/modules/messaging/standard-objects/message.object-metadata';
|
||||
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||
import { GmailMessage } from 'src/modules/messaging/types/gmail-message';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
@ -38,9 +37,67 @@ export class MessageService {
|
||||
private readonly messageThreadService: MessageThreadService,
|
||||
) {}
|
||||
|
||||
public async saveMessagesWithinTransaction(
|
||||
messages: GmailMessage[],
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
gmailMessageChannelId: string,
|
||||
workspaceId: string,
|
||||
transactionManager: EntityManager,
|
||||
): Promise<Map<string, string>> {
|
||||
const messageExternalIdsAndIdsMap = new Map<string, string>();
|
||||
|
||||
for (const message of messages) {
|
||||
const existingMessageChannelMessageAssociationsCount =
|
||||
await this.messageChannelMessageAssociationRepository.countByMessageExternalIdsAndMessageChannelId(
|
||||
[message.externalId],
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (existingMessageChannelMessageAssociationsCount > 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: This does not handle all thread merging use cases and might create orphan threads.
|
||||
const savedOrExistingMessageThreadId =
|
||||
await this.messageThreadService.saveMessageThreadOrReturnExistingMessageThread(
|
||||
message.headerMessageId,
|
||||
message.messageThreadExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const savedOrExistingMessageId =
|
||||
await this.saveMessageOrReturnExistingMessage(
|
||||
message,
|
||||
savedOrExistingMessageThreadId,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
messageExternalIdsAndIdsMap.set(
|
||||
message.externalId,
|
||||
savedOrExistingMessageId,
|
||||
);
|
||||
|
||||
await this.messageChannelMessageAssociationRepository.insert(
|
||||
gmailMessageChannelId,
|
||||
savedOrExistingMessageId,
|
||||
message.externalId,
|
||||
savedOrExistingMessageThreadId,
|
||||
message.messageThreadExternalId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
}
|
||||
|
||||
return messageExternalIdsAndIdsMap;
|
||||
}
|
||||
|
||||
public async saveMessages(
|
||||
messages: GmailMessage[],
|
||||
dataSourceMetadata: DataSourceEntity,
|
||||
workspaceDataSource: DataSource,
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
gmailMessageChannelId: string,
|
||||
@ -101,7 +158,6 @@ export class MessageService {
|
||||
message,
|
||||
savedOrExistingMessageThreadId,
|
||||
connectedAccount,
|
||||
dataSourceMetadata,
|
||||
workspaceId,
|
||||
manager,
|
||||
);
|
||||
@ -136,7 +192,6 @@ export class MessageService {
|
||||
message: GmailMessage,
|
||||
messageThreadId: string,
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
dataSourceMetadata: DataSourceEntity,
|
||||
workspaceId: string,
|
||||
manager: EntityManager,
|
||||
): Promise<string> {
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||
import { MessageParticipantRepository } from 'src/modules/messaging/repositories/message-participant.repository';
|
||||
import {
|
||||
@ -31,14 +33,65 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
async saveMessagesAndCreateContacts(
|
||||
public async saveMessagesAndEmitContactCreationEventWithinTransaction(
|
||||
messagesToSave: GmailMessage[],
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
workspaceId: string,
|
||||
gmailMessageChannel: ObjectRecord<MessageChannelObjectMetadata>,
|
||||
transactionManager: EntityManager,
|
||||
) {
|
||||
const messageExternalIdsAndIdsMap =
|
||||
await this.messageService.saveMessagesWithinTransaction(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
gmailMessageChannel.id,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
const participantsWithMessageId: (ParticipantWithMessageId & {
|
||||
shouldCreateContact: boolean;
|
||||
})[] = messagesToSave.flatMap((message) => {
|
||||
const messageId = messageExternalIdsAndIdsMap.get(message.externalId);
|
||||
|
||||
return messageId
|
||||
? message.participants.map((participant) => ({
|
||||
...participant,
|
||||
messageId,
|
||||
shouldCreateContact:
|
||||
gmailMessageChannel.isContactAutoCreationEnabled &&
|
||||
message.participants.find((p) => p.role === 'from')?.handle ===
|
||||
connectedAccount.handle,
|
||||
}))
|
||||
: [];
|
||||
});
|
||||
|
||||
await this.messageParticipantRepository.saveMessageParticipants(
|
||||
participantsWithMessageId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (gmailMessageChannel.isContactAutoCreationEnabled) {
|
||||
const contactsToCreate = participantsWithMessageId.filter(
|
||||
(participant) => participant.shouldCreateContact,
|
||||
);
|
||||
|
||||
this.eventEmitter.emit(`createContacts`, {
|
||||
workspaceId,
|
||||
connectedAccountHandle: connectedAccount.handle,
|
||||
contactsToCreate,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async saveMessagesAndEmitContactCreation(
|
||||
messagesToSave: GmailMessage[],
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
workspaceId: string,
|
||||
gmailMessageChannelId: string,
|
||||
jobName?: string,
|
||||
) {
|
||||
const { dataSource: workspaceDataSource, dataSourceMetadata } =
|
||||
const { dataSource: workspaceDataSource } =
|
||||
await this.workspaceDataSourceService.connectedToWorkspaceDataSourceAndReturnMetadata(
|
||||
workspaceId,
|
||||
);
|
||||
@ -47,7 +100,6 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
|
||||
const messageExternalIdsAndIdsMap = await this.messageService.saveMessages(
|
||||
messagesToSave,
|
||||
dataSourceMetadata,
|
||||
workspaceDataSource,
|
||||
connectedAccount,
|
||||
gmailMessageChannelId,
|
||||
@ -57,7 +109,7 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} saving messages for workspace ${workspaceId} and account ${
|
||||
`Saving messages for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
} in ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -100,13 +152,12 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
gmailMessageChannel,
|
||||
workspaceId,
|
||||
connectedAccount,
|
||||
jobName,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} saving message participants for workspace ${workspaceId} and account in ${
|
||||
`Saving message participants for workspace ${workspaceId} and account in ${
|
||||
connectedAccount.id
|
||||
} ${endTime - startTime}ms`,
|
||||
);
|
||||
@ -119,7 +170,6 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
gmailMessageChannel: ObjectRecord<MessageChannelObjectMetadata>,
|
||||
workspaceId: string,
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
jobName?: string,
|
||||
) {
|
||||
try {
|
||||
await this.messageParticipantRepository.saveMessageParticipants(
|
||||
@ -140,7 +190,7 @@ export class SaveMessageAndEmitContactCreationEventService {
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`${jobName} error saving message participants for workspace ${workspaceId} and account ${connectedAccount.id}`,
|
||||
`Error saving message participants for workspace ${workspaceId} and account ${connectedAccount.id}`,
|
||||
error,
|
||||
);
|
||||
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { FeatureFlagKeys } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import {
|
||||
RelationMetadataType,
|
||||
@ -6,6 +7,7 @@ import {
|
||||
import { messageChannelStandardFieldIds } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
|
||||
import { standardObjectIds } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
|
||||
import { FieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/field-metadata.decorator';
|
||||
import { Gate } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/gate.decorator';
|
||||
import { IsNullable } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/is-nullable.decorator';
|
||||
import { IsSystem } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/is-system.decorator';
|
||||
import { ObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/object-metadata.decorator';
|
||||
@ -14,6 +16,13 @@ import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||
|
||||
export enum MessageChannelSyncStatus {
|
||||
PENDING = 'PENDING',
|
||||
ONGOING = 'ONGOING',
|
||||
SUCCEEDED = 'SUCCEEDED',
|
||||
FAILED = 'FAILED',
|
||||
}
|
||||
|
||||
@ObjectMetadata({
|
||||
standardId: standardObjectIds.messageChannel,
|
||||
namePlural: 'messageChannels',
|
||||
@ -102,4 +111,81 @@ export class MessageChannelObjectMetadata extends BaseObjectMetadata {
|
||||
})
|
||||
@IsNullable()
|
||||
messageChannelMessageAssociations: MessageChannelMessageAssociationObjectMetadata[];
|
||||
|
||||
@FieldMetadata({
|
||||
standardId: messageChannelStandardFieldIds.syncExternalId,
|
||||
type: FieldMetadataType.TEXT,
|
||||
label: 'Last sync external ID',
|
||||
description: 'Last sync external ID',
|
||||
icon: 'IconHistory',
|
||||
})
|
||||
@Gate({
|
||||
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||
})
|
||||
syncExternalId: string;
|
||||
|
||||
@FieldMetadata({
|
||||
standardId: messageChannelStandardFieldIds.syncedAt,
|
||||
type: FieldMetadataType.DATE_TIME,
|
||||
label: 'Last sync date',
|
||||
description: 'Last sync date',
|
||||
icon: 'IconHistory',
|
||||
})
|
||||
@Gate({
|
||||
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||
})
|
||||
@IsNullable()
|
||||
syncedAt: string;
|
||||
|
||||
@FieldMetadata({
|
||||
standardId: messageChannelStandardFieldIds.syncStatus,
|
||||
type: FieldMetadataType.SELECT,
|
||||
label: 'Last sync status',
|
||||
description: 'Last sync status',
|
||||
icon: 'IconHistory',
|
||||
options: [
|
||||
{
|
||||
value: MessageChannelSyncStatus.PENDING,
|
||||
label: 'Pending',
|
||||
position: 0,
|
||||
color: 'blue',
|
||||
},
|
||||
{
|
||||
value: MessageChannelSyncStatus.ONGOING,
|
||||
label: 'Ongoing',
|
||||
position: 1,
|
||||
color: 'yellow',
|
||||
},
|
||||
{
|
||||
value: MessageChannelSyncStatus.SUCCEEDED,
|
||||
label: 'Succeeded',
|
||||
position: 2,
|
||||
color: 'green',
|
||||
},
|
||||
{
|
||||
value: MessageChannelSyncStatus.FAILED,
|
||||
label: 'Failed',
|
||||
position: 3,
|
||||
color: 'red',
|
||||
},
|
||||
],
|
||||
})
|
||||
@Gate({
|
||||
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||
})
|
||||
@IsNullable()
|
||||
syncStatus: MessageChannelSyncStatus;
|
||||
|
||||
@FieldMetadata({
|
||||
standardId: messageChannelStandardFieldIds.ongoingSyncStartedAt,
|
||||
type: FieldMetadataType.DATE_TIME,
|
||||
label: 'Ongoing sync started at',
|
||||
description: 'Ongoing sync started at',
|
||||
icon: 'IconHistory',
|
||||
})
|
||||
@Gate({
|
||||
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||
})
|
||||
@IsNullable()
|
||||
ongoingSyncStartedAt: string;
|
||||
}
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
export type GmailError = {
|
||||
code: number;
|
||||
errors: {
|
||||
domain: string;
|
||||
reason: string;
|
||||
message: string;
|
||||
locationType?: string;
|
||||
location?: string;
|
||||
}[];
|
||||
message: string;
|
||||
};
|
||||
Reference in New Issue
Block a user