Refactor backend folder structure (#4505)
* Refactor backend folder structure Co-authored-by: Charles Bochet <charles@twenty.com> * fix tests * fix * move yoga hooks --------- Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Weiko
@ -0,0 +1,128 @@
|
||||
import { HttpService } from '@nestjs/axios';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { AxiosResponse } from 'axios';
|
||||
|
||||
import { BatchQueries } from 'src/modules/messaging/types/batch-queries';
|
||||
import { GmailMessageParsedResponse } from 'src/modules/messaging/types/gmail-message-parsed-response';
|
||||
|
||||
@Injectable()
|
||||
export class FetchByBatchesService {
|
||||
constructor(private readonly httpService: HttpService) {}
|
||||
|
||||
async fetchAllByBatches(
|
||||
queries: BatchQueries,
|
||||
accessToken: string,
|
||||
boundary: string,
|
||||
): Promise<AxiosResponse<any, any>[]> {
|
||||
const batchLimit = 50;
|
||||
|
||||
let batchOffset = 0;
|
||||
|
||||
let batchResponses: AxiosResponse<any, any>[] = [];
|
||||
|
||||
while (batchOffset < queries.length) {
|
||||
const batchResponse = await this.fetchBatch(
|
||||
queries,
|
||||
accessToken,
|
||||
batchOffset,
|
||||
batchLimit,
|
||||
boundary,
|
||||
);
|
||||
|
||||
batchResponses = batchResponses.concat(batchResponse);
|
||||
|
||||
batchOffset += batchLimit;
|
||||
}
|
||||
|
||||
return batchResponses;
|
||||
}
|
||||
|
||||
async fetchBatch(
|
||||
queries: BatchQueries,
|
||||
accessToken: string,
|
||||
batchOffset: number,
|
||||
batchLimit: number,
|
||||
boundary: string,
|
||||
): Promise<AxiosResponse<any, any>> {
|
||||
const limitedQueries = queries.slice(batchOffset, batchOffset + batchLimit);
|
||||
|
||||
const response = await this.httpService.axiosRef.post(
|
||||
'/',
|
||||
this.createBatchBody(limitedQueries, boundary),
|
||||
{
|
||||
headers: {
|
||||
'Content-Type': 'multipart/mixed; boundary=' + boundary,
|
||||
Authorization: 'Bearer ' + accessToken,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
createBatchBody(queries: BatchQueries, boundary: string): string {
|
||||
let batchBody: string[] = [];
|
||||
|
||||
queries.forEach(function (call) {
|
||||
const method = 'GET';
|
||||
const uri = call.uri;
|
||||
|
||||
batchBody = batchBody.concat([
|
||||
'--',
|
||||
boundary,
|
||||
'\r\n',
|
||||
'Content-Type: application/http',
|
||||
'\r\n\r\n',
|
||||
|
||||
method,
|
||||
' ',
|
||||
uri,
|
||||
'\r\n\r\n',
|
||||
]);
|
||||
});
|
||||
|
||||
return batchBody.concat(['--', boundary, '--']).join('');
|
||||
}
|
||||
|
||||
parseBatch(
|
||||
responseCollection: AxiosResponse<any, any>,
|
||||
): GmailMessageParsedResponse[] {
|
||||
const responseItems: GmailMessageParsedResponse[] = [];
|
||||
|
||||
const boundary = this.getBatchSeparator(responseCollection);
|
||||
|
||||
const responseLines: string[] = responseCollection.data.split(
|
||||
'--' + boundary,
|
||||
);
|
||||
|
||||
responseLines.forEach(function (response) {
|
||||
const startJson = response.indexOf('{');
|
||||
const endJson = response.lastIndexOf('}');
|
||||
|
||||
if (startJson < 0 || endJson < 0) return;
|
||||
|
||||
const responseJson = response.substring(startJson, endJson + 1);
|
||||
|
||||
const item = JSON.parse(responseJson);
|
||||
|
||||
responseItems.push(item);
|
||||
});
|
||||
|
||||
return responseItems;
|
||||
}
|
||||
|
||||
getBatchSeparator(responseCollection: AxiosResponse<any, any>): string {
|
||||
const headers = responseCollection.headers;
|
||||
|
||||
const contentType: string = headers['content-type'];
|
||||
|
||||
if (!contentType) return '';
|
||||
|
||||
const components = contentType.split('; ');
|
||||
|
||||
const boundary = components.find((item) => item.startsWith('boundary='));
|
||||
|
||||
return boundary?.replace('boundary=', '').trim() || '';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,157 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { AxiosResponse } from 'axios';
|
||||
import { simpleParser } from 'mailparser';
|
||||
import planer from 'planer';
|
||||
|
||||
import { GmailMessage } from 'src/modules/messaging/types/gmail-message';
|
||||
import { MessageQuery } from 'src/modules/messaging/types/message-or-thread-query';
|
||||
import { GmailMessageParsedResponse } from 'src/modules/messaging/types/gmail-message-parsed-response';
|
||||
import { FetchByBatchesService } from 'src/modules/messaging/services/fetch-by-batch.service';
|
||||
import { formatAddressObjectAsParticipants } from 'src/modules/messaging/services/utils/format-address-object-as-participants.util';
|
||||
|
||||
@Injectable()
|
||||
export class FetchMessagesByBatchesService {
|
||||
private readonly logger = new Logger(FetchMessagesByBatchesService.name);
|
||||
|
||||
constructor(private readonly fetchByBatchesService: FetchByBatchesService) {}
|
||||
|
||||
async fetchAllMessages(
|
||||
queries: MessageQuery[],
|
||||
accessToken: string,
|
||||
jobName?: string,
|
||||
workspaceId?: string,
|
||||
connectedAccountId?: string,
|
||||
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
|
||||
let startTime = Date.now();
|
||||
const batchResponses = await this.fetchByBatchesService.fetchAllByBatches(
|
||||
queries,
|
||||
accessToken,
|
||||
'batch_gmail_messages',
|
||||
);
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
|
||||
queries.length
|
||||
} messages in ${endTime - startTime}ms`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
const formattedResponse =
|
||||
await this.formatBatchResponsesAsGmailMessages(batchResponses);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
|
||||
queries.length
|
||||
} messages in ${endTime - startTime}ms`,
|
||||
);
|
||||
|
||||
return formattedResponse;
|
||||
}
|
||||
|
||||
async formatBatchResponseAsGmailMessage(
|
||||
responseCollection: AxiosResponse<any, any>,
|
||||
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
|
||||
const parsedResponses = this.fetchByBatchesService.parseBatch(
|
||||
responseCollection,
|
||||
) as GmailMessageParsedResponse[];
|
||||
|
||||
const errors: any = [];
|
||||
|
||||
const formattedResponse = Promise.all(
|
||||
parsedResponses.map(async (message: GmailMessageParsedResponse) => {
|
||||
if (message.error) {
|
||||
errors.push(message.error);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { historyId, id, threadId, internalDate, raw } = message;
|
||||
|
||||
const body = atob(raw?.replace(/-/g, '+').replace(/_/g, '/'));
|
||||
|
||||
try {
|
||||
const parsed = await simpleParser(body, {
|
||||
skipHtmlToText: true,
|
||||
skipImageLinks: true,
|
||||
skipTextToHtml: true,
|
||||
maxHtmlLengthToParse: 0,
|
||||
});
|
||||
|
||||
const { subject, messageId, from, to, cc, bcc, text, attachments } =
|
||||
parsed;
|
||||
|
||||
if (!from) throw new Error('From value is missing');
|
||||
|
||||
const participants = [
|
||||
...formatAddressObjectAsParticipants(from, 'from'),
|
||||
...formatAddressObjectAsParticipants(to, 'to'),
|
||||
...formatAddressObjectAsParticipants(cc, 'cc'),
|
||||
...formatAddressObjectAsParticipants(bcc, 'bcc'),
|
||||
];
|
||||
|
||||
let textWithoutReplyQuotations = text;
|
||||
|
||||
if (text)
|
||||
try {
|
||||
textWithoutReplyQuotations = planer.extractFrom(
|
||||
text,
|
||||
'text/plain',
|
||||
);
|
||||
} catch (error) {
|
||||
console.log(
|
||||
'Error while trying to remove reply quotations',
|
||||
error,
|
||||
);
|
||||
}
|
||||
|
||||
const messageFromGmail: GmailMessage = {
|
||||
historyId,
|
||||
externalId: id,
|
||||
headerMessageId: messageId || '',
|
||||
subject: subject || '',
|
||||
messageThreadExternalId: threadId,
|
||||
internalDate,
|
||||
fromHandle: from.value[0].address || '',
|
||||
fromDisplayName: from.value[0].name || '',
|
||||
participants,
|
||||
text: textWithoutReplyQuotations || '',
|
||||
attachments,
|
||||
};
|
||||
|
||||
return messageFromGmail;
|
||||
} catch (error) {
|
||||
console.log('Error', error);
|
||||
|
||||
errors.push(error);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const filteredMessages = (await formattedResponse).filter(
|
||||
(message) => message,
|
||||
) as GmailMessage[];
|
||||
|
||||
return { messages: filteredMessages, errors };
|
||||
}
|
||||
|
||||
async formatBatchResponsesAsGmailMessages(
|
||||
batchResponses: AxiosResponse<any, any>[],
|
||||
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
|
||||
const messagesAndErrors = await Promise.all(
|
||||
batchResponses.map(async (response) => {
|
||||
return this.formatBatchResponseAsGmailMessage(response);
|
||||
}),
|
||||
);
|
||||
|
||||
const messages = messagesAndErrors.map((item) => item.messages).flat();
|
||||
|
||||
const errors = messagesAndErrors.map((item) => item.errors).flat();
|
||||
|
||||
return { messages, errors };
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,258 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { FetchMessagesByBatchesService } from 'src/modules/messaging/services/fetch-messages-by-batches.service';
|
||||
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import {
|
||||
GmailFullSyncJobData,
|
||||
GmailFullSyncJob,
|
||||
} from 'src/modules/messaging/jobs/gmail-full-sync.job';
|
||||
import { ConnectedAccountService } from 'src/modules/connected-account/repositories/connected-account/connected-account.service';
|
||||
import { MessageChannelService } from 'src/modules/messaging/repositories/message-channel/message-channel.service';
|
||||
import { MessageChannelMessageAssociationService } from 'src/modules/messaging/repositories/message-channel-message-association/message-channel-message-association.service';
|
||||
import { createQueriesFromMessageIds } from 'src/modules/messaging/utils/create-queries-from-message-ids.util';
|
||||
import { gmailSearchFilterExcludeEmails } from 'src/modules/messaging/utils/gmail-search-filter.util';
|
||||
import { BlocklistService } from 'src/modules/connected-account/repositories/blocklist/blocklist.service';
|
||||
import { SaveMessagesAndCreateContactsService } from 'src/modules/messaging/services/save-messages-and-create-contacts.service';
|
||||
import {
|
||||
FeatureFlagEntity,
|
||||
FeatureFlagKeys,
|
||||
} from 'src/engine/modules/feature-flag/feature-flag.entity';
|
||||
|
||||
@Injectable()
|
||||
export class GmailFullSyncService {
|
||||
private readonly logger = new Logger(GmailFullSyncService.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
|
||||
@Inject(MessageQueue.messagingQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
private readonly connectedAccountService: ConnectedAccountService,
|
||||
private readonly messageChannelService: MessageChannelService,
|
||||
private readonly messageChannelMessageAssociationService: MessageChannelMessageAssociationService,
|
||||
private readonly blocklistService: BlocklistService,
|
||||
private readonly saveMessagesAndCreateContactsService: SaveMessagesAndCreateContactsService,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
|
||||
public async fetchConnectedAccountThreads(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
nextPageToken?: string,
|
||||
): Promise<void> {
|
||||
const connectedAccount = await this.connectedAccountService.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId} during full-sync`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const accessToken = connectedAccount.accessToken;
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
const workspaceMemberId = connectedAccount.accountOwnerId;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
|
||||
);
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelService.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-syn`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const gmailMessageChannelId = gmailMessageChannel.id;
|
||||
|
||||
const gmailClient =
|
||||
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||
|
||||
const isBlocklistEnabledFeatureFlag =
|
||||
await this.featureFlagRepository.findOneBy({
|
||||
workspaceId,
|
||||
key: FeatureFlagKeys.IsBlocklistEnabled,
|
||||
value: true,
|
||||
});
|
||||
|
||||
const isBlocklistEnabled =
|
||||
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
|
||||
|
||||
const blocklist = isBlocklistEnabled
|
||||
? await this.blocklistService.getByWorkspaceMemberId(
|
||||
workspaceMemberId,
|
||||
workspaceId,
|
||||
)
|
||||
: [];
|
||||
|
||||
const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
|
||||
let startTime = Date.now();
|
||||
|
||||
const messages = await gmailClient.users.messages.list({
|
||||
userId: 'me',
|
||||
maxResults: 500,
|
||||
pageToken: nextPageToken,
|
||||
q: gmailSearchFilterExcludeEmails(blocklistedEmails),
|
||||
});
|
||||
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} getting messages list in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
const messagesData = messages.data.messages;
|
||||
|
||||
const messageExternalIds = messagesData
|
||||
? messagesData.map((message) => message.id || '')
|
||||
: [];
|
||||
|
||||
if (!messageExternalIds || messageExternalIds?.length === 0) {
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
const existingMessageChannelMessageAssociations =
|
||||
await this.messageChannelMessageAssociationService.getByMessageExternalIdsAndMessageChannelId(
|
||||
messageExternalIds,
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId}: getting existing message channel message associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
const existingMessageChannelMessageAssociationsExternalIds =
|
||||
existingMessageChannelMessageAssociations.map(
|
||||
(messageChannelMessageAssociation) =>
|
||||
messageChannelMessageAssociation.messageExternalId,
|
||||
);
|
||||
|
||||
const messagesToFetch = messageExternalIds.filter(
|
||||
(messageExternalId) =>
|
||||
!existingMessageChannelMessageAssociationsExternalIds.includes(
|
||||
messageExternalId,
|
||||
),
|
||||
);
|
||||
|
||||
const messageQueries = createQueriesFromMessageIds(messagesToFetch);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
const { messages: messagesToSave, errors } =
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
'gmail full-sync',
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId}: fetching all messages in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
if (messagesToSave.length > 0) {
|
||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannelId,
|
||||
'gmail full-sync',
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
|
||||
);
|
||||
}
|
||||
const lastModifiedMessageId = messagesToFetch[0];
|
||||
|
||||
const historyId = messagesToSave.find(
|
||||
(message) => message.externalId === lastModifiedMessageId,
|
||||
)?.historyId;
|
||||
|
||||
if (!historyId) {
|
||||
throw new Error(
|
||||
`No historyId found for ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
|
||||
);
|
||||
}
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.connectedAccountService.updateLastSyncHistoryIdIfHigher(
|
||||
historyId,
|
||||
connectedAccount.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating last sync history id in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} ${
|
||||
nextPageToken ? `and ${nextPageToken} pageToken` : ''
|
||||
}done.`,
|
||||
);
|
||||
|
||||
if (messages.data.nextPageToken) {
|
||||
await this.messageQueueService.add<GmailFullSyncJobData>(
|
||||
GmailFullSyncJob.name,
|
||||
{
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
nextPageToken: messages.data.nextPageToken,
|
||||
},
|
||||
{
|
||||
retryLimit: 2,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,427 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { gmail_v1 } from 'googleapis';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { FetchMessagesByBatchesService } from 'src/modules/messaging/services/fetch-messages-by-batches.service';
|
||||
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import {
|
||||
GmailFullSyncJob,
|
||||
GmailFullSyncJobData,
|
||||
} from 'src/modules/messaging/jobs/gmail-full-sync.job';
|
||||
import { ConnectedAccountService } from 'src/modules/connected-account/repositories/connected-account/connected-account.service';
|
||||
import { MessageChannelService } from 'src/modules/messaging/repositories/message-channel/message-channel.service';
|
||||
import { MessageService } from 'src/modules/messaging/repositories/message/message.service';
|
||||
import { createQueriesFromMessageIds } from 'src/modules/messaging/utils/create-queries-from-message-ids.util';
|
||||
import { GmailMessage } from 'src/modules/messaging/types/gmail-message';
|
||||
import { isPersonEmail } from 'src/modules/messaging/utils/is-person-email.util';
|
||||
import { BlocklistService } from 'src/modules/connected-account/repositories/blocklist/blocklist.service';
|
||||
import { SaveMessagesAndCreateContactsService } from 'src/modules/messaging/services/save-messages-and-create-contacts.service';
|
||||
import {
|
||||
FeatureFlagEntity,
|
||||
FeatureFlagKeys,
|
||||
} from 'src/engine/modules/feature-flag/feature-flag.entity';
|
||||
|
||||
@Injectable()
|
||||
export class GmailPartialSyncService {
|
||||
private readonly logger = new Logger(GmailPartialSyncService.name);
|
||||
|
||||
constructor(
|
||||
private readonly gmailClientProvider: GmailClientProvider,
|
||||
private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
|
||||
@Inject(MessageQueue.messagingQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
private readonly connectedAccountService: ConnectedAccountService,
|
||||
private readonly messageChannelService: MessageChannelService,
|
||||
private readonly messageService: MessageService,
|
||||
private readonly blocklistService: BlocklistService,
|
||||
private readonly saveMessagesAndCreateContactsService: SaveMessagesAndCreateContactsService,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
|
||||
public async fetchConnectedAccountThreads(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
maxResults = 500,
|
||||
): Promise<void> {
|
||||
const connectedAccount = await this.connectedAccountService.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
this.logger.error(
|
||||
`Connected account ${connectedAccountId} not found in workspace ${workspaceId} during partial-sync`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const lastSyncHistoryId = connectedAccount.lastSyncHistoryId;
|
||||
|
||||
if (!lastSyncHistoryId) {
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId}: no lastSyncHistoryId, falling back to full sync.`,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const accessToken = connectedAccount.accessToken;
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during partial-sync`,
|
||||
);
|
||||
}
|
||||
|
||||
let startTime = Date.now();
|
||||
|
||||
const { history, historyId, error } = await this.getHistoryFromGmail(
|
||||
refreshToken,
|
||||
lastSyncHistoryId,
|
||||
maxResults,
|
||||
);
|
||||
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId} getting history in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
if (error && error.code === 404) {
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId}: invalid lastSyncHistoryId, falling back to full sync.`,
|
||||
);
|
||||
|
||||
await this.connectedAccountService.deleteHistoryId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (error && error.code === 429) {
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId}: Error 429: ${error.message}, partial sync will be retried later.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
throw new Error(
|
||||
`Error getting history for ${connectedAccountId} in workspace ${workspaceId} during partial-sync:
|
||||
${JSON.stringify(error)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!historyId) {
|
||||
throw new Error(
|
||||
`No historyId found for ${connectedAccountId} in workspace ${workspaceId} during partial-sync`,
|
||||
);
|
||||
}
|
||||
|
||||
if (historyId === lastSyncHistoryId || !history?.length) {
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to update.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelService.getFirstByConnectedAccountId(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId} during partial-sync`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const gmailMessageChannelId = gmailMessageChannel.id;
|
||||
|
||||
const { messagesAdded, messagesDeleted } =
|
||||
await this.getMessageIdsFromHistory(history);
|
||||
|
||||
const messageQueries = createQueriesFromMessageIds(messagesAdded);
|
||||
|
||||
const { messages, errors } =
|
||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||
messageQueries,
|
||||
accessToken,
|
||||
'gmail partial-sync',
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
);
|
||||
|
||||
const isBlocklistEnabledFeatureFlag =
|
||||
await this.featureFlagRepository.findOneBy({
|
||||
workspaceId,
|
||||
key: FeatureFlagKeys.IsBlocklistEnabled,
|
||||
value: true,
|
||||
});
|
||||
|
||||
const isBlocklistEnabled =
|
||||
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
|
||||
|
||||
const blocklist = isBlocklistEnabled
|
||||
? await this.blocklistService.getByWorkspaceMemberId(
|
||||
connectedAccount.accountOwnerId,
|
||||
workspaceId,
|
||||
)
|
||||
: [];
|
||||
|
||||
const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
|
||||
|
||||
const messagesToSave = messages.filter(
|
||||
(message) =>
|
||||
!this.shouldSkipImport(
|
||||
connectedAccount.handle,
|
||||
message,
|
||||
blocklistedEmails,
|
||||
),
|
||||
);
|
||||
|
||||
if (messagesToSave.length !== 0) {
|
||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
||||
messagesToSave,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
gmailMessageChannelId,
|
||||
'gmail partial-sync',
|
||||
);
|
||||
}
|
||||
|
||||
if (messagesDeleted.length !== 0) {
|
||||
startTime = Date.now();
|
||||
|
||||
await this.messageService.deleteMessages(
|
||||
messagesDeleted,
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId}: deleting messages in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
this.logger.error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId} during partial-sync: ${JSON.stringify(
|
||||
errors,
|
||||
null,
|
||||
2,
|
||||
)}`,
|
||||
);
|
||||
const errorsCanBeIgnored = errors.every((error) => error.code === 404);
|
||||
const errorsShouldBeRetried = errors.some((error) => error.code === 429);
|
||||
|
||||
if (errorsShouldBeRetried) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!errorsCanBeIgnored) {
|
||||
throw new Error(
|
||||
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId} during partial-sync`,
|
||||
);
|
||||
}
|
||||
}
|
||||
startTime = Date.now();
|
||||
|
||||
await this.connectedAccountService.updateLastSyncHistoryId(
|
||||
historyId,
|
||||
connectedAccount.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId} updating lastSyncHistoryId in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`gmail partial-sync for workspace ${workspaceId} and account ${connectedAccountId} done.`,
|
||||
);
|
||||
}
|
||||
|
||||
private async getMessageIdsFromHistory(
|
||||
history: gmail_v1.Schema$History[],
|
||||
): Promise<{
|
||||
messagesAdded: string[];
|
||||
messagesDeleted: string[];
|
||||
}> {
|
||||
const { messagesAdded, messagesDeleted } = history.reduce(
|
||||
(
|
||||
acc: {
|
||||
messagesAdded: string[];
|
||||
messagesDeleted: string[];
|
||||
},
|
||||
history,
|
||||
) => {
|
||||
const messagesAdded = history.messagesAdded?.map(
|
||||
(messageAdded) => messageAdded.message?.id || '',
|
||||
);
|
||||
|
||||
const messagesDeleted = history.messagesDeleted?.map(
|
||||
(messageDeleted) => messageDeleted.message?.id || '',
|
||||
);
|
||||
|
||||
if (messagesAdded) acc.messagesAdded.push(...messagesAdded);
|
||||
if (messagesDeleted) acc.messagesDeleted.push(...messagesDeleted);
|
||||
|
||||
return acc;
|
||||
},
|
||||
{ messagesAdded: [], messagesDeleted: [] },
|
||||
);
|
||||
|
||||
const uniqueMessagesAdded = messagesAdded.filter(
|
||||
(messageId) => !messagesDeleted.includes(messageId),
|
||||
);
|
||||
|
||||
const uniqueMessagesDeleted = messagesDeleted.filter(
|
||||
(messageId) => !messagesAdded.includes(messageId),
|
||||
);
|
||||
|
||||
return {
|
||||
messagesAdded: uniqueMessagesAdded,
|
||||
messagesDeleted: uniqueMessagesDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
private async getHistoryFromGmail(
|
||||
refreshToken: string,
|
||||
lastSyncHistoryId: string,
|
||||
maxResults: number,
|
||||
): Promise<{
|
||||
history: gmail_v1.Schema$History[];
|
||||
historyId?: string | null;
|
||||
error?: {
|
||||
code: number;
|
||||
errors: {
|
||||
domain: string;
|
||||
reason: string;
|
||||
message: string;
|
||||
locationType?: string;
|
||||
location?: string;
|
||||
}[];
|
||||
message: string;
|
||||
};
|
||||
}> {
|
||||
const gmailClient =
|
||||
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||
|
||||
const fullHistory: gmail_v1.Schema$History[] = [];
|
||||
|
||||
try {
|
||||
const history = await gmailClient.users.history.list({
|
||||
userId: 'me',
|
||||
startHistoryId: lastSyncHistoryId,
|
||||
historyTypes: ['messageAdded', 'messageDeleted'],
|
||||
maxResults,
|
||||
});
|
||||
|
||||
let nextPageToken = history?.data?.nextPageToken;
|
||||
|
||||
const historyId = history?.data?.historyId;
|
||||
|
||||
if (history?.data?.history) {
|
||||
fullHistory.push(...history.data.history);
|
||||
}
|
||||
|
||||
while (nextPageToken) {
|
||||
const nextHistory = await gmailClient.users.history.list({
|
||||
userId: 'me',
|
||||
startHistoryId: lastSyncHistoryId,
|
||||
historyTypes: ['messageAdded', 'messageDeleted'],
|
||||
maxResults,
|
||||
pageToken: nextPageToken,
|
||||
});
|
||||
|
||||
nextPageToken = nextHistory?.data?.nextPageToken;
|
||||
|
||||
if (nextHistory?.data?.history) {
|
||||
fullHistory.push(...nextHistory.data.history);
|
||||
}
|
||||
}
|
||||
|
||||
return { history: fullHistory, historyId };
|
||||
} catch (error) {
|
||||
const errorData = error?.response?.data?.error;
|
||||
|
||||
if (errorData) {
|
||||
return { history: [], error: errorData };
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async fallbackToFullSync(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
) {
|
||||
await this.messageQueueService.add<GmailFullSyncJobData>(
|
||||
GmailFullSyncJob.name,
|
||||
{ workspaceId, connectedAccountId },
|
||||
{
|
||||
retryLimit: 2,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private isHandleBlocked = (
|
||||
selfHandle: string,
|
||||
message: GmailMessage,
|
||||
blocklistedEmails: string[],
|
||||
): boolean => {
|
||||
// If the message is received, check if the sender is in the blocklist
|
||||
// If the message is sent, check if any of the recipients with role 'to' is in the blocklist
|
||||
|
||||
if (message.fromHandle === selfHandle) {
|
||||
return message.participants.some(
|
||||
(participant) =>
|
||||
participant.role === 'to' &&
|
||||
blocklistedEmails.includes(participant.handle),
|
||||
);
|
||||
}
|
||||
|
||||
return blocklistedEmails.includes(message.fromHandle);
|
||||
};
|
||||
|
||||
private shouldSkipImport(
|
||||
selfHandle: string,
|
||||
message: GmailMessage,
|
||||
blocklistedEmails: string[],
|
||||
): boolean {
|
||||
return (
|
||||
!isPersonEmail(message.fromHandle) ||
|
||||
this.isHandleBlocked(selfHandle, message, blocklistedEmails)
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { OAuth2Client } from 'google-auth-library';
|
||||
import { gmail_v1, google } from 'googleapis';
|
||||
|
||||
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
|
||||
|
||||
@Injectable()
|
||||
export class GmailClientProvider {
|
||||
constructor(private readonly environmentService: EnvironmentService) {}
|
||||
|
||||
public async getGmailClient(refreshToken: string): Promise<gmail_v1.Gmail> {
|
||||
const oAuth2Client = await this.getOAuth2Client(refreshToken);
|
||||
|
||||
const gmailClient = google.gmail({
|
||||
version: 'v1',
|
||||
auth: oAuth2Client,
|
||||
});
|
||||
|
||||
return gmailClient;
|
||||
}
|
||||
|
||||
private async getOAuth2Client(refreshToken: string): Promise<OAuth2Client> {
|
||||
const gmailClientId = this.environmentService.get('AUTH_GOOGLE_CLIENT_ID');
|
||||
const gmailClientSecret = this.environmentService.get(
|
||||
'AUTH_GOOGLE_CLIENT_SECRET',
|
||||
);
|
||||
|
||||
const oAuth2Client = new google.auth.OAuth2(
|
||||
gmailClientId,
|
||||
gmailClientSecret,
|
||||
);
|
||||
|
||||
oAuth2Client.setCredentials({
|
||||
refresh_token: refreshToken,
|
||||
});
|
||||
|
||||
return oAuth2Client;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
|
||||
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||
|
||||
@Module({
|
||||
imports: [EnvironmentModule],
|
||||
providers: [GmailClientProvider],
|
||||
exports: [GmailClientProvider],
|
||||
})
|
||||
export class MessagingProvidersModule {}
|
||||
@ -0,0 +1,181 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { MessageChannelService } from 'src/modules/messaging/repositories/message-channel/message-channel.service';
|
||||
import { MessageParticipantService } from 'src/modules/messaging/repositories/message-participant/message-participant.service';
|
||||
import { MessageService } from 'src/modules/messaging/repositories/message/message.service';
|
||||
import { CreateCompanyAndContactService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.service';
|
||||
import {
|
||||
GmailMessage,
|
||||
ParticipantWithMessageId,
|
||||
} from 'src/modules/messaging/types/gmail-message';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
||||
|
||||
@Injectable()
|
||||
export class SaveMessagesAndCreateContactsService {
|
||||
private readonly logger = new Logger(
|
||||
SaveMessagesAndCreateContactsService.name,
|
||||
);
|
||||
|
||||
constructor(
|
||||
private readonly messageService: MessageService,
|
||||
private readonly messageChannelService: MessageChannelService,
|
||||
private readonly createCompaniesAndContactsService: CreateCompanyAndContactService,
|
||||
private readonly messageParticipantService: MessageParticipantService,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
) {}
|
||||
|
||||
async saveMessagesAndCreateContacts(
|
||||
messagesToSave: GmailMessage[],
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
workspaceId: string,
|
||||
gmailMessageChannelId: string,
|
||||
jobName?: string,
|
||||
) {
|
||||
const { dataSource: workspaceDataSource, dataSourceMetadata } =
|
||||
await this.workspaceDataSourceService.connectedToWorkspaceDataSourceAndReturnMetadata(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
let startTime = Date.now();
|
||||
|
||||
const messageExternalIdsAndIdsMap = await this.messageService.saveMessages(
|
||||
messagesToSave,
|
||||
dataSourceMetadata,
|
||||
workspaceDataSource,
|
||||
connectedAccount,
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} saving messages for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
} in ${endTime - startTime}ms`,
|
||||
);
|
||||
|
||||
const gmailMessageChannel =
|
||||
await this.messageChannelService.getFirstByConnectedAccountId(
|
||||
connectedAccount.id,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!gmailMessageChannel) {
|
||||
this.logger.error(
|
||||
`No message channel found for connected account ${connectedAccount.id} in workspace ${workspaceId} in saveMessagesAndCreateContacts`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const isContactAutoCreationEnabled =
|
||||
gmailMessageChannel.isContactAutoCreationEnabled;
|
||||
|
||||
const participantsWithMessageId: ParticipantWithMessageId[] =
|
||||
messagesToSave.flatMap((message) => {
|
||||
const messageId = messageExternalIdsAndIdsMap.get(message.externalId);
|
||||
|
||||
return messageId
|
||||
? message.participants.map((participant) => ({
|
||||
...participant,
|
||||
messageId,
|
||||
}))
|
||||
: [];
|
||||
});
|
||||
|
||||
const contactsToCreate = messagesToSave
|
||||
.filter((message) => connectedAccount.handle === message.fromHandle)
|
||||
.flatMap((message) => message.participants);
|
||||
|
||||
if (isContactAutoCreationEnabled) {
|
||||
startTime = Date.now();
|
||||
|
||||
await workspaceDataSource?.transaction(
|
||||
async (transactionManager: EntityManager) => {
|
||||
await this.createCompaniesAndContactsService.createCompaniesAndContacts(
|
||||
connectedAccount.handle,
|
||||
contactsToCreate,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
const handles = participantsWithMessageId.map(
|
||||
(participant) => participant.handle,
|
||||
);
|
||||
|
||||
const messageParticipantsWithoutPersonIdAndWorkspaceMemberId =
|
||||
await this.messageParticipantService.getByHandlesWithoutPersonIdAndWorkspaceMemberId(
|
||||
handles,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation(
|
||||
messageParticipantsWithoutPersonIdAndWorkspaceMemberId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} creating companies and contacts for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
} in ${endTime - startTime}ms`,
|
||||
);
|
||||
}
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.tryToSaveMessageParticipantsOrDeleteMessagesIfError(
|
||||
participantsWithMessageId,
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
connectedAccount,
|
||||
jobName,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`${jobName} saving message participants for workspace ${workspaceId} and account in ${
|
||||
connectedAccount.id
|
||||
} ${endTime - startTime}ms`,
|
||||
);
|
||||
}
|
||||
|
||||
private async tryToSaveMessageParticipantsOrDeleteMessagesIfError(
|
||||
participantsWithMessageId: ParticipantWithMessageId[],
|
||||
gmailMessageChannelId: string,
|
||||
workspaceId: string,
|
||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||
jobName?: string,
|
||||
) {
|
||||
try {
|
||||
await this.messageParticipantService.saveMessageParticipants(
|
||||
participantsWithMessageId,
|
||||
workspaceId,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`${jobName} error saving message participants for workspace ${workspaceId} and account ${connectedAccount.id}`,
|
||||
error,
|
||||
);
|
||||
|
||||
const messagesToDelete = participantsWithMessageId.map(
|
||||
(participant) => participant.messageId,
|
||||
);
|
||||
|
||||
await this.messageService.deleteMessages(
|
||||
messagesToDelete,
|
||||
gmailMessageChannelId,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
|
||||
import { DataSourceModule } from 'src/engine-metadata/data-source/data-source.module';
|
||||
import { MessageThreadModule } from 'src/modules/messaging/repositories/message-thread/message-thread.module';
|
||||
import { MessageModule } from 'src/modules/messaging/repositories/message/message.module';
|
||||
import { ThreadCleanerService } from 'src/modules/messaging/services/thread-cleaner/thread-cleaner.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
DataSourceModule,
|
||||
TypeORMModule,
|
||||
MessageThreadModule,
|
||||
MessageModule,
|
||||
],
|
||||
providers: [ThreadCleanerService],
|
||||
exports: [ThreadCleanerService],
|
||||
})
|
||||
export class ThreadCleanerModule {}
|
||||
@ -0,0 +1,37 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
|
||||
import { DataSourceService } from 'src/engine-metadata/data-source/data-source.service';
|
||||
import { MessageThreadService } from 'src/modules/messaging/repositories/message-thread/message-thread.service';
|
||||
import { MessageService } from 'src/modules/messaging/repositories/message/message.service';
|
||||
import { deleteUsingPagination } from 'src/modules/messaging/services/thread-cleaner/utils/delete-using-pagination.util';
|
||||
|
||||
@Injectable()
|
||||
export class ThreadCleanerService {
|
||||
constructor(
|
||||
private readonly dataSourceService: DataSourceService,
|
||||
private readonly typeORMService: TypeORMService,
|
||||
private readonly messageService: MessageService,
|
||||
private readonly messageThreadService: MessageThreadService,
|
||||
) {}
|
||||
|
||||
public async cleanWorkspaceThreads(workspaceId: string) {
|
||||
await deleteUsingPagination(
|
||||
workspaceId,
|
||||
500,
|
||||
this.messageService.getNonAssociatedMessageIdsPaginated.bind(
|
||||
this.messageService,
|
||||
),
|
||||
this.messageService.deleteByIds.bind(this.messageService),
|
||||
);
|
||||
|
||||
await deleteUsingPagination(
|
||||
workspaceId,
|
||||
500,
|
||||
this.messageThreadService.getOrphanThreadIdsPaginated.bind(
|
||||
this.messageThreadService,
|
||||
),
|
||||
this.messageThreadService.deleteByIds.bind(this.messageThreadService),
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
import { deleteUsingPagination } from './delete-using-pagination.util';
|
||||
|
||||
describe('deleteUsingPagination', () => {
|
||||
it('should delete items using pagination', async () => {
|
||||
const workspaceId = 'workspace123';
|
||||
const batchSize = 10;
|
||||
const getterPaginated = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(['id1', 'id2'])
|
||||
.mockResolvedValueOnce([]);
|
||||
const deleter = jest.fn();
|
||||
const transactionManager = undefined;
|
||||
|
||||
await deleteUsingPagination(
|
||||
workspaceId,
|
||||
batchSize,
|
||||
getterPaginated,
|
||||
deleter,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
expect(getterPaginated).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
batchSize,
|
||||
0,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
expect(getterPaginated).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
batchSize,
|
||||
batchSize,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
expect(deleter).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
['id1', 'id2'],
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
expect(deleter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,38 @@
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
export const deleteUsingPagination = async (
|
||||
workspaceId: string,
|
||||
batchSize: number,
|
||||
getterPaginated: (
|
||||
limit: number,
|
||||
offset: number,
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) => Promise<string[]>,
|
||||
deleter: (
|
||||
ids: string[],
|
||||
workspaceId: string,
|
||||
transactionManager?: EntityManager,
|
||||
) => Promise<void>,
|
||||
transactionManager?: EntityManager,
|
||||
) => {
|
||||
let offset = 0;
|
||||
let hasMoreData = true;
|
||||
|
||||
while (hasMoreData) {
|
||||
const idsToDelete = await getterPaginated(
|
||||
batchSize,
|
||||
offset,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
if (idsToDelete.length > 0) {
|
||||
await deleter(idsToDelete, workspaceId, transactionManager);
|
||||
} else {
|
||||
hasMoreData = false;
|
||||
}
|
||||
|
||||
offset += batchSize;
|
||||
}
|
||||
};
|
||||
@ -0,0 +1,37 @@
|
||||
import { formatAddressObjectAsParticipants } from 'src/modules/messaging/services/utils/format-address-object-as-participants.util';
|
||||
|
||||
describe('formatAddressObjectAsParticipants', () => {
|
||||
it('should format address object as participants', () => {
|
||||
const addressObject = {
|
||||
value: [
|
||||
{ name: 'John Doe', address: 'john.doe @example.com' },
|
||||
{ name: 'Jane Smith', address: 'jane.smith@example.com ' },
|
||||
],
|
||||
html: '',
|
||||
text: '',
|
||||
};
|
||||
|
||||
const result = formatAddressObjectAsParticipants(addressObject, 'from');
|
||||
|
||||
expect(result).toEqual([
|
||||
{
|
||||
role: 'from',
|
||||
handle: 'john.doe@example.com',
|
||||
displayName: 'John Doe',
|
||||
},
|
||||
{
|
||||
role: 'from',
|
||||
handle: 'jane.smith@example.com',
|
||||
displayName: 'Jane Smith',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return an empty array if address object is undefined', () => {
|
||||
const addressObject = undefined;
|
||||
|
||||
const result = formatAddressObjectAsParticipants(addressObject, 'to');
|
||||
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,37 @@
|
||||
import { AddressObject } from 'mailparser';
|
||||
|
||||
import { Participant } from 'src/modules/messaging/types/gmail-message';
|
||||
|
||||
const formatAddressObjectAsArray = (
|
||||
addressObject: AddressObject | AddressObject[],
|
||||
): AddressObject[] => {
|
||||
return Array.isArray(addressObject) ? addressObject : [addressObject];
|
||||
};
|
||||
|
||||
const removeSpacesAndLowerCase = (email: string): string => {
|
||||
return email.replace(/\s/g, '').toLowerCase();
|
||||
};
|
||||
|
||||
export const formatAddressObjectAsParticipants = (
|
||||
addressObject: AddressObject | AddressObject[] | undefined,
|
||||
role: 'from' | 'to' | 'cc' | 'bcc',
|
||||
): Participant[] => {
|
||||
if (!addressObject) return [];
|
||||
const addressObjects = formatAddressObjectAsArray(addressObject);
|
||||
|
||||
const participants = addressObjects.map((addressObject) => {
|
||||
const emailAdresses = addressObject.value;
|
||||
|
||||
return emailAdresses.map((emailAddress) => {
|
||||
const { name, address } = emailAddress;
|
||||
|
||||
return {
|
||||
role,
|
||||
handle: address ? removeSpacesAndLowerCase(address) : '',
|
||||
displayName: name || '',
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
return participants.flat();
|
||||
};
|
||||
Reference in New Issue
Block a user