admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

5620 implement throttle logic for message and calendar sync (#5718)

Browse Source 
Closes #5620 and improve messages filters
This commit is contained in:
bosiraphael
2024-06-04 10:29:05 +02:00
committed by GitHub
parent 32d4b37d37
commit 3f9f2c3ba6
13 changed files with 120 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts
View File

@ -2,4 +2,5 @@ export const MESSAGING_GMAIL_EXCLUDED_CATEGORIES = [
'promotions',
'social',
'forums',
'updates',
];

5
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-full-message-list-fetch.service.ts
View File

@ -77,6 +77,11 @@ export class MessagingGmailFullMessageListFetchService {
return;
}
await this.messageChannelRepository.resetThrottlePauseUntilAndThrottleFailureCount(
messageChannel.id,
workspaceId,
);
await this.messagingChannelSyncStatusService.scheduleMessagesImport(
messageChannel.id,
workspaceId,

8
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts
View File

@ -21,6 +21,7 @@ import { MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messa
import { MessagingGmailFetchMessagesByBatchesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-fetch-messages-by-batches.service';
import { MessagingErrorHandlingService } from 'src/modules/messaging/common/services/messaging-error-handling.service';
import { MessagingSaveMessagesAndEnqueueContactCreationService } from 'src/modules/messaging/common/services/messaging-save-messages-and-enqueue-contact-creation.service';
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
@Injectable()
export class MessagingGmailMessagesImportService {
@ -39,6 +40,8 @@ export class MessagingGmailMessagesImportService {
private readonly messagingTelemetryService: MessagingTelemetryService,
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
@InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
private readonly messageChannelRepository: MessageChannelRepository,
) {}
async processMessageBatchImport(
@ -134,6 +137,11 @@ export class MessagingGmailMessagesImportService {
);
}
await this.messageChannelRepository.resetThrottlePauseUntilAndThrottleFailureCount(
messageChannel.id,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,

5
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-partial-message-list-fetch.service.ts
View File

@ -74,6 +74,11 @@ export class MessagingGmailPartialMessageListFetchService {
return;
}
await this.messageChannelRepository.resetThrottlePauseUntilAndThrottleFailureCount(
messageChannel.id,
workspaceId,
);
if (!historyId) {
throw new Error(
`No historyId found for ${connectedAccount.id} in workspace ${workspaceId} in gmail history response.`,

9
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts
View File

@ -75,6 +75,13 @@ export class MessagingMessageListFetchJob
return;
}
if (
messageChannel.throttlePauseUntil &&
messageChannel.throttlePauseUntil > new Date()
) {
return;
}
switch (messageChannel.syncSubStatus) {
case MessageChannelSyncSubStatus.PARTIAL_MESSAGE_LIST_FETCH_PENDING:
this.logger.log(
@ -85,6 +92,7 @@ export class MessagingMessageListFetchJob
eventName: 'partial_message_list_fetch.started',
workspaceId,
connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.gmailPartialMessageListFetchV2Service.processMessageListFetch(
@ -111,6 +119,7 @@ export class MessagingMessageListFetchJob
eventName: 'full_message_list_fetch.started',
workspaceId,
connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.gmailFullMessageListFetchService.processMessageListFetch(

7
packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts
View File

@ -45,6 +45,13 @@ export class MessagingMessagesImportJob
messageChannelId: messageChannel.id,
});
if (
messageChannel.throttlePauseUntil &&
messageChannel.throttlePauseUntil > new Date()
) {
continue;
}
const connectedAccount =
await this.connectedAccountRepository.getConnectedAccountOrThrow(
workspaceId,

2
packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts
View File

@ -38,7 +38,7 @@ const filterOutIcsAttachments = (messages: GmailMessage[]) => {
const isPersonEmail = (email: string): boolean => {
const nonPersonalPattern =
/noreply|no-reply|do_not_reply|no\.reply|^(info@|contact@|hello@|support@|feedback@|service@|help@|invites@|invite@|welcome@|alerts@|team@)/;
/noreply|no-reply|do_not_reply|no\.reply|^(info@|contact@|hello@|support@|feedback@|service@|help@|invites@|invite@|welcome@|alerts@|team@|notifications@|notification@|news@)/;
return !nonPersonalPattern.test(email);
};