mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor sync sub status and throttle (#5734)

Browse Source 
- Rename syncSubStatus to syncStage
- Rename ongoingSyncStartedAt to syncStageStartedAt
- Remove throttlePauseUntil from db and compute it with
syncStageStartedAt and throttleFailureCount
This commit is contained in:
bosiraphael
2024-06-04 16:52:57 +02:00
committed by GitHub
parent ce1469cf0c
commit 234e062232
14 changed files with 148 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts
View File

@ -7,7 +7,7 @@ import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repos
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
import {
MessageChannelWorkspaceEntity,
MessageChannelSyncSubStatus,
MessageChannelSyncStage,
MessageChannelSyncStatus,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
@ -24,9 +24,9 @@ export class MessagingChannelSyncStatusService {
messageChannelId: string,
workspaceId: string,
) {
await this.messageChannelRepository.updateSyncSubStatus(
await this.messageChannelRepository.updateSyncStage(
messageChannelId,
MessageChannelSyncSubStatus.FULL_MESSAGE_LIST_FETCH_PENDING,
MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
workspaceId,
);
}
@ -35,9 +35,9 @@ export class MessagingChannelSyncStatusService {
messageChannelId: string,
workspaceId: string,
) {
await this.messageChannelRepository.updateSyncSubStatus(
await this.messageChannelRepository.updateSyncStage(
messageChannelId,
MessageChannelSyncSubStatus.PARTIAL_MESSAGE_LIST_FETCH_PENDING,
MessageChannelSyncStage.PARTIAL_MESSAGE_LIST_FETCH_PENDING,
workspaceId,
);
}
@ -46,9 +46,9 @@ export class MessagingChannelSyncStatusService {
messageChannelId: string,
workspaceId: string,
) {
await this.messageChannelRepository.updateSyncSubStatus(
await this.messageChannelRepository.updateSyncStage(
messageChannelId,
MessageChannelSyncSubStatus.MESSAGES_IMPORT_PENDING,
MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
workspaceId,
);
}
@ -68,6 +68,16 @@ export class MessagingChannelSyncStatusService {
workspaceId,
);
await this.messageChannelRepository.resetSyncStageStartedAt(
messageChannelId,
workspaceId,
);
await this.messageChannelRepository.resetThrottleFailureCount(
messageChannelId,
workspaceId,
);
await this.scheduleFullMessageListFetch(messageChannelId, workspaceId);
}
@ -75,9 +85,9 @@ export class MessagingChannelSyncStatusService {
messageChannelId: string,
workspaceId: string,
) {
await this.messageChannelRepository.updateSyncSubStatus(
await this.messageChannelRepository.updateSyncStage(
messageChannelId,
MessageChannelSyncSubStatus.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
workspaceId,
);
@ -105,9 +115,9 @@ export class MessagingChannelSyncStatusService {
messageChannelId: string,
workspaceId: string,
) {
await this.messageChannelRepository.updateSyncSubStatus(
await this.messageChannelRepository.updateSyncStage(
messageChannelId,
MessageChannelSyncSubStatus.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
workspaceId,
);
}
@ -120,9 +130,9 @@ export class MessagingChannelSyncStatusService {
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
);
await this.messageChannelRepository.updateSyncSubStatus(
await this.messageChannelRepository.updateSyncStage(
messageChannelId,
MessageChannelSyncSubStatus.FAILED,
MessageChannelSyncStage.FAILED,
workspaceId,
);
@ -141,9 +151,9 @@ export class MessagingChannelSyncStatusService {
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
);
await this.messageChannelRepository.updateSyncSubStatus(
await this.messageChannelRepository.updateSyncStage(
messageChannelId,
MessageChannelSyncSubStatus.FAILED,
MessageChannelSyncStage.FAILED,
workspaceId,
);

10
packages/twenty-server/src/modules/messaging/common/services/messaging-error-handling.service.ts
View File

@ -10,7 +10,6 @@ import { MessagingTelemetryService } from 'src/modules/messaging/common/services
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
import { MESSAGING_THROTTLE_DURATION } from 'src/modules/messaging/common/constants/messaging-throttle-duration';
import { MESSAGING_THROTTLE_MAX_ATTEMPTS } from 'src/modules/messaging/common/constants/messaging-throttle-max-attempts';
type SyncStep =
@ -212,13 +211,8 @@ export class MessagingErrorHandlingService {
messageChannel: ObjectRecord<MessageChannelWorkspaceEntity>,
workspaceId: string,
): Promise<void> {
const throttleDuration =
MESSAGING_THROTTLE_DURATION *
Math.pow(2, messageChannel.throttleFailureCount);
await this.messageChannelRepository.updateThrottlePauseUntilAndIncrementThrottleFailureCount(
await this.messageChannelRepository.incrementThrottleFailureCount(
messageChannel.id,
throttleDuration,
workspaceId,
);
@ -227,7 +221,7 @@ export class MessagingErrorHandlingService {
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
message: `Throttling for ${throttleDuration}ms`,
message: `Increment throttle failure count to ${messageChannel.throttleFailureCount}`,
});
}
}