admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

5620 implement throttle logic for message and calendar sync (#5718)

Browse Source 
Closes #5620 and improve messages filters
This commit is contained in:
bosiraphael
2024-06-04 10:29:05 +02:00
committed by GitHub
parent 32d4b37d37
commit 3f9f2c3ba6
13 changed files with 120 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/twenty-server/src/modules/messaging/common/constants/messaging-throttle-duration.ts Normal file
View File

@ -0,0 +1 @@
export const MESSAGING_THROTTLE_DURATION = 1000 * 60 * 1; // 1 minute

1
packages/twenty-server/src/modules/messaging/common/constants/messaging-throttle-max-attempts.ts Normal file
View File

@ -0,0 +1 @@
export const MESSAGING_THROTTLE_MAX_ATTEMPTS = 4;

35
packages/twenty-server/src/modules/messaging/common/repositories/message-channel.repository.ts
View File

@ -240,4 +240,39 @@ export class MessageChannelRepository {
transactionManager,
);
}
public async updateThrottlePauseUntilAndIncrementThrottleFailureCount(
id: string,
throttleDurationMs: number,
workspaceId: string,
transactionManager?: EntityManager,
) {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`UPDATE ${dataSourceSchema}."messageChannel" SET "throttlePauseUntil" = NOW() + ($1 || ' milliseconds')::interval, "throttleFailureCount" = "throttleFailureCount" + 1
WHERE "id" = $2`,
[throttleDurationMs, id],
workspaceId,
transactionManager,
);
}
public async resetThrottlePauseUntilAndThrottleFailureCount(
id: string,
workspaceId: string,
transactionManager?: EntityManager,
) {
const dataSourceSchema =
this.workspaceDataSourceService.getSchemaName(workspaceId);
await this.workspaceDataSourceService.executeRawQuery(
`UPDATE ${dataSourceSchema}."messageChannel" SET "throttlePauseUntil" = NULL, "throttleFailureCount" = 0
WHERE "id" = $1`,
[id],
workspaceId,
transactionManager,
);
}
}

47
packages/twenty-server/src/modules/messaging/common/services/messaging-error-handling.service.ts
View File

@ -9,6 +9,9 @@ import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/s
import { MessagingTelemetryService } from 'src/modules/messaging/common/services/messaging-telemetry.service';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service';
import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository';
import { MESSAGING_THROTTLE_DURATION } from 'src/modules/messaging/common/constants/messaging-throttle-duration';
import { MESSAGING_THROTTLE_MAX_ATTEMPTS } from 'src/modules/messaging/common/constants/messaging-throttle-max-attempts';
type SyncStep =
| 'partial-message-list-fetch'
@ -27,6 +30,8 @@ export class MessagingErrorHandlingService {
private readonly connectedAccountRepository: ConnectedAccountRepository,
private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService,
private readonly messagingTelemetryService: MessagingTelemetryService,
@InjectObjectMetadataRepository(MessageChannelWorkspaceEntity)
private readonly messageChannelRepository: MessageChannelRepository,
) {}
public async handleGmailError(
@ -100,7 +105,7 @@ export class MessagingErrorHandlingService {
}
}
public async handleRateLimitExceeded(
private async handleRateLimitExceeded(
error: GmailError,
syncStep: SyncStep,
messageChannel: ObjectRecord<MessageChannelWorkspaceEntity>,
@ -114,6 +119,19 @@ export class MessagingErrorHandlingService {
message: `${error.code}: ${error.reason}`,
});
if (
messageChannel.throttleFailureCount >= MESSAGING_THROTTLE_MAX_ATTEMPTS
) {
await this.messagingChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
messageChannel.id,
workspaceId,
);
return;
}
await this.throttle(messageChannel, workspaceId);
switch (syncStep) {
case 'full-message-list-fetch':
await this.messagingChannelSyncStatusService.scheduleFullMessageListFetch(
@ -141,7 +159,7 @@ export class MessagingErrorHandlingService {
}
}
public async handleInsufficientPermissions(
private async handleInsufficientPermissions(
error: GmailError,
syncStep: SyncStep,
messageChannel: ObjectRecord<MessageChannelWorkspaceEntity>,
@ -166,7 +184,7 @@ export class MessagingErrorHandlingService {
);
}
public async handleNotFound(
private async handleNotFound(
error: GmailError,
syncStep: SyncStep,
messageChannel: ObjectRecord<MessageChannelWorkspaceEntity>,
@ -189,4 +207,27 @@ export class MessagingErrorHandlingService {
workspaceId,
);
}
private async throttle(
messageChannel: ObjectRecord<MessageChannelWorkspaceEntity>,
workspaceId: string,
): Promise<void> {
const throttleDuration =
MESSAGING_THROTTLE_DURATION *
Math.pow(2, messageChannel.throttleFailureCount);
await this.messageChannelRepository.updateThrottlePauseUntilAndIncrementThrottleFailureCount(
messageChannel.id,
throttleDuration,
workspaceId,
);
await this.messagingTelemetryService.track({
eventName: 'message_channel.throttle',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
message: `Throttling for ${throttleDuration}ms`,
});
}
}