mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

7142 make messaging full message list fetch idempotent (#7148)

Browse Source 
- Add message deletion and thread cleaning during full message list
fetch
- Add thread cleaning  during partial message list fetch
- Delete provider from cache key
This commit is contained in:
Raphaël Bosi
2024-09-19 18:32:25 +02:00
committed by GitHub
parent 8964d26d5b
commit 6a5f9492d3
8 changed files with 51 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts
View File

@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { In } from 'typeorm';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
@ -10,6 +10,7 @@ import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/s
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
import {
MessageImportExceptionHandlerService,
MessageImportSyncStep,
@ -25,6 +26,7 @@ export class MessagingFullMessageListFetchService {
private readonly twentyORMManager: TwentyORMManager,
private readonly messagingGetMessageListService: MessagingGetMessageListService,
private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService,
private readonly messagingMessageCleanerService: MessagingMessageCleanerService,
) {}
public async processMessageListFetch(
@ -51,7 +53,6 @@ export class MessagingFullMessageListFetchService {
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
messageExternalId: Any(messageExternalIds),
},
});
@ -61,17 +62,35 @@ export class MessagingFullMessageListFetchService {
messageChannelMessageAssociation.messageExternalId,
);
const messageIdsToImport = messageExternalIds.filter(
const messageExternalIdsToImport = messageExternalIds.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
if (messageIdsToImport.length) {
const messageExternalIdsToDelete =
existingMessageChannelMessageAssociationsExternalIds.filter(
(existingMessageCMAExternalId) =>
existingMessageCMAExternalId &&
!messageExternalIds.includes(existingMessageCMAExternalId),
);
if (messageExternalIdsToDelete.length) {
await messageChannelMessageAssociationRepository.delete({
messageChannelId: messageChannel.id,
messageExternalId: In(messageExternalIdsToDelete),
});
await this.messagingMessageCleanerService.cleanWorkspaceThreads(
workspaceId,
);
}
if (messageExternalIdsToImport.length) {
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:gmail:${messageChannel.id}`,
messageIdsToImport,
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageExternalIdsToImport,
);
}

4
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts
View File

@ -114,7 +114,7 @@ export class MessagingMessagesImportService {
);
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:gmail:${messageChannel.id}`,
`messages-to-import:${workspaceId}:${messageChannel.id}`,
MESSAGING_GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
);
@ -186,7 +186,7 @@ export class MessagingMessagesImportService {
);
} catch (error) {
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:gmail:${messageChannel.id}`,
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);

20
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts
View File

@ -1,6 +1,6 @@
import { Injectable, Logger } from '@nestjs/common';
import { Any } from 'typeorm';
import { In } from 'typeorm';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
@ -10,6 +10,7 @@ import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/s
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
import {
MessageImportExceptionHandlerService,
MessageImportSyncStep,
@ -29,6 +30,7 @@ export class MessagingPartialMessageListFetchService {
private readonly messageChannelSyncStatusService: MessageChannelSyncStatusService,
private readonly twentyORMManager: TwentyORMManager,
private readonly messageImportErrorHandlerService: MessageImportExceptionHandlerService,
private readonly messagingMessageCleanerService: MessagingMessageCleanerService,
) {}
public async processMessageListFetch(
@ -77,7 +79,7 @@ export class MessagingPartialMessageListFetchService {
}
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:gmail:${messageChannel.id}`,
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageExternalIds,
);
@ -90,10 +92,16 @@ export class MessagingPartialMessageListFetchService {
'messageChannelMessageAssociation',
);
await messageChannelMessageAssociationRepository.delete({
messageChannelId: messageChannel.id,
messageExternalId: Any(messageExternalIdsToDelete),
});
if (messageExternalIdsToDelete.length) {
await messageChannelMessageAssociationRepository.delete({
messageChannelId: messageChannel.id,
messageExternalId: In(messageExternalIdsToDelete),
});
await this.messagingMessageCleanerService.cleanWorkspaceThreads(
workspaceId,
);
}
this.logger.log(
`Deleted ${messageExternalIdsToDelete.length} messages for workspace ${workspaceId} and account ${connectedAccount.id}`,