From 2efc794b434f7124f76a31df419ec49d751f06dc Mon Sep 17 00:00:00 2001 From: Weiko Date: Tue, 16 Apr 2024 17:18:06 +0200 Subject: [PATCH] [messaging] Add message deletion during partial sync (#4972) ## Context - Rename remaining V2 services. - Delete messages in DB when gmail history tells us they've been deleted. I removed the logic where we store those in a cache since it's a bit overkill because we don't need to query gmail and can use those ids directly. The strategy is to delete the message channel message association of the current channel, not the message or the thread since they can still be linked to other channels. However, we will need to call the threadCleaner service on the workspace to remove orphan threads/non-associated messages. Note: deletion for full-sync is a bit tricky because we need the full list of message ids to compare with the DB and make sure we don't over-delete. Currently, to keep memory, we don't have a variable that holds all ids as we flush it after each page. Easier solution would be to wipe everything before each full sync but it's probably not great for the user experience if they are currently manipulating messages since full-sync can happen without a user intervention (if a partial sync fails due to historyId being invalidated by google for some reason) --- .../messaging/timeline-messaging.service.ts | 15 +++++++++++++++ .../integrations/message-queue/jobs.module.ts | 8 ++++---- .../modules/messaging/jobs/gmail-full-sync.job.ts | 4 ++-- .../messaging/jobs/gmail-partial-sync.job.ts | 2 +- .../gmail-full-sync.module.ts} | 8 ++++---- .../gmail-full-sync.service.ts} | 4 ++-- .../gmail-partial-sync.module.ts} | 8 ++++---- .../gmail-partial-sync.service.ts} | 12 ++++++++++-- 8 files changed, 42 insertions(+), 19 deletions(-) rename packages/twenty-server/src/modules/messaging/services/{gmail-full-sync-v2/gmail-full-sync.v2.module.ts => gmail-full-sync/gmail-full-sync.module.ts} (87%) rename packages/twenty-server/src/modules/messaging/services/{gmail-full-sync-v2/gmail-full-sync.v2.service.ts => gmail-full-sync/gmail-full-sync.service.ts} (98%) rename packages/twenty-server/src/modules/messaging/services/{gmail-partial-sync-v2/gmail-partial-sync-v2.module.ts => gmail-partial-sync/gmail-partial-sync.module.ts} (86%) rename packages/twenty-server/src/modules/messaging/services/{gmail-partial-sync-v2/gmail-partial-sync-v2.service.ts => gmail-partial-sync/gmail-partial-sync.service.ts} (94%) diff --git a/packages/twenty-server/src/engine/core-modules/messaging/timeline-messaging.service.ts b/packages/twenty-server/src/engine/core-modules/messaging/timeline-messaging.service.ts index 9b962cae8..3a186d6c0 100644 --- a/packages/twenty-server/src/engine/core-modules/messaging/timeline-messaging.service.ts +++ b/packages/twenty-server/src/engine/core-modules/messaging/timeline-messaging.service.ts @@ -58,6 +58,11 @@ export class TimelineMessagingService { ${dataSourceSchema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id WHERE "messageParticipant"."personId" = ANY($1) + AND EXISTS ( + SELECT 1 + FROM ${dataSourceSchema}."messageChannelMessageAssociation" mcma + WHERE mcma."messageId" = message.id + ) GROUP BY message."messageThreadId", message.id @@ -128,6 +133,11 @@ export class TimelineMessagingService { ${dataSourceSchema}."message" message WHERE message."messageThreadId" = ANY($1) + AND EXISTS ( + SELECT 1 + FROM ${dataSourceSchema}."messageChannelMessageAssociation" mcma + WHERE mcma."messageId" = message.id + ) GROUP BY message."messageThreadId" `, @@ -248,6 +258,11 @@ export class TimelineMessagingService { ${dataSourceSchema}."messageParticipant" "messageParticipant" ON "messageParticipant"."messageId" = message.id WHERE "messageParticipant"."personId" = ANY($1) + AND EXISTS ( + SELECT 1 + FROM ${dataSourceSchema}."messageChannelMessageAssociation" mcma + WHERE mcma."messageId" = message.id + ) `, [personIds], workspaceId, diff --git a/packages/twenty-server/src/engine/integrations/message-queue/jobs.module.ts b/packages/twenty-server/src/engine/integrations/message-queue/jobs.module.ts index 83985f45e..843692594 100644 --- a/packages/twenty-server/src/engine/integrations/message-queue/jobs.module.ts +++ b/packages/twenty-server/src/engine/integrations/message-queue/jobs.module.ts @@ -48,8 +48,8 @@ import { GmailFullSyncJob } from 'src/modules/messaging/jobs/gmail-full-sync.job import { GmailPartialSyncJob } from 'src/modules/messaging/jobs/gmail-partial-sync.job'; import { MessagingCreateCompanyAndContactAfterSyncJob } from 'src/modules/messaging/jobs/messaging-create-company-and-contact-after-sync.job'; import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module'; -import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module'; -import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module'; +import { GmailFullSyncModule } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.module'; +import { GmailPartialSyncModule } from 'src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.module'; import { MessageParticipantModule } from 'src/modules/messaging/services/message-participant/message-participant.module'; import { ThreadCleanerModule } from 'src/modules/messaging/services/thread-cleaner/thread-cleaner.module'; import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata'; @@ -81,9 +81,9 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj MessageChannelObjectMetadata, EventObjectMetadata, ]), - GmailFullSynV2Module, + GmailFullSyncModule, GmailFetchMessageContentFromCacheModule, - GmailPartialSyncV2Module, + GmailPartialSyncModule, CalendarEventParticipantModule, ], providers: [ diff --git a/packages/twenty-server/src/modules/messaging/jobs/gmail-full-sync.job.ts b/packages/twenty-server/src/modules/messaging/jobs/gmail-full-sync.job.ts index 427c6e141..714d28125 100644 --- a/packages/twenty-server/src/modules/messaging/jobs/gmail-full-sync.job.ts +++ b/packages/twenty-server/src/modules/messaging/jobs/gmail-full-sync.job.ts @@ -3,7 +3,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface'; import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service'; -import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service'; +import { GmailFullSyncService } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service'; export type GmailFullSyncJobData = { workspaceId: string; @@ -16,7 +16,7 @@ export class GmailFullSyncJob implements MessageQueueJob { constructor( private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService, - private readonly gmailFullSyncV2Service: GmailFullSyncV2Service, + private readonly gmailFullSyncV2Service: GmailFullSyncService, ) {} async handle(data: GmailFullSyncJobData): Promise { diff --git a/packages/twenty-server/src/modules/messaging/jobs/gmail-partial-sync.job.ts b/packages/twenty-server/src/modules/messaging/jobs/gmail-partial-sync.job.ts index bc1e6c283..9b8a3b85f 100644 --- a/packages/twenty-server/src/modules/messaging/jobs/gmail-partial-sync.job.ts +++ b/packages/twenty-server/src/modules/messaging/jobs/gmail-partial-sync.job.ts @@ -3,7 +3,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface'; import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service'; -import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service'; +import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.service'; export type GmailPartialSyncJobData = { workspaceId: string; diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module.ts b/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.module.ts similarity index 87% rename from packages/twenty-server/src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module.ts rename to packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.module.ts index c0bc6bc73..0ed81b98c 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.module.ts @@ -7,7 +7,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata'; import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata'; import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module'; -import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service'; +import { GmailFullSyncService } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service'; import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module'; import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata'; import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata'; @@ -25,7 +25,7 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), WorkspaceDataSourceModule, ], - providers: [GmailFullSyncV2Service], - exports: [GmailFullSyncV2Service], + providers: [GmailFullSyncService], + exports: [GmailFullSyncService], }) -export class GmailFullSynV2Module {} +export class GmailFullSyncModule {} diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts similarity index 98% rename from packages/twenty-server/src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service.ts rename to packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts index e6c98e1f7..7743cb2d6 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts @@ -29,8 +29,8 @@ import { gmailSearchFilterExcludeEmails } from 'src/modules/messaging/utils/gmai import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; @Injectable() -export class GmailFullSyncV2Service { - private readonly logger = new Logger(GmailFullSyncV2Service.name); +export class GmailFullSyncService { + private readonly logger = new Logger(GmailFullSyncService.name); constructor( private readonly gmailClientProvider: GmailClientProvider, diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module.ts b/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.module.ts similarity index 86% rename from packages/twenty-server/src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module.ts rename to packages/twenty-server/src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.module.ts index dcd759d80..bcc4ef34d 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.module.ts @@ -7,7 +7,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata'; import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata'; import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module'; -import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service'; +import { GmailPartialSyncV2Service as GmailPartialSyncService } from 'src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.service'; import { MessageModule } from 'src/modules/messaging/services/message/message.module'; import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module'; import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module'; @@ -27,7 +27,7 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), WorkspaceDataSourceModule, ], - providers: [GmailPartialSyncV2Service], - exports: [GmailPartialSyncV2Service], + providers: [GmailPartialSyncService], + exports: [GmailPartialSyncService], }) -export class GmailPartialSyncV2Module {} +export class GmailPartialSyncModule {} diff --git a/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service.ts b/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.service.ts similarity index 94% rename from packages/twenty-server/src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service.ts rename to packages/twenty-server/src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.service.ts index cb64d9247..5d8db0f99 100644 --- a/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service.ts +++ b/packages/twenty-server/src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.service.ts @@ -24,6 +24,8 @@ import { GmailFullSyncJob, GmailFullSyncJobData, } from 'src/modules/messaging/jobs/gmail-full-sync.job'; +import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata'; +import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/repositories/message-channel-message-association.repository'; @Injectable() export class GmailPartialSyncV2Service { @@ -40,6 +42,10 @@ export class GmailPartialSyncV2Service { @InjectCacheStorage(CacheStorageNamespace.Messaging) private readonly cacheStorage: CacheStorageService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, + @InjectObjectMetadataRepository( + MessageChannelMessageAssociationObjectMetadata, + ) + private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository, ) {} public async fetchConnectedAccountThreads( @@ -201,9 +207,11 @@ export class GmailPartialSyncV2Service { messagesAdded, ); - await this.cacheStorage.setAdd( - `messages-to-delete:${workspaceId}:gmail:${gmailMessageChannel.id}`, + await this.messageChannelMessageAssociationRepository.deleteByMessageExternalIdsAndMessageChannelId( messagesDeleted, + gmailMessageChannel.id, + workspaceId, + transactionManager, ); await this.messageChannelRepository.updateLastSyncCursorIfHigher(