[messaing] improve messaging import (#4650)

* [messaging] improve full-sync fetching strategy

* fix

* rebase

* fix

* fix

* fix rebase

* fix

* fix

* fix

* fix

* fix

* remove deletion

* fix setPop with memory storage

* fix pgBoss and remove unnecessary job

* fix throw

* fix

* add timeout to ongoing sync

packages/twenty-server/src/modules/messaging/jobs/gmail-full-sync-v2.job.ts

@@ -0,0 +1,48 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
+
+import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
+import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
+
+export type GmailFullSyncV2JobData = {
+  workspaceId: string;
+  connectedAccountId: string;
+};
+
+@Injectable()
+export class GmailFullSyncV2Job
+  implements MessageQueueJob<GmailFullSyncV2JobData>
+{
+  private readonly logger = new Logger(GmailFullSyncV2Job.name);
+
+  constructor(
+    private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
+    private readonly gmailFullSyncV2Service: GmailFullSyncV2Service,
+  ) {}
+
+  async handle(data: GmailFullSyncV2JobData): Promise<void> {
+    this.logger.log(
+      `gmail full-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
+    );
+
+    try {
+      await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
+        data.workspaceId,
+        data.connectedAccountId,
+      );
+    } catch (e) {
+      this.logger.error(
+        `Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
+        e,
+      );
+
+      return;
+    }
+
+    await this.gmailFullSyncV2Service.fetchConnectedAccountThreads(
+      data.workspaceId,
+      data.connectedAccountId,
+    );
+  }
+}

packages/twenty-server/src/modules/messaging/jobs/gmail-full-sync.job.ts

@@ -44,7 +44,6 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
     await this.gmailFullSyncService.fetchConnectedAccountThreads(
       data.workspaceId,
       data.connectedAccountId,
-      data.nextPageToken,
     );
   }
 }

packages/twenty-server/src/modules/messaging/jobs/gmail-partial-sync-v2.job.ts

@@ -0,0 +1,48 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
+
+import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
+import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
+
+export type GmailPartialSyncV2JobData = {
+  workspaceId: string;
+  connectedAccountId: string;
+};
+
+@Injectable()
+export class GmailPartialSyncV2Job
+  implements MessageQueueJob<GmailPartialSyncV2JobData>
+{
+  private readonly logger = new Logger(GmailPartialSyncV2Job.name);
+
+  constructor(
+    private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
+    private readonly gmailPartialSyncV2Service: GmailPartialSyncV2Service,
+  ) {}
+
+  async handle(data: GmailPartialSyncV2JobData): Promise<void> {
+    this.logger.log(
+      `gmail partial-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
+    );
+
+    try {
+      await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
+        data.workspaceId,
+        data.connectedAccountId,
+      );
+    } catch (e) {
+      this.logger.error(
+        `Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
+        e,
+      );
+
+      return;
+    }
+
+    await this.gmailPartialSyncV2Service.fetchConnectedAccountThreads(
+      data.workspaceId,
+      data.connectedAccountId,
+    );
+  }
+}