[Messaging] Gmail Full sync pagination (#3664)

packages/twenty-server/src/workspace/messaging/services/fetch-workspace-messages.module.ts

@@ -3,7 +3,6 @@ import { Module } from '@nestjs/common';
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { EnvironmentModule } from 'src/integrations/environment/environment.module';
 import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
-import { MessagingModule } from 'src/workspace/messaging/messaging.module';
 import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
 import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service';
 import { GmailFullSyncService } from 'src/workspace/messaging/services/gmail-full-sync.service';
@@ -12,12 +11,7 @@ import { GmailRefreshAccessTokenService } from 'src/workspace/messaging/services
 import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
 
 @Module({
-  imports: [
-    MessagingModule,
-    TypeORMModule,
-    DataSourceModule,
-    EnvironmentModule,
-  ],
+  imports: [TypeORMModule, DataSourceModule, EnvironmentModule],
   providers: [
     GmailFullSyncService,
     GmailPartialSyncService,

packages/twenty-server/src/workspace/messaging/services/gmail-full-sync.service.ts

@@ -1,8 +1,14 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 
 import { FetchMessagesByBatchesService } from 'src/workspace/messaging/services/fetch-messages-by-batches.service';
 import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
 import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
+import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
+import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
+import {
+  GmailFullSyncJobData,
+  GmailFullSyncJob,
+} from 'src/workspace/messaging/jobs/gmail-full-sync.job';
 
 @Injectable()
 export class GmailFullSyncService {
@@ -10,18 +16,23 @@ export class GmailFullSyncService {
     private readonly gmailClientProvider: GmailClientProvider,
     private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
     private readonly utils: MessagingUtilsService,
+    @Inject(MessageQueue.messagingQueue)
+    private readonly messageQueueService: MessageQueueService,
   ) {}
 
   public async fetchConnectedAccountThreads(
     workspaceId: string,
     connectedAccountId: string,
-    maxResults = 500,
+    nextPageToken?: string,
   ): Promise<void> {
-    const { workspaceDataSource, dataSourceMetadata, connectedAccount } =
-      await this.utils.getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
-        workspaceId,
-        connectedAccountId,
-      );
+    const { workspaceDataSource, dataSourceMetadata } =
+      await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
+
+    const connectedAccount = await this.utils.getConnectedAcountByIdOrFail(
+      connectedAccountId,
+      dataSourceMetadata,
+      workspaceDataSource,
+    );
 
     const accessToken = connectedAccount.accessToken;
     const refreshToken = connectedAccount.refreshToken;
@@ -48,7 +59,8 @@ export class GmailFullSyncService {
 
     const messages = await gmailClient.users.messages.list({
       userId: 'me',
-      maxResults,
+      maxResults: 500,
+      pageToken: nextPageToken,
     });
 
     const messagesData = messages.data.messages;
@@ -119,5 +131,20 @@ export class GmailFullSyncService {
       dataSourceMetadata,
       workspaceDataSource,
     );
+
+    if (messages.data.nextPageToken) {
+      await this.messageQueueService.add<GmailFullSyncJobData>(
+        GmailFullSyncJob.name,
+        {
+          workspaceId,
+          connectedAccountId,
+          nextPageToken: messages.data.nextPageToken,
+        },
+        {
+          id: `${workspaceId}-${connectedAccountId}`,
+          retryLimit: 2,
+        },
+      );
+    }
   }
 }

packages/twenty-server/src/workspace/messaging/services/gmail-partial-sync.service.ts

@@ -28,11 +28,14 @@ export class GmailPartialSyncService {
     lastSyncHistoryId: string,
     maxResults: number,
   ) {
-    const { connectedAccount } =
-      await this.utils.getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
-        workspaceId,
-        connectedAccountId,
-      );
+    const { workspaceDataSource, dataSourceMetadata } =
+      await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
+
+    const connectedAccount = await this.utils.getConnectedAcountByIdOrFail(
+      connectedAccountId,
+      dataSourceMetadata,
+      workspaceDataSource,
+    );
 
     const gmailClient = await this.gmailClientProvider.getGmailClient(
       connectedAccount.refreshToken,
@@ -53,17 +56,19 @@ export class GmailPartialSyncService {
     connectedAccountId: string,
     maxResults = 500,
   ): Promise<void> {
-    const { workspaceDataSource, dataSourceMetadata, connectedAccount } =
-      await this.utils.getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
-        workspaceId,
-        connectedAccountId,
-      );
+    const { workspaceDataSource, dataSourceMetadata } =
+      await this.utils.getDataSourceMetadataWorkspaceMetadata(workspaceId);
+
+    const connectedAccount = await this.utils.getConnectedAcountByIdOrFail(
+      connectedAccountId,
+      dataSourceMetadata,
+      workspaceDataSource,
+    );
 
     const lastSyncHistoryId = connectedAccount.lastSyncHistoryId;
 
     if (!lastSyncHistoryId) {
       // Fall back to full sync
-
       await this.messageQueueService.add<GmailFullSyncJobData>(
         GmailFullSyncJob.name,
         { workspaceId, connectedAccountId },

packages/twenty-server/src/workspace/messaging/services/messaging-utils.service.ts

@@ -202,39 +202,39 @@ export class MessagingUtilsService {
     );
   }
 
-  public async getConnectedAccountsFromWorkspaceId(
-    workspaceId: string,
+  public async getConnectedAccounts(
+    dataSourceMetadata: DataSourceEntity,
+    workspaceDataSource: DataSource,
   ): Promise<any[]> {
-    const dataSourceMetadata =
-      await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
-        workspaceId,
-      );
-
-    const workspaceDataSource =
-      await this.typeORMService.connectToDataSource(dataSourceMetadata);
-
-    if (!workspaceDataSource) {
-      throw new Error('No workspace data source found');
-    }
-
     const connectedAccounts = await workspaceDataSource?.query(
       `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'google'`,
     );
 
+    return connectedAccounts;
+  }
+
+  public async getConnectedAcountByIdOrFail(
+    connectedAccountId: string,
+    dataSourceMetadata: DataSourceEntity,
+    workspaceDataSource: DataSource,
+  ): Promise<any> {
+    const connectedAccounts = await workspaceDataSource?.query(
+      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "id" = $1`,
+      [connectedAccountId],
+    );
+
     if (!connectedAccounts || connectedAccounts.length === 0) {
       throw new Error('No connected account found');
     }
 
-    return connectedAccounts;
+    return connectedAccounts[0];
   }
 
-  public async getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
+  public async getDataSourceMetadataWorkspaceMetadata(
     workspaceId: string,
-    connectedAccountId: string,
   ): Promise<{
     dataSourceMetadata: DataSourceEntity;
     workspaceDataSource: DataSource;
-    connectedAccount: any;
   }> {
     const dataSourceMetadata =
       await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
@@ -248,19 +248,9 @@ export class MessagingUtilsService {
       throw new Error('No workspace data source found');
     }
 
-    const connectedAccounts = await workspaceDataSource?.query(
-      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'google' AND "id" = $1`,
-      [connectedAccountId],
-    );
-
-    if (!connectedAccounts || connectedAccounts.length === 0) {
-      throw new Error('No connected account found');
-    }
-
     return {
       dataSourceMetadata,
       workspaceDataSource,
-      connectedAccount: connectedAccounts[0],
     };
   }