Add fetch connected account job (#3313)

* Add fetch connected account job

* add featureFlag check

packages/twenty-server/src/integrations/message-queue/jobs.module.ts

@@ -2,12 +2,13 @@ import { Module } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
 import { HttpModule } from '@nestjs/axios';
 
-import { FetchMessagesJob } from 'src/workspace/messaging/jobs/fetch-messages.job';
+import { FetchAllMessagesFromConnectedAccountJob } from 'src/workspace/messaging/jobs/fetch-all-messages-from-connected-account.job';
 import { CallWebhookJobsJob } from 'src/workspace/workspace-query-runner/jobs/call-webhook-jobs.job';
 import { CallWebhookJob } from 'src/workspace/workspace-query-runner/jobs/call-webhook.job';
 import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
 import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
 import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
+import { FetchWorkspaceMessagesModule } from 'src/workspace/messaging/services/fetch-workspace-messages.module';
 
 @Module({
   imports: [
@@ -15,11 +16,12 @@ import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
     ObjectMetadataModule,
     DataSourceModule,
     HttpModule,
+    FetchWorkspaceMessagesModule,
   ],
   providers: [
     {
-      provide: FetchMessagesJob.name,
+      provide: FetchAllMessagesFromConnectedAccountJob.name,
-      useClass: FetchMessagesJob,
+      useClass: FetchAllMessagesFromConnectedAccountJob,
     },
     {
       provide: CallWebhookJobsJob.name,

packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts

@@ -1,11 +1,19 @@
 import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
 
+import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
+import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
+import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
 import { FetchWorkspaceMessagesCommand } from 'src/workspace/messaging/commands/fetch-workspace-messages.command';
 import { MessagingModule } from 'src/workspace/messaging/messaging.module';
-import { FetchWorkspaceMessagesModule } from 'src/workspace/messaging/services/fetch-workspace-messages.module';
 
 @Module({
-  imports: [MessagingModule, FetchWorkspaceMessagesModule],
+  imports: [
+    MessagingModule,
+    DataSourceModule,
+    TypeORMModule,
+    TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
+  ],
   providers: [FetchWorkspaceMessagesCommand],
 })
 export class FetchWorkspaceMessagesCommandsModule {}

packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages.command.ts

@@ -1,6 +1,11 @@
-import { Command, CommandRunner, Option } from 'nest-commander';
+import { InjectRepository } from '@nestjs/typeorm';
 
-import { FetchWorkspaceMessagesService } from 'src/workspace/messaging/services/fetch-workspace-messages.service';
+import { Command, CommandRunner, Option } from 'nest-commander';
+import { Repository } from 'typeorm';
+
+import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
+import { TypeORMService } from 'src/database/typeorm/typeorm.service';
+import { DataSourceService } from 'src/metadata/data-source/data-source.service';
 import { MessagingProducer } from 'src/workspace/messaging/producers/messaging-producer';
 
 interface FetchWorkspaceMessagesOptions {
@@ -13,8 +18,12 @@ interface FetchWorkspaceMessagesOptions {
 })
 export class FetchWorkspaceMessagesCommand extends CommandRunner {
   constructor(
-    private readonly fetchWorkspaceMessagesService: FetchWorkspaceMessagesService,
+    private readonly dataSourceService: DataSourceService,
+    private readonly typeORMService: TypeORMService,
     private readonly messagingProducer: MessagingProducer,
+
+    @InjectRepository(FeatureFlagEntity, 'core')
+    private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
   ) {
     super();
   }
@@ -23,14 +32,17 @@ export class FetchWorkspaceMessagesCommand extends CommandRunner {
     _passedParam: string[],
     options: FetchWorkspaceMessagesOptions,
   ): Promise<void> {
-    await this.messagingProducer.enqueueFetchMessages(
+    const isMessagingEnabled = await this.featureFlagRepository.findOneBy({
-      { workspaceId: options.workspaceId },
+      workspaceId: options.workspaceId,
-      options.workspaceId,
+      key: 'IS_MESSAGING_ENABLED',
-    );
+      value: true,
+    });
 
-    await this.fetchWorkspaceMessagesService.fetchWorkspaceMessages(
+    if (!isMessagingEnabled) {
-      options.workspaceId,
+      throw new Error('Messaging is not enabled for this workspace');
-    );
+    }
+
+    await this.fetchWorkspaceMessages(options.workspaceId);
 
     return;
   }
@@ -43,4 +55,33 @@ export class FetchWorkspaceMessagesCommand extends CommandRunner {
   parseWorkspaceId(value: string): string {
     return value;
   }
+
+  private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
+    const dataSourceMetadata =
+      await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
+        workspaceId,
+      );
+
+    const workspaceDataSource =
+      await this.typeORMService.connectToDataSource(dataSourceMetadata);
+
+    if (!workspaceDataSource) {
+      throw new Error('No workspace data source found');
+    }
+
+    const connectedAccounts = await workspaceDataSource?.query(
+      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'gmail'`,
+    );
+
+    if (!connectedAccounts || connectedAccounts.length === 0) {
+      throw new Error('No connected account found');
+    }
+
+    for (const connectedAccount of connectedAccounts) {
+      await this.messagingProducer.enqueueFetchAllMessagesFromConnectedAccount(
+        { workspaceId, connectedAccountId: connectedAccount.id },
+        `${workspaceId}-${connectedAccount.id}`,
+      );
+    }
+  }
 }

packages/twenty-server/src/workspace/messaging/jobs/fetch-all-messages-from-connected-account.job.ts

@@ -0,0 +1,42 @@
+import { Injectable } from '@nestjs/common';
+
+import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
+
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+import { RefreshAccessTokenService } from 'src/workspace/messaging/services/refresh-access-token.service';
+import { FetchWorkspaceMessagesService } from 'src/workspace/messaging/services/fetch-workspace-messages.service';
+
+export type FetchAllMessagesFromConnectedAccountJobData = {
+  workspaceId: string;
+  connectedAccountId: string;
+};
+
+@Injectable()
+export class FetchAllMessagesFromConnectedAccountJob
+  implements MessageQueueJob<FetchAllMessagesFromConnectedAccountJobData>
+{
+  constructor(
+    private readonly environmentService: EnvironmentService,
+    private readonly refreshAccessTokenService: RefreshAccessTokenService,
+    private readonly fetchWorkspaceMessagesService: FetchWorkspaceMessagesService,
+  ) {}
+
+  async handle(
+    data: FetchAllMessagesFromConnectedAccountJobData,
+  ): Promise<void> {
+    console.log(
+      `fetching messages for workspace ${data.workspaceId} and account ${
+        data.connectedAccountId
+      } with ${this.environmentService.getMessageQueueDriverType()}`,
+    );
+    await this.refreshAccessTokenService.refreshAndSaveAccessToken(
+      data.workspaceId,
+      data.connectedAccountId,
+    );
+
+    await this.fetchWorkspaceMessagesService.fetchConnectedAccountThreads(
+      data.workspaceId,
+      data.connectedAccountId,
+    );
+  }
+}

packages/twenty-server/src/workspace/messaging/jobs/fetch-messages.job.ts

@@ -1,22 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
-
-import { EnvironmentService } from 'src/integrations/environment/environment.service';
-
-export type FetchMessagesJobData = {
-  workspaceId: string;
-};
-
-@Injectable()
-export class FetchMessagesJob implements MessageQueueJob<FetchMessagesJobData> {
-  constructor(private readonly environmentService: EnvironmentService) {}
-
-  async handle(data: FetchMessagesJobData): Promise<void> {
-    console.log(
-      `fetching messages for workspace ${
-        data.workspaceId
-      } with ${this.environmentService.getMessageQueueDriverType()}`,
-    );
-  }
-}

packages/twenty-server/src/workspace/messaging/producers/messaging-producer.ts

@@ -3,9 +3,9 @@ import { Inject, Injectable } from '@nestjs/common';
 import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
 import {
-  FetchMessagesJob,
+  FetchAllMessagesFromConnectedAccountJob,
-  FetchMessagesJobData,
+  FetchAllMessagesFromConnectedAccountJobData,
-} from 'src/workspace/messaging/jobs/fetch-messages.job';
+} from 'src/workspace/messaging/jobs/fetch-all-messages-from-connected-account.job';
 
 @Injectable()
 export class MessagingProducer {
@@ -14,9 +14,12 @@ export class MessagingProducer {
     private readonly messageQueueService: MessageQueueService,
   ) {}
 
-  async enqueueFetchMessages(data: FetchMessagesJobData, singletonKey: string) {
+  async enqueueFetchAllMessagesFromConnectedAccount(
-    await this.messageQueueService.add<FetchMessagesJobData>(
+    data: FetchAllMessagesFromConnectedAccountJobData,
-      FetchMessagesJob.name,
+    singletonKey: string,
+  ) {
+    await this.messageQueueService.add<FetchAllMessagesFromConnectedAccountJobData>(
+      FetchAllMessagesFromConnectedAccountJob.name,
       data,
       {
         id: singletonKey,

packages/twenty-server/src/workspace/messaging/providers/gmail/gmail-client.provider.ts

@@ -0,0 +1,39 @@
+import { Injectable } from '@nestjs/common';
+
+import { OAuth2Client } from 'google-auth-library';
+import { gmail_v1, google } from 'googleapis';
+
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+
+@Injectable()
+export class GmailClientProvider {
+  constructor(private readonly environmentService: EnvironmentService) {}
+
+  public async getGmailClient(refreshToken: string): Promise<gmail_v1.Gmail> {
+    const oAuth2Client = await this.getOAuth2Client(refreshToken);
+
+    const gmailClient = google.gmail({
+      version: 'v1',
+      auth: oAuth2Client,
+    });
+
+    return gmailClient;
+  }
+
+  private async getOAuth2Client(refreshToken: string): Promise<OAuth2Client> {
+    const gmailClientId = this.environmentService.getAuthGoogleClientId();
+    const gmailClientSecret =
+      this.environmentService.getAuthGoogleClientSecret();
+
+    const oAuth2Client = new google.auth.OAuth2(
+      gmailClientId,
+      gmailClientSecret,
+    );
+
+    oAuth2Client.setCredentials({
+      refresh_token: refreshToken,
+    });
+
+    return oAuth2Client;
+  }
+}

packages/twenty-server/src/workspace/messaging/providers/messaging-providers.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+
+import { EnvironmentModule } from 'src/integrations/environment/environment.module';
+import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
+
+@Module({
+  imports: [EnvironmentModule],
+  providers: [GmailClientProvider],
+  exports: [GmailClientProvider],
+})
+export class MessagingProvidersModule {}

packages/twenty-server/src/workspace/messaging/services/fetch-batch-messages.service.ts

@@ -8,10 +8,12 @@ import { MessageOrThreadQuery } from 'src/workspace/messaging/types/messageOrThr
 import { GmailMessageParsedResponse } from 'src/workspace/messaging/types/gmailMessageParsedResponse';
 import { GmailThreadParsedResponse } from 'src/workspace/messaging/types/gmailThreadParsedResponse';
 import { GmailThread } from 'src/workspace/messaging/types/gmailThread';
+import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
 
 @Injectable()
 export class FetchBatchMessagesService {
   private readonly httpService: AxiosInstance;
+  private readonly gmailClientProvider: GmailClientProvider;
 
   constructor() {
     this.httpService = axios.create({

packages/twenty-server/src/workspace/messaging/services/fetch-workspace-messages.module.ts

@@ -3,17 +3,25 @@ import { Module } from '@nestjs/common';
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { EnvironmentModule } from 'src/integrations/environment/environment.module';
 import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
+import { MessagingModule } from 'src/workspace/messaging/messaging.module';
+import { MessagingProvidersModule } from 'src/workspace/messaging/providers/messaging-providers.module';
 import { FetchBatchMessagesService } from 'src/workspace/messaging/services/fetch-batch-messages.service';
 import { FetchWorkspaceMessagesService } from 'src/workspace/messaging/services/fetch-workspace-messages.service';
 import { RefreshAccessTokenService } from 'src/workspace/messaging/services/refresh-access-token.service';
 
 @Module({
-  imports: [TypeORMModule, DataSourceModule, EnvironmentModule],
+  imports: [
+    MessagingModule,
+    TypeORMModule,
+    DataSourceModule,
+    EnvironmentModule,
+    MessagingProvidersModule,
+  ],
   providers: [
     FetchWorkspaceMessagesService,
     FetchBatchMessagesService,
     RefreshAccessTokenService,
   ],
-  exports: [FetchWorkspaceMessagesService],
+  exports: [FetchWorkspaceMessagesService, RefreshAccessTokenService],
 })
 export class FetchWorkspaceMessagesModule {}

packages/twenty-server/src/workspace/messaging/services/fetch-workspace-messages.service.ts

@@ -1,58 +1,47 @@
 import { Injectable } from '@nestjs/common';
 
-import { gmail_v1, google } from 'googleapis';
+import { gmail_v1 } from 'googleapis';
 import { v4 } from 'uuid';
 import { DataSource } from 'typeorm';
 
 import { TypeORMService } from 'src/database/typeorm/typeorm.service';
-import { EnvironmentService } from 'src/integrations/environment/environment.service';
 import { DataSourceService } from 'src/metadata/data-source/data-source.service';
 import { FetchBatchMessagesService } from 'src/workspace/messaging/services/fetch-batch-messages.service';
 import { GmailMessage } from 'src/workspace/messaging/types/gmailMessage';
 import { MessageOrThreadQuery } from 'src/workspace/messaging/types/messageOrThreadQuery';
 import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity';
-import { RefreshAccessTokenService } from 'src/workspace/messaging/services/refresh-access-token.service';
+import { GmailClientProvider } from 'src/workspace/messaging/providers/gmail/gmail-client.provider';
 
 @Injectable()
 export class FetchWorkspaceMessagesService {
   constructor(
-    private readonly environmentService: EnvironmentService,
+    private readonly gmailClientProvider: GmailClientProvider,
     private readonly dataSourceService: DataSourceService,
     private readonly typeORMService: TypeORMService,
     private readonly fetchBatchMessagesService: FetchBatchMessagesService,
-    private readonly refreshAccessTokenService: RefreshAccessTokenService,
   ) {}
 
-  async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
+  public async fetchConnectedAccountThreads(
-    await this.refreshAccessTokenService.refreshAndSaveAccessToken(
-      workspaceId,
-      '20202020-0687-4c41-b707-ed1bfca972a7',
-    );
-    await this.fetchWorkspaceMemberThreads(
-      workspaceId,
-      '20202020-0687-4c41-b707-ed1bfca972a7',
-    );
-  }
-
-  async fetchWorkspaceMemberThreads(
     workspaceId: string,
-    workspaceMemberId: string,
+    connectedAccountId: string,
     maxResults = 500,
   ): Promise<void> {
     const { workspaceDataSource, dataSourceMetadata, connectedAccount } =
       await this.getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
         workspaceId,
-        workspaceMemberId,
+        connectedAccountId,
       );
 
     const accessToken = connectedAccount.accessToken;
     const refreshToken = connectedAccount.refreshToken;
+    const workspaceMemberId = connectedAccount.workspaceMemberId;
 
     if (!refreshToken) {
       throw new Error('No refresh token found');
     }
 
-    const gmailClient = await this.getGmailClient(refreshToken);
+    const gmailClient =
+      await this.gmailClientProvider.getGmailClient(refreshToken);
 
     const threads = await gmailClient.users.threads.list({
       userId: 'me',
@@ -121,30 +110,7 @@ export class FetchWorkspaceMessagesService {
     );
   }
 
-  async getGmailClient(refreshToken: string): Promise<gmail_v1.Gmail> {
+  private async saveMessageThreads(
-    const gmailClientId = this.environmentService.getAuthGoogleClientId();
-
-    const gmailClientSecret =
-      this.environmentService.getAuthGoogleClientSecret();
-
-    const oAuth2Client = new google.auth.OAuth2(
-      gmailClientId,
-      gmailClientSecret,
-    );
-
-    oAuth2Client.setCredentials({
-      refresh_token: refreshToken,
-    });
-
-    const gmailClient = google.gmail({
-      version: 'v1',
-      auth: oAuth2Client,
-    });
-
-    return gmailClient;
-  }
-
-  async saveMessageThreads(
     threads: gmail_v1.Schema$Thread[],
     dataSourceMetadata: DataSourceEntity,
     workspaceDataSource: DataSource,
@@ -167,7 +133,7 @@ export class FetchWorkspaceMessagesService {
     }
   }
 
-  async saveMessages(
+  private async saveMessages(
     messages: GmailMessage[],
     dataSourceMetadata: DataSourceEntity,
     workspaceDataSource: DataSource,
@@ -225,7 +191,7 @@ export class FetchWorkspaceMessagesService {
     }
   }
 
-  async getAllSavedMessagesIdsAndMessageThreadsIdsForConnectedAccount(
+  private async getAllSavedMessagesIdsAndMessageThreadsIdsForConnectedAccount(
     dataSourceMetadata: DataSourceEntity,
     workspaceDataSource: DataSource,
     connectedAccountId: string,
@@ -252,9 +218,9 @@ export class FetchWorkspaceMessagesService {
     };
   }
 
-  async getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
+  private async getDataSourceMetadataWorkspaceMetadataAndConnectedAccount(
     workspaceId: string,
-    workspaceMemberId: string,
+    connectedAccountId: string,
   ): Promise<{
     dataSourceMetadata: DataSourceEntity;
     workspaceDataSource: DataSource;
@@ -265,17 +231,16 @@ export class FetchWorkspaceMessagesService {
         workspaceId,
       );
 
-    const workspaceDataSource = await this.typeORMService.connectToDataSource(
+    const workspaceDataSource =
-      dataSourceMetadata,
+      await this.typeORMService.connectToDataSource(dataSourceMetadata);
-    );
 
     if (!workspaceDataSource) {
       throw new Error('No workspace data source found');
     }
 
     const connectedAccounts = await workspaceDataSource?.query(
-      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'gmail' AND "accountOwnerId" = $1`,
+      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'gmail' AND "id" = $1`,
-      [workspaceMemberId],
+      [connectedAccountId],
     );
 
     if (!connectedAccounts || connectedAccounts.length === 0) {

packages/twenty-server/src/workspace/messaging/services/refresh-access-token.service.ts

@@ -16,7 +16,7 @@ export class RefreshAccessTokenService {
 
   async refreshAndSaveAccessToken(
     workspaceId: string,
-    workspaceMemberId: string,
+    connectedAccountId: string,
   ): Promise<void> {
     const dataSourceMetadata =
       await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
@@ -31,8 +31,8 @@ export class RefreshAccessTokenService {
     }
 
     const connectedAccounts = await workspaceDataSource?.query(
-      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'gmail' AND "accountOwnerId" = $1`,
+      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'gmail' AND "id" = $1`,
-      [workspaceMemberId],
+      [connectedAccountId],
     );
 
     if (!connectedAccounts || connectedAccounts.length === 0) {