3239 create a command to do a partial sync with the gmail api using the historyid (#3405)

* create utils service

* getLastSyncHistoryId

* getHistory

* add historyTypes messageAdded and messageDeleted

* getMessageIdsAndThreadIdsNotInDatabase

* wip

* fix messageThreadId null

* no need to fetch threads anymore

* get messagesAdded in partial sync

* adding errors

* save lastSyncHistoryId

* improve

* renaming

* create partial sync job

* improve partial sync

* adding messages with partial sync is working

* now adding messages with partial sync is working

* deleting messages and empty threads is working

* wip

* wip

* fix bug to delete threads

* update partial sync to cover edge cases

* renaming

* modify ambiguous naming

* renaming

packages/twenty-server/src/workspace/messaging/commands/fetch-workspace-messages-commands.module.ts

@@ -4,8 +4,10 @@ import { TypeOrmModule } from '@nestjs/typeorm';
 import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { DataSourceModule } from 'src/metadata/data-source/data-source.module';
-import { FetchWorkspaceMessagesCommand } from 'src/workspace/messaging/commands/fetch-workspace-messages.command';
+import { GmailFullSyncCommand } from 'src/workspace/messaging/commands/gmail-full-sync.command';
+import { GmailPartialSyncCommand } from 'src/workspace/messaging/commands/gmail-partial-sync.command';
 import { MessagingModule } from 'src/workspace/messaging/messaging.module';
+import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
 
 @Module({
   imports: [
@@ -14,6 +16,10 @@ import { MessagingModule } from 'src/workspace/messaging/messaging.module';
     TypeORMModule,
     TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
   ],
-  providers: [FetchWorkspaceMessagesCommand],
+  providers: [
+    GmailFullSyncCommand,
+    GmailPartialSyncCommand,
+    MessagingUtilsService,
+  ],
 })
 export class FetchWorkspaceMessagesCommandsModule {}

packages/twenty-server/src/workspace/messaging/commands/gmail-full-sync.command.ts

@@ -4,23 +4,21 @@ import { Command, CommandRunner, Option } from 'nest-commander';
 import { Repository } from 'typeorm';
 
 import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
-import { TypeORMService } from 'src/database/typeorm/typeorm.service';
-import { DataSourceService } from 'src/metadata/data-source/data-source.service';
 import { MessagingProducer } from 'src/workspace/messaging/producers/messaging-producer';
+import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
 
-interface FetchWorkspaceMessagesOptions {
+interface GmailFullSyncOptions {
   workspaceId: string;
 }
 
 @Command({
-  name: 'workspace:fetch-messages',
+  name: 'workspace:gmail-full-sync',
   description: 'Fetch messages of all workspaceMembers in a workspace.',
 })
-export class FetchWorkspaceMessagesCommand extends CommandRunner {
+export class GmailFullSyncCommand extends CommandRunner {
   constructor(
-    private readonly dataSourceService: DataSourceService,
-    private readonly typeORMService: TypeORMService,
     private readonly messagingProducer: MessagingProducer,
+    private readonly utils: MessagingUtilsService,
 
     @InjectRepository(FeatureFlagEntity, 'core')
     private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
@@ -30,7 +28,7 @@ export class FetchWorkspaceMessagesCommand extends CommandRunner {
 
   async run(
     _passedParam: string[],
-    options: FetchWorkspaceMessagesOptions,
+    options: GmailFullSyncOptions,
   ): Promise<void> {
     const isMessagingEnabled = await this.featureFlagRepository.findOneBy({
       workspaceId: options.workspaceId,
@@ -57,28 +55,11 @@ export class FetchWorkspaceMessagesCommand extends CommandRunner {
   }
 
   private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
-    const dataSourceMetadata =
-      await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
-        workspaceId,
-      );
-
-    const workspaceDataSource =
-      await this.typeORMService.connectToDataSource(dataSourceMetadata);
-
-    if (!workspaceDataSource) {
-      throw new Error('No workspace data source found');
-    }
-
-    const connectedAccounts = await workspaceDataSource?.query(
-      `SELECT * FROM ${dataSourceMetadata.schema}."connectedAccount" WHERE "provider" = 'gmail'`,
-    );
-
-    if (!connectedAccounts || connectedAccounts.length === 0) {
-      throw new Error('No connected account found');
-    }
+    const connectedAccounts =
+      await this.utils.getConnectedAccountsFromWorkspaceId(workspaceId);
 
     for (const connectedAccount of connectedAccounts) {
-      await this.messagingProducer.enqueueFetchAllMessagesFromConnectedAccount(
+      await this.messagingProducer.enqueueGmailFullSync(
         { workspaceId, connectedAccountId: connectedAccount.id },
         `${workspaceId}-${connectedAccount.id}`,
       );

packages/twenty-server/src/workspace/messaging/commands/gmail-partial-sync.command.ts

@@ -0,0 +1,68 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command, CommandRunner, Option } from 'nest-commander';
+import { Repository } from 'typeorm';
+
+import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
+import { MessagingProducer } from 'src/workspace/messaging/producers/messaging-producer';
+import { MessagingUtilsService } from 'src/workspace/messaging/services/messaging-utils.service';
+
+interface GmailPartialSyncOptions {
+  workspaceId: string;
+}
+
+@Command({
+  name: 'workspace:gmail-partial-sync',
+  description: 'Fetch messages of all workspaceMembers in a workspace.',
+})
+export class GmailPartialSyncCommand extends CommandRunner {
+  constructor(
+    private readonly messagingProducer: MessagingProducer,
+    private readonly utils: MessagingUtilsService,
+
+    @InjectRepository(FeatureFlagEntity, 'core')
+    private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
+  ) {
+    super();
+  }
+
+  async run(
+    _passedParam: string[],
+    options: GmailPartialSyncOptions,
+  ): Promise<void> {
+    const isMessagingEnabled = await this.featureFlagRepository.findOneBy({
+      workspaceId: options.workspaceId,
+      key: 'IS_MESSAGING_ENABLED',
+      value: true,
+    });
+
+    if (!isMessagingEnabled) {
+      throw new Error('Messaging is not enabled for this workspace');
+    }
+
+    await this.fetchWorkspaceMessages(options.workspaceId);
+
+    return;
+  }
+
+  @Option({
+    flags: '-w, --workspace-id [workspace_id]',
+    description: 'workspace id',
+    required: true,
+  })
+  parseWorkspaceId(value: string): string {
+    return value;
+  }
+
+  private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
+    const connectedAccounts =
+      await this.utils.getConnectedAccountsFromWorkspaceId(workspaceId);
+
+    for (const connectedAccount of connectedAccounts) {
+      await this.messagingProducer.enqueueGmailPartialSync(
+        { workspaceId, connectedAccountId: connectedAccount.id },
+        `${workspaceId}-${connectedAccount.id}`,
+      );
+    }
+  }
+}