[Microsoft integration] getFullMessageList (#9544)

Creation of the GmailGetMessageListService
Implementation of the driver to MS Graph API getFullMessageList

packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command.ts

@@ -4,7 +4,8 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decora
 import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
 import { CalendarEventListFetchCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job';
-import { CALENDAR_EVENTS_IMPORT_CRON_PATTERN } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job';
+
+const CALENDAR_EVENTS_LIST_CRON_PATTERN = '*/5 * * * *';
 
 @Command({
   name: 'cron:calendar:calendar-event-list-fetch',
@@ -23,7 +24,9 @@ export class CalendarEventListFetchCronCommand extends CommandRunner {
       CalendarEventListFetchCronJob.name,
       undefined,
       {
-        repeat: { pattern: CALENDAR_EVENTS_IMPORT_CRON_PATTERN },
+        repeat: {
+          pattern: CALENDAR_EVENTS_LIST_CRON_PATTERN,
+        },
       },
     );
   }

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts

@@ -0,0 +1,30 @@
+import { Module } from '@nestjs/common';
+
+import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module';
+import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
+import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
+import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module';
+import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
+import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider';
+
+import { MicrosoftGetMessageListService } from './services/microsoft-get-message-list.service';
+
+@Module({
+  imports: [
+    EnvironmentModule,
+    MessagingCommonModule,
+    FeatureFlagModule,
+    OAuth2ClientManagerModule,
+    WorkspaceDataSourceModule,
+    ObjectMetadataRepositoryModule,
+  ],
+  providers: [
+    MicrosoftClientProvider,
+    MicrosoftGetMessageListService,
+    MicrosoftOAuth2ClientManagerService,
+  ],
+  exports: [MicrosoftGetMessageListService, MicrosoftClientProvider],
+})
+export class MessagingMicrosoftDriverModule {}

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts

@@ -0,0 +1,32 @@
+import { Injectable } from '@nestjs/common';
+
+import { Client } from '@microsoft/microsoft-graph-client';
+
+import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+
+@Injectable()
+export class MicrosoftClientProvider {
+  constructor(
+    private readonly microsoftOAuth2ClientManagerService: MicrosoftOAuth2ClientManagerService,
+  ) {}
+
+  public async getMicrosoftClient(
+    connectedAccount: Pick<
+      ConnectedAccountWorkspaceEntity,
+      'refreshToken' | 'id'
+    >,
+  ): Promise<Client> {
+    try {
+      return await this.microsoftOAuth2ClientManagerService.getOAuth2Client(
+        connectedAccount.refreshToken,
+      );
+    } catch (error) {
+      throw new Error(
+        `Failed to get Microsoft client: ${
+          error instanceof Error ? error.message : 'Unknown error'
+        }`,
+      );
+    }
+  }
+}

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts

@@ -0,0 +1,57 @@
+import { Injectable } from '@nestjs/common';
+
+import {
+  PageCollection,
+  PageIterator,
+  PageIteratorCallback,
+} from '@microsoft/microsoft-graph-client';
+
+import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider';
+import { GetFullMessageListResponse } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
+
+// Microsoft API limit is 1000 messages per request on this endpoint
+const MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT = 1000;
+
+@Injectable()
+export class MicrosoftGetMessageListService {
+  constructor(
+    private readonly microsoftClientProvider: MicrosoftClientProvider,
+  ) {}
+
+  public async getFullMessageList(
+    connectedAccount: Pick<
+      ConnectedAccountWorkspaceEntity,
+      'provider' | 'refreshToken' | 'id'
+    >,
+    syncCursor?: string,
+  ): Promise<GetFullMessageListResponse> {
+    const messageExternalIds: string[] = [];
+
+    const microsoftClient =
+      await this.microsoftClientProvider.getMicrosoftClient(connectedAccount);
+
+    const response: PageCollection = await microsoftClient
+      .api(syncCursor || '/me/mailfolders/inbox/messages/delta?$select=id')
+      .version('beta')
+      .headers({
+        Prefer: `odata.maxpagesize=${MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT}`,
+      })
+      .get();
+
+    const callback: PageIteratorCallback = (data) => {
+      messageExternalIds.push(data.id);
+
+      return true;
+    };
+
+    const pageIterator = new PageIterator(microsoftClient, response, callback);
+
+    await pageIterator.iterate();
+
+    return {
+      messageExternalIds: messageExternalIds,
+      nextSyncCursor: pageIterator.getDeltaLink() || '',
+    };
+  }
+}

packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts

@@ -18,6 +18,7 @@ import { MessagingMessageListFetchCronJob } from 'src/modules/messaging/message-
 import { MessagingMessagesImportCronJob } from 'src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job';
 import { MessagingOngoingStaleCronJob } from 'src/modules/messaging/message-import-manager/crons/jobs/messaging-ongoing-stale.cron.job';
 import { MessagingGmailDriverModule } from 'src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module';
+import { MessagingMicrosoftDriverModule } from 'src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module';
 import { MessagingAddSingleMessageToCacheForImportJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job';
 import { MessagingCleanCacheJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-clean-cache';
 import { MessagingMessageListFetchJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job';
@@ -40,6 +41,7 @@ import { MessagingMonitoringModule } from 'src/modules/messaging/monitoring/mess
     RefreshAccessTokenManagerModule,
     WorkspaceDataSourceModule,
     MessagingGmailDriverModule,
+    MessagingMicrosoftDriverModule,
     MessagingCommonModule,
     TypeOrmModule.forFeature([Workspace], 'core'),
     TypeOrmModule.forFeature([DataSourceEntity], 'metadata'),

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
 import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service';
+import { MicrosoftGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service';
 import {
   MessageImportException,
   MessageImportExceptionCode,
@@ -22,6 +23,7 @@ export type GetPartialMessageListResponse = {
 export class MessagingGetMessageListService {
   constructor(
     private readonly gmailGetMessageListService: GmailGetMessageListService,
+    private readonly microsoftGetMessageListService: MicrosoftGetMessageListService,
   ) {}
 
   public async getFullMessageList(
@@ -36,11 +38,9 @@ export class MessagingGetMessageListService {
           connectedAccount,
         );
       case 'microsoft':
-        // TODO: Placeholder
-        return {
-          messageExternalIds: [],
-          nextSyncCursor: '',
-        };
+        return this.microsoftGetMessageListService.getFullMessageList(
+          connectedAccount,
+        );
       default:
         throw new MessageImportException(
           `Provider ${connectedAccount.provider} is not supported`,

packages/twenty-website/src/content/developers/self-hosting/setup.mdx

@@ -54,6 +54,59 @@ Register the following recurring jobs:
 yarn command:prod cron:messaging:messages-import
 yarn command:prod cron:messaging:message-list-fetch
 yarn command:prod cron:calendar:calendar-event-list-fetch
+yarn command:prod cron:calendar:calendar-event-import
+yarn command:prod cron:messaging:ongoing-stale
+yarn command:prod cron:calendar:ongoing-stale
+```
+
+## For Outlook and Outlook Calendar (Microsoft 365)
+
+### Create a project in Microsoft Azure
+
+You will need to create a project in [Microsoft Azure](https://portal.azure.com/#view/Microsoft_AAD_IAM/AppGalleryBladeV2) and get the credentials.
+
+Then you can set the following environment variables:
+
+- `AUTH_MICROSOFT_ENABLED=true`
+- `AUTH_MICROSOFT_CLIENT_ID=<client-id>`
+- `AUTH_MICROSOFT_TENANT_ID=<tenant-id>`
+- `AUTH_MICROSOFT_CLIENT_SECRET=<client-secret>`
+- `AUTH_MICROSOFT_CALLBACK_URL=https://<your-domain>/auth/microsoft/redirect` if you want to use Microsoft SSO
+- `AUTH_MICROSOFT_APIS_CALLBACK_URL=https://<your-domain>/auth/microsoft-apis/get-access-token`
+
+### Enable APIs
+
+On Microsoft Azure Console enable the following APIs in "Permissions":
+
+- Microsoft Graph: Mail.Read
+- Microsoft Graph: Calendars.Read
+- Microsoft Graph: User.Read.All
+- Microsoft Graph: openid
+- Microsoft Graph: email
+- Microsoft Graph: profile
+- Microsoft Graph: offline_access
+
+### Authorized redirect URIs
+
+You need to add the following redirect URIs to your project:
+- `https://<your-domain>/auth/microsoft/redirect` if you want to use Microsoft SSO
+- `https://<your-domain>/auth/microsoft-apis/get-access-token`
+
+### If your app is in test mode
+
+If your app is in test mode, you will need to add test users to your project.
+
+Add your test users to the "Users and groups" section.
+
+### Start the cron jobs
+
+Register the following recurring jobs:
+```
+# from your worker container
+yarn command:prod cron:messaging:messages-import
+yarn command:prod cron:messaging:message-list-fetch
+yarn command:prod cron:calendar:calendar-event-list-fetch
+yarn command:prod cron:calendar:calendar-event-import
 yarn command:prod cron:messaging:ongoing-stale
 yarn command:prod cron:calendar:ongoing-stale
 ```