From 34ee64a36c6dcd255b7043e1efb687e12b65356f Mon Sep 17 00:00:00 2001 From: Guillim Date: Mon, 13 Jan 2025 10:13:37 +0100 Subject: [PATCH] [Microsoft integration] getFullMessageList (#9544) Creation of the GmailGetMessageListService Implementation of the driver to MS Graph API getFullMessageList --- .../calendar-event-list-fetch.cron.command.ts | 7 ++- .../messaging-microsoft-driver.module.ts | 30 ++++++++++ .../providers/microsoft-client.provider.ts | 32 +++++++++++ .../microsoft-get-message-list.service.ts | 57 +++++++++++++++++++ .../messaging-import-manager.module.ts | 2 + .../messaging-get-message-list.service.ts | 10 ++-- .../content/developers/self-hosting/setup.mdx | 53 +++++++++++++++++ 7 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command.ts index 7fce8ebdb..80f341279 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/commands/calendar-event-list-fetch.cron.command.ts @@ -4,7 +4,8 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decora import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; import { CalendarEventListFetchCronJob } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job'; -import { CALENDAR_EVENTS_IMPORT_CRON_PATTERN } from 'src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-events-import.cron.job'; + +const CALENDAR_EVENTS_LIST_CRON_PATTERN = '*/5 * * * *'; @Command({ name: 'cron:calendar:calendar-event-list-fetch', @@ -23,7 +24,9 @@ export class CalendarEventListFetchCronCommand extends CommandRunner { CalendarEventListFetchCronJob.name, undefined, { - repeat: { pattern: CALENDAR_EVENTS_IMPORT_CRON_PATTERN }, + repeat: { + pattern: CALENDAR_EVENTS_LIST_CRON_PATTERN, + }, }, ); } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts new file mode 100644 index 000000000..e3afd094e --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts @@ -0,0 +1,30 @@ +import { Module } from '@nestjs/common'; + +import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; +import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; +import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; +import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; +import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module'; +import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; +import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; + +import { MicrosoftGetMessageListService } from './services/microsoft-get-message-list.service'; + +@Module({ + imports: [ + EnvironmentModule, + MessagingCommonModule, + FeatureFlagModule, + OAuth2ClientManagerModule, + WorkspaceDataSourceModule, + ObjectMetadataRepositoryModule, + ], + providers: [ + MicrosoftClientProvider, + MicrosoftGetMessageListService, + MicrosoftOAuth2ClientManagerService, + ], + exports: [MicrosoftGetMessageListService, MicrosoftClientProvider], +}) +export class MessagingMicrosoftDriverModule {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts new file mode 100644 index 000000000..0dfc149b4 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts @@ -0,0 +1,32 @@ +import { Injectable } from '@nestjs/common'; + +import { Client } from '@microsoft/microsoft-graph-client'; + +import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; + +@Injectable() +export class MicrosoftClientProvider { + constructor( + private readonly microsoftOAuth2ClientManagerService: MicrosoftOAuth2ClientManagerService, + ) {} + + public async getMicrosoftClient( + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'refreshToken' | 'id' + >, + ): Promise { + try { + return await this.microsoftOAuth2ClientManagerService.getOAuth2Client( + connectedAccount.refreshToken, + ); + } catch (error) { + throw new Error( + `Failed to get Microsoft client: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + ); + } + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts new file mode 100644 index 000000000..da526016d --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -0,0 +1,57 @@ +import { Injectable } from '@nestjs/common'; + +import { + PageCollection, + PageIterator, + PageIteratorCallback, +} from '@microsoft/microsoft-graph-client'; + +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { GetFullMessageListResponse } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service'; + +// Microsoft API limit is 1000 messages per request on this endpoint +const MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT = 1000; + +@Injectable() +export class MicrosoftGetMessageListService { + constructor( + private readonly microsoftClientProvider: MicrosoftClientProvider, + ) {} + + public async getFullMessageList( + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'provider' | 'refreshToken' | 'id' + >, + syncCursor?: string, + ): Promise { + const messageExternalIds: string[] = []; + + const microsoftClient = + await this.microsoftClientProvider.getMicrosoftClient(connectedAccount); + + const response: PageCollection = await microsoftClient + .api(syncCursor || '/me/mailfolders/inbox/messages/delta?$select=id') + .version('beta') + .headers({ + Prefer: `odata.maxpagesize=${MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT}`, + }) + .get(); + + const callback: PageIteratorCallback = (data) => { + messageExternalIds.push(data.id); + + return true; + }; + + const pageIterator = new PageIterator(microsoftClient, response, callback); + + await pageIterator.iterate(); + + return { + messageExternalIds: messageExternalIds, + nextSyncCursor: pageIterator.getDeltaLink() || '', + }; + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts index c13c9485c..d5320042f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/messaging-import-manager.module.ts @@ -18,6 +18,7 @@ import { MessagingMessageListFetchCronJob } from 'src/modules/messaging/message- import { MessagingMessagesImportCronJob } from 'src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job'; import { MessagingOngoingStaleCronJob } from 'src/modules/messaging/message-import-manager/crons/jobs/messaging-ongoing-stale.cron.job'; import { MessagingGmailDriverModule } from 'src/modules/messaging/message-import-manager/drivers/gmail/messaging-gmail-driver.module'; +import { MessagingMicrosoftDriverModule } from 'src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module'; import { MessagingAddSingleMessageToCacheForImportJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job'; import { MessagingCleanCacheJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-clean-cache'; import { MessagingMessageListFetchJob } from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job'; @@ -40,6 +41,7 @@ import { MessagingMonitoringModule } from 'src/modules/messaging/monitoring/mess RefreshAccessTokenManagerModule, WorkspaceDataSourceModule, MessagingGmailDriverModule, + MessagingMicrosoftDriverModule, MessagingCommonModule, TypeOrmModule.forFeature([Workspace], 'core'), TypeOrmModule.forFeature([DataSourceEntity], 'metadata'), diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts index c6bf2e68b..6fe1e5924 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { GmailGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service'; +import { MicrosoftGetMessageListService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service'; import { MessageImportException, MessageImportExceptionCode, @@ -22,6 +23,7 @@ export type GetPartialMessageListResponse = { export class MessagingGetMessageListService { constructor( private readonly gmailGetMessageListService: GmailGetMessageListService, + private readonly microsoftGetMessageListService: MicrosoftGetMessageListService, ) {} public async getFullMessageList( @@ -36,11 +38,9 @@ export class MessagingGetMessageListService { connectedAccount, ); case 'microsoft': - // TODO: Placeholder - return { - messageExternalIds: [], - nextSyncCursor: '', - }; + return this.microsoftGetMessageListService.getFullMessageList( + connectedAccount, + ); default: throw new MessageImportException( `Provider ${connectedAccount.provider} is not supported`, diff --git a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx index 52dce83f3..cbdfe2f81 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx @@ -54,6 +54,59 @@ Register the following recurring jobs: yarn command:prod cron:messaging:messages-import yarn command:prod cron:messaging:message-list-fetch yarn command:prod cron:calendar:calendar-event-list-fetch +yarn command:prod cron:calendar:calendar-event-import +yarn command:prod cron:messaging:ongoing-stale +yarn command:prod cron:calendar:ongoing-stale +``` + +## For Outlook and Outlook Calendar (Microsoft 365) + +### Create a project in Microsoft Azure + +You will need to create a project in [Microsoft Azure](https://portal.azure.com/#view/Microsoft_AAD_IAM/AppGalleryBladeV2) and get the credentials. + +Then you can set the following environment variables: + +- `AUTH_MICROSOFT_ENABLED=true` +- `AUTH_MICROSOFT_CLIENT_ID=` +- `AUTH_MICROSOFT_TENANT_ID=` +- `AUTH_MICROSOFT_CLIENT_SECRET=` +- `AUTH_MICROSOFT_CALLBACK_URL=https:///auth/microsoft/redirect` if you want to use Microsoft SSO +- `AUTH_MICROSOFT_APIS_CALLBACK_URL=https:///auth/microsoft-apis/get-access-token` + +### Enable APIs + +On Microsoft Azure Console enable the following APIs in "Permissions": + +- Microsoft Graph: Mail.Read +- Microsoft Graph: Calendars.Read +- Microsoft Graph: User.Read.All +- Microsoft Graph: openid +- Microsoft Graph: email +- Microsoft Graph: profile +- Microsoft Graph: offline_access + +### Authorized redirect URIs + +You need to add the following redirect URIs to your project: +- `https:///auth/microsoft/redirect` if you want to use Microsoft SSO +- `https:///auth/microsoft-apis/get-access-token` + +### If your app is in test mode + +If your app is in test mode, you will need to add test users to your project. + +Add your test users to the "Users and groups" section. + +### Start the cron jobs + +Register the following recurring jobs: +``` +# from your worker container +yarn command:prod cron:messaging:messages-import +yarn command:prod cron:messaging:message-list-fetch +yarn command:prod cron:calendar:calendar-event-list-fetch +yarn command:prod cron:calendar:calendar-event-import yarn command:prod cron:messaging:ongoing-stale yarn command:prod cron:calendar:ongoing-stale ```