From fc484bde2d43ae232c24c7d3ff76c8174f4474a1 Mon Sep 17 00:00:00 2001 From: Guillim Date: Wed, 15 Jan 2025 09:48:57 +0100 Subject: [PATCH] [Outlook integration] part 2 : GetMessages (#9612) ### Introducing - mock files in order to setup unit test on parsing outlook messages - special spec files for development purposes : dev.spec files. They are CI skipped with xdescribe but very useful for iterating on new messages format - main functionality : getMessages. We use microsoft default client to do so, using the $batch endpoint to group calls by 20 ### documentation final touch to add troubleshooting tips --- .../services/email-alias-manager.service.ts | 2 + .../services/refresh-access-token.service.ts | 57 +++-- .../messaging-microsoft-driver.module.ts | 12 +- .../microsoft/mocks/microsoft-api-examples.ts | 241 ++++++++++++++++++ .../microsoft-fetch-by-batch.service.ts | 57 +++++ ...osoft-get-message-list.service.dev.spec.ts | 57 +++++ .../microsoft-get-message-list.service.ts | 2 +- .../microsoft-get-messages.interface.ts | 45 ++++ ...microsoft-get-messages.service.dev.spec.ts | 58 +++++ .../microsoft-get-messages.service.spec.ts | 174 +++++++++++++ .../microsoft-get-messages.service.ts | 141 ++++++++++ .../microsoft-handle-error.service.ts | 32 +++ .../messaging-get-messages.service.ts | 8 + .../content/developers/self-hosting/setup.mdx | 4 + .../self-hosting/troubleshooting.mdx | 4 + 15 files changed, 867 insertions(+), 27 deletions(-) create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/mocks/microsoft-api-examples.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts diff --git a/packages/twenty-server/src/modules/connected-account/email-alias-manager/services/email-alias-manager.service.ts b/packages/twenty-server/src/modules/connected-account/email-alias-manager/services/email-alias-manager.service.ts index e4ec7e083..e4a583e92 100644 --- a/packages/twenty-server/src/modules/connected-account/email-alias-manager/services/email-alias-manager.service.ts +++ b/packages/twenty-server/src/modules/connected-account/email-alias-manager/services/email-alias-manager.service.ts @@ -17,6 +17,8 @@ export class EmailAliasManagerService { let handleAliases: string[]; switch (connectedAccount.provider) { + case 'microsoft': + return; case 'google': handleAliases = await this.googleEmailAliasManagerService.getHandleAliases( diff --git a/packages/twenty-server/src/modules/connected-account/refresh-access-token-manager/services/refresh-access-token.service.ts b/packages/twenty-server/src/modules/connected-account/refresh-access-token-manager/services/refresh-access-token.service.ts index 853401356..1e2cc7306 100644 --- a/packages/twenty-server/src/modules/connected-account/refresh-access-token-manager/services/refresh-access-token.service.ts +++ b/packages/twenty-server/src/modules/connected-account/refresh-access-token-manager/services/refresh-access-token.service.ts @@ -20,6 +20,7 @@ export class RefreshAccessTokenService { workspaceId: string, ): Promise { const refreshToken = connectedAccount.refreshToken; + let accessToken: string; if (!refreshToken) { throw new RefreshAccessTokenException( @@ -28,33 +29,39 @@ export class RefreshAccessTokenService { ); } - let accessToken: string; + switch (connectedAccount.provider) { + case 'microsoft': + return ''; + case 'google': { + try { + accessToken = await this.refreshAccessToken( + connectedAccount, + refreshToken, + ); + } catch (error) { + throw new RefreshAccessTokenException( + `Error refreshing access token for connected account ${connectedAccount.id} in workspace ${workspaceId}: ${error.message}`, + RefreshAccessTokenExceptionCode.REFRESH_ACCESS_TOKEN_FAILED, + ); + } - try { - accessToken = await this.refreshAccessToken( - connectedAccount, - refreshToken, - ); - } catch (error) { - throw new RefreshAccessTokenException( - `Error refreshing access token for connected account ${connectedAccount.id} in workspace ${workspaceId}: ${error.message}`, - RefreshAccessTokenExceptionCode.REFRESH_ACCESS_TOKEN_FAILED, - ); + const connectedAccountRepository = + await this.twentyORMManager.getRepository( + 'connectedAccount', + ); + + await connectedAccountRepository.update( + { id: connectedAccount.id }, + { + accessToken, + }, + ); + + return accessToken; + } + default: + throw new Error('Provider not supported for access token refresh'); } - - const connectedAccountRepository = - await this.twentyORMManager.getRepository( - 'connectedAccount', - ); - - await connectedAccountRepository.update( - { id: connectedAccount.id }, - { - accessToken, - }, - ); - - return accessToken; } async refreshAccessToken( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts index e3afd094e..2c19db33f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/messaging-microsoft-driver.module.ts @@ -8,6 +8,9 @@ import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-accou import { OAuth2ClientManagerModule } from 'src/modules/connected-account/oauth2-client-manager/oauth2-client-manager.module'; import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { MicrosoftFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service'; +import { MicrosoftGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service'; +import { MicrosoftHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service'; import { MicrosoftGetMessageListService } from './services/microsoft-get-message-list.service'; @@ -23,8 +26,15 @@ import { MicrosoftGetMessageListService } from './services/microsoft-get-message providers: [ MicrosoftClientProvider, MicrosoftGetMessageListService, + MicrosoftGetMessagesService, + MicrosoftFetchByBatchService, + MicrosoftHandleErrorService, MicrosoftOAuth2ClientManagerService, ], - exports: [MicrosoftGetMessageListService, MicrosoftClientProvider], + exports: [ + MicrosoftGetMessageListService, + MicrosoftClientProvider, + MicrosoftGetMessagesService, + ], }) export class MessagingMicrosoftDriverModule {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/mocks/microsoft-api-examples.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/mocks/microsoft-api-examples.ts new file mode 100644 index 000000000..0e08ccb67 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/mocks/microsoft-api-examples.ts @@ -0,0 +1,241 @@ +export const microsoftGraphWithMessages = { + '@odata.context': + 'https://graph.microsoft.com/beta/$metadata#Collection(message)', + value: [ + { + '@odata.type': '#microsoft.graph.message', + '@odata.etag': 'W/"CQAAABYAAAAadQ+1xAL8SLCZzf1KYyk+AAACItFa"', + id: 'AAMkAGZlMDQ1NjU5LTUzN2UtNDAyMC1hNmVlLTZhZmExMGU3ZDU1NwBGAAAAAADzAhgkpMbwQYnkXH1D-Va3BwAadQ_1xAL8SLCZzf1KYyk_AAAAAAEMAAAadQ_1xAL8SLCZzf1KYyk_AAACJSmSAAA=', + }, + ], + '@odata.nextLink': + "https://graph.microsoft.com/beta/me/mailFolders('inbox')/messages/delta?$skiptoken=jWnSM_TVmEdmKBzfVjDdNbDwpt3yYSUqEf9CFdhRcTxhbogC9oaTvY1ZdONMplHuz0pwtPay_qkEcFQ5RLEuDZ3O6IgnI5FXRcfekzOECWlL7zRVdGBidZ5TkXmXV7O7P8cxtvBMFJ2_dV951teFMatpdnD6hvksBK0Ff4tJKfo.HvZwAw_DM9PR3xf90ThtbqSdMCkGCHNPkjpaedxSBN3", +}; + +export const microsoftGraphBatchWithTwoMessagesResponse = [ + { + responses: [ + { + id: '2', + status: 200, + headers: { + 'Cache-Control': 'private', + 'Content-Type': + 'application/json; odata.metadata=minimal; odata.streaming=true; IEEE754Compatible=false; charset=utf-8', + }, + body: { + '@odata.context': + "https://graph.microsoft.com/v1.0/$metadata#users('599cbaf7-873d-4b6f-b374-f87d3605e9e0')/messages/$entity", + '@odata.etag': 'W/"CQAAABYAAAAadQ+1xAL8SLCZzf1KYyk+AAACItFa"', + id: 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAAiVYkAAA', + createdDateTime: '2025-01-10T13:31:37Z', + lastModifiedDateTime: '2025-01-10T13:31:38Z', + changeKey: 'CQAAABYAAAAadQ+1xAL8SLCZzf1KYyk+AAACItFa', + categories: [], + receivedDateTime: '2025-01-10T13:31:37Z', + sentDateTime: '2025-01-10T13:31:34Z', + hasAttachments: true, + internetMessageId: + '', + subject: 'test email John: number 4', + bodyPreview: 'test 4', + importance: 'normal', + parentFolderId: + 'AAMkAGZlMDQ1NjU5LTUzN2UtNDAyMC1hNmVlLTZhZmExMGU3ZDU1NwAuAAAAAADzAhgkpMbwQYnkXH1D-Va3AQAadQ_1xAL8SLCZzf1KYyk_AAAAAAEMAAA=', + conversationId: + 'AAQkAGZlMDQ1NjU5LTUzN2UtNDAyMC1hNmVlLTZhZmExMGU3ZDU1NwAQAAZhOZ86nXZElRkxyGJRiY8=', + conversationIndex: 'AQHbY2PrBmE5nzqddkSVGTHIYlGJjw==', + isDeliveryReceiptRequested: null, + isReadReceiptRequested: false, + isRead: false, + isDraft: false, + webLink: + 'https://outlook.office365.com/owa/?ItemID=AAkALgAAAAAAHYQDEapmEc2byACqAC%2FEWg0AGnUPtcQC%2FEiwmc39SmMpPgAAAiVYkAAA&exvsurl=1&viewmodel=ReadMessageItem', + inferenceClassification: 'focused', + body: { + contentType: 'text', + content: 'plain text format test 4', + }, + sender: { + emailAddress: { + name: 'John l', + address: 'John.l@outlook.fr', + }, + }, + from: { + emailAddress: { + name: 'John l', + address: 'John.l@outlook.fr', + }, + }, + toRecipients: [ + { + emailAddress: { + name: 'Walker', + address: 'walker@felixacme.onmicrosoft.com', + }, + }, + ], + ccRecipients: [], + bccRecipients: [], + replyTo: [], + flag: { + flagStatus: 'notFlagged', + }, + }, + }, + { + id: '1', + status: 200, + headers: { + 'Cache-Control': 'private', + 'Content-Type': + 'application/json; odata.metadata=minimal; odata.streaming=true; IEEE754Compatible=false; charset=utf-8', + }, + body: { + '@odata.context': + "https://graph.microsoft.com/v1.0/$metadata#users('599cbaf7-873d-4b6f-b374-f87d3605e9e0')/messages/$entity", + '@odata.etag': 'W/"CQAAABYAAAAadQ+1xAL8SLCZzf1KYyk+AAADw4Em"', + id: 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAA8ZAfgAA', + createdDateTime: '2025-01-13T09:38:06Z', + lastModifiedDateTime: '2025-01-13T11:50:48Z', + changeKey: 'CQAAABYAAAAadQ+1xAL8SLCZzf1KYyk+AAADw4Em', + categories: [], + receivedDateTime: '2025-01-13T09:38:06Z', + sentDateTime: '2025-01-13T09:38:01Z', + hasAttachments: false, + internetMessageId: + '', + subject: 'test subject', + bodyPreview: 'You now have 2 licenses', + importance: 'normal', + parentFolderId: + 'AAMkAGZlMDQ1NjU5LTUzN2UtNDAyMC1hNmVlLTZhZmExMGU3ZDU1NwAuAAAAAADzAhgkpMbwQYnkXH1D-Va3AQAadQ_1xAL8SLCZzf1KYyk_AAAAAAEMAAA=', + conversationId: + 'AAQkAGZlMDQ1NjU5LTUzN2UtNDAyMC1hNmVlLTZhZmExMGU3ZDU1NwAQADz34qcnxpxEidnAJbZA-OI=', + conversationIndex: 'AQHbZZ7ZPPfipyfGnESJ2cAltkD84g==', + isDeliveryReceiptRequested: null, + isReadReceiptRequested: false, + isRead: false, + isDraft: false, + webLink: + 'https://outlook.office365.com/owa/?ItemID=AAkALgAAAAAAHYQDEapmEc2byACqAC%2FEWg0AGnUPtcQC%2FEiwmc39SmMpPgAAA8ZAfgAA&exvsurl=1&viewmodel=ReadMessageItem', + inferenceClassification: 'focused', + body: { + contentType: 'text', + content: 'You will send a message in the plain text format', + }, + sender: { + emailAddress: { + name: 'Microsoft', + address: 'microsoft-noreply@microsoft.com', + }, + }, + from: { + emailAddress: { + name: 'Microsoft', + address: 'microsoft-noreply@microsoft.com', + }, + }, + toRecipients: [ + { + emailAddress: { + name: 'Walker', + address: 'walker@felixacme.onmicrosoft.com', + }, + }, + ], + ccRecipients: [ + { + emailAddress: { + name: 'Antoine', + address: 'antoine@gmail.com', + }, + }, + { + emailAddress: { + name: 'Cyril@acme2.com', + address: 'cyril@acme2.com', + }, + }, + ], + bccRecipients: [], + replyTo: [], + flag: { + flagStatus: 'notFlagged', + }, + }, + }, + ], + }, +]; + +export const microsoftGraphBatchWithHtmlMessagesResponse = [ + { + responses: [ + { + id: '2', + status: 200, + headers: { + 'Cache-Control': 'private', + 'Content-Type': + 'application/json; odata.metadata=minimal; odata.streaming=true; IEEE754Compatible=false; charset=utf-8', + }, + body: { + '@odata.context': + "https://graph.microsoft.com/v1.0/$metadata#users('599cbaf7-873d-4b6f-b374-f87d3605e9e0')/messages/$entity", + '@odata.etag': 'W/"CQAAABYAAAAadQ+1xAL8SLCZzf1KYyk+AAACItFa"', + id: 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAAiVYkAAA', + createdDateTime: '2025-01-10T13:31:37Z', + lastModifiedDateTime: '2025-01-10T13:31:38Z', + changeKey: 'CQAAABYAAAAadQ+1xAL8SLCZzf1KYyk+AAACItFa', + categories: [], + receivedDateTime: '2025-01-10T13:31:37Z', + sentDateTime: '2025-01-10T13:31:34Z', + hasAttachments: true, + internetMessageId: + '', + subject: 'test email John: number 5', + bodyPreview: 'test 5', + importance: 'normal', + parentFolderId: + 'AAMkAGZlMDQ1NjU5LTUzN2UtNDAyMC1hNmVlLTZhZmExMGU3ZDU1NwAuAAAAAADzAhgkpMbwQYnkXH1D-Va3AQAadQ_1xAL8SLCZzf1KYyk_AAAAAAEMAAA=', + conversationId: + 'AAQkAGZlMDQ1NjU5LTUzN2UtNDAyMC1hNmVlLTZhZmExMGU3ZDU1NwAQAAZhOZ86nXZElRkxyGJRiY9=', + conversationIndex: 'AQHbY2PrBmE5nzqddkSVGTHIYlGJjr==', + isDeliveryReceiptRequested: null, + isReadReceiptRequested: false, + isRead: false, + isDraft: false, + webLink: + 'https://outlook.office365.com/owa/?ItemID=AAkALgAAAAAAHYQDEapmEc2byACqAC%2FEWg0AGnUPtcQC%2FEiwmc39SmMpPgAAAiVYkAAA&exvsurl=1&viewmodel=ReadMessageItem', + inferenceClassification: 'focused', + body: { + contentType: 'html', + content: + '\r\n

 
test 4
', + }, + sender: { + emailAddress: { + name: 'John l', + address: 'John.l@outlook.fr', + }, + }, + from: { + emailAddress: { + name: 'John l', + address: 'John.l@outlook.fr', + }, + }, + toRecipients: [], + ccRecipients: [], + bccRecipients: [], + replyTo: [], + flag: { + flagStatus: 'notFlagged', + }, + }, + }, + ], + }, +]; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts new file mode 100644 index 000000000..3f4ae6a0b --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts @@ -0,0 +1,57 @@ +import { Injectable } from '@nestjs/common'; + +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { MicrosoftGraphBatchResponse } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface'; + +@Injectable() +export class MicrosoftFetchByBatchService { + constructor( + private readonly microsoftClientProvider: MicrosoftClientProvider, + ) {} + + async fetchAllByBatches( + messageIds: string[], + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'refreshToken' | 'id' + >, + ): Promise<{ + messageIdsByBatch: string[][]; + batchResponses: MicrosoftGraphBatchResponse[]; + }> { + const batchLimit = 20; + const batchResponses: MicrosoftGraphBatchResponse[] = []; + const messageIdsByBatch: string[][] = []; + + const client = + await this.microsoftClientProvider.getMicrosoftClient(connectedAccount); + + for (let i = 0; i < messageIds.length; i += batchLimit) { + const batchMessageIds = messageIds.slice(i, i + batchLimit); + + messageIdsByBatch.push(batchMessageIds); + + const batchRequests = batchMessageIds.map((messageId, index) => ({ + id: (index + 1).toString(), + method: 'GET', + url: `/me/messages/${messageId}`, + headers: { + 'Content-Type': 'application/json', + Prefer: 'outlook.body-content-type="text"', + }, + })); + + const batchResponse = await client + .api('/$batch') + .post({ requests: batchRequests }); + + batchResponses.push(batchResponse); + } + + return { + messageIdsByBatch, + batchResponses, + }; + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts new file mode 100644 index 000000000..76f98385a --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts @@ -0,0 +1,57 @@ +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; +import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; +import { ConnectedAccountProvider } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; + +import { MicrosoftGetMessageListService } from './microsoft-get-message-list.service'; +import { MicrosoftHandleErrorService } from './microsoft-handle-error.service'; + +const refreshToken = 'replace-with-your-refresh-token'; + +xdescribe('Microsoft dev tests : get message list service', () => { + let service: MicrosoftGetMessageListService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [EnvironmentModule.forRoot({})], + providers: [ + MicrosoftGetMessageListService, + MicrosoftClientProvider, + MicrosoftHandleErrorService, + MicrosoftOAuth2ClientManagerService, + ConfigService, + ], + }).compile(); + + service = module.get( + MicrosoftGetMessageListService, + ); + }); + + const mockConnectedAccount = { + id: 'connected-account-id', + provider: ConnectedAccountProvider.MICROSOFT, + refreshToken: refreshToken, + }; + + it('Should fetch and return message list successfully', async () => { + const result = await service.getFullMessageList(mockConnectedAccount); + + expect(result.messageExternalIds.length).toBeGreaterThan(0); + }); + + it('Should throw token error', async () => { + const mockConnectedAccountUnvalid = { + id: 'connected-account-id', + provider: ConnectedAccountProvider.MICROSOFT, + refreshToken: 'invalid-token', + }; + + await expect( + service.getFullMessageList(mockConnectedAccountUnvalid), + ).rejects.toThrowError('Access token is undefined or empty'); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts index da526016d..b271270e2 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -35,7 +35,7 @@ export class MicrosoftGetMessageListService { .api(syncCursor || '/me/mailfolders/inbox/messages/delta?$select=id') .version('beta') .headers({ - Prefer: `odata.maxpagesize=${MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT}`, + Prefer: `odata.maxpagesize=${MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT}, IdType="ImmutableId"`, }) .get(); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts new file mode 100644 index 000000000..14dc42d51 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface.ts @@ -0,0 +1,45 @@ +export interface MicrosoftGraphBatchResponse { + responses: { + id: string; + status: number; + headers?: { + 'Cache-Control': string; + 'Content-Type': string; + }; + body?: { + '@odata.context'?: string; + '@odata.etag'?: string; + id?: string; + createdDateTime?: string; + lastModifiedDateTime?: string; + changeKey?: string; + categories?: any[]; + receivedDateTime?: string; + sentDateTime?: string; + hasAttachments?: boolean; + internetMessageId?: string; + subject?: string; + bodyPreview?: string; + importance?: string; + parentFolderId?: string; + conversationId?: string; + conversationIndex?: string; + isDeliveryReceiptRequested?: boolean | null; + isReadReceiptRequested?: boolean; + isRead?: boolean; + isDraft?: boolean; + webLink?: string; + inferenceClassification?: string; + body?: { + contentType?: string; + content?: string; + }; + sender?: { + emailAddress?: { + name?: string; + address?: string; + }; + }; + }; + }[]; +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts new file mode 100644 index 000000000..43fd7563e --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts @@ -0,0 +1,58 @@ +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; +import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; +import { ConnectedAccountProvider } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { MicrosoftFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service'; +import { MicrosoftGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service'; + +import { MicrosoftHandleErrorService } from './microsoft-handle-error.service'; + +const mockMessageIds = [ + 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAA8ZAfgAA', + 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAAiVYkAAA', +]; + +const refreshToken = 'replace-with-your-refresh-token'; + +xdescribe('Microsoft dev tests : get messages service', () => { + let service: MicrosoftGetMessagesService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [EnvironmentModule.forRoot({})], + providers: [ + MicrosoftGetMessagesService, + MicrosoftHandleErrorService, + MicrosoftClientProvider, + MicrosoftOAuth2ClientManagerService, + MicrosoftFetchByBatchService, + ConfigService, + ], + }).compile(); + + service = module.get( + MicrosoftGetMessagesService, + ); + }); + + const mockConnectedAccount = { + id: 'connected-account-id', + provider: ConnectedAccountProvider.MICROSOFT, + handle: 'John.Walker@outlook.fr', + handleAliases: '', + refreshToken: refreshToken, + }; + + it('should fetch and format messages successfully', async () => { + const result = await service.getMessages( + mockMessageIds, + mockConnectedAccount, + 'workspace-1', + ); + + expect(result).toHaveLength(1); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts new file mode 100644 index 000000000..a0652fa22 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts @@ -0,0 +1,174 @@ +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { EnvironmentModule } from 'src/engine/core-modules/environment/environment.module'; +import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; +import { ConnectedAccountProvider } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { + microsoftGraphBatchWithHtmlMessagesResponse, + microsoftGraphBatchWithTwoMessagesResponse, +} from 'src/modules/messaging/message-import-manager/drivers/microsoft/mocks/microsoft-api-examples'; +import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { MicrosoftFetchByBatchService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service'; +import { MicrosoftGraphBatchResponse } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface'; +import { MicrosoftGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service'; + +import { MicrosoftHandleErrorService } from './microsoft-handle-error.service'; + +describe('Microsoft get messages service', () => { + let service: MicrosoftGetMessagesService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [EnvironmentModule.forRoot({})], + providers: [ + MicrosoftGetMessagesService, + MicrosoftHandleErrorService, + MicrosoftClientProvider, + MicrosoftOAuth2ClientManagerService, + MicrosoftFetchByBatchService, + ConfigService, + ], + }).compile(); + + service = module.get( + MicrosoftGetMessagesService, + ); + }); + + it('Should be defined', () => { + expect(service).toBeDefined(); + }); + + it('Should format batch responses as messages', () => { + const batchResponses: MicrosoftGraphBatchResponse[] = + microsoftGraphBatchWithTwoMessagesResponse; + const connectedAccount = { + id: 'connected-account-id', + provider: ConnectedAccountProvider.MICROSOFT, + refreshToken: 'refresh-token', + handle: 'John.l@outlook.fr', + handleAliases: '', + }; + const messages = service.formatBatchResponsesAsMessages( + batchResponses, + connectedAccount, + ); + + expect(messages).toHaveLength(2); + + const responseExample1 = + microsoftGraphBatchWithTwoMessagesResponse[0].responses[0]; + + expect( + messages.find( + (message) => + message.externalId === + 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAAiVYkAAA', + ), + ).toStrictEqual({ + externalId: + 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAAiVYkAAA', + subject: responseExample1.body.subject, + receivedAt: new Date(responseExample1.body.receivedDateTime), + text: responseExample1.body.body.content, + headerMessageId: responseExample1.body.internetMessageId, + messageThreadExternalId: responseExample1.body.conversationId, + direction: 'OUTGOING', + participants: [ + { + displayName: 'John l', + handle: 'john.l@outlook.fr', + role: 'from', + }, + { + displayName: 'Walker', + handle: 'walker@felixacme.onmicrosoft.com', + role: 'to', + }, + ], + attachments: [], + }); + + const responseExample2 = + microsoftGraphBatchWithTwoMessagesResponse[0].responses[1]; + + expect( + messages.filter( + (message) => + message.externalId === + 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAA8ZAfgAA', + )[0], + ).toStrictEqual({ + externalId: + 'AAkALgAAAAAAHYQDEapmEc2byACqAC-EWg0AGnUPtcQC-Eiwmc39SmMpPgAAA8ZAfgAA', + subject: responseExample2.body.subject, + receivedAt: new Date(responseExample2.body.receivedDateTime), + text: responseExample2.body.body.content, + headerMessageId: responseExample2.body.internetMessageId, + messageThreadExternalId: responseExample2.body.conversationId, + direction: 'INCOMING', + participants: [ + { + displayName: 'Microsoft', + handle: 'microsoft-noreply@microsoft.com', + role: 'from', + }, + { + displayName: 'Walker', + handle: 'walker@felixacme.onmicrosoft.com', + role: 'to', + }, + { + displayName: 'Antoine', + handle: 'antoine@gmail.com', + role: 'cc', + }, + { + displayName: 'Cyril@acme2.com', + handle: 'cyril@acme2.com', + role: 'cc', + }, + ], + attachments: [], + }); + }); + + it('Should set empty text for html responses', () => { + const batchResponses: MicrosoftGraphBatchResponse[] = + microsoftGraphBatchWithHtmlMessagesResponse; + const connectedAccount = { + id: 'connected-account-id', + provider: ConnectedAccountProvider.MICROSOFT, + refreshToken: 'refresh-token', + handle: 'John.l@outlook.fr', + handleAliases: '', + }; + const messages = service.formatBatchResponsesAsMessages( + batchResponses, + connectedAccount, + ); + + const responseExample = + microsoftGraphBatchWithHtmlMessagesResponse[0].responses[0]; + + expect(messages[0]).toStrictEqual({ + externalId: responseExample.body.id, + subject: responseExample.body.subject, + receivedAt: new Date(responseExample.body.receivedDateTime), + text: '', + headerMessageId: responseExample.body.internetMessageId, + messageThreadExternalId: responseExample.body.conversationId, + direction: 'OUTGOING', + participants: [ + { + displayName: responseExample.body.sender.emailAddress.name, + handle: + responseExample.body.sender.emailAddress.address.toLowerCase(), + role: 'from', + }, + ], + attachments: [], + }); + }); +}); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts new file mode 100644 index 000000000..64cbcd06d --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts @@ -0,0 +1,141 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { computeMessageDirection } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-message-direction.util'; +import { MicrosoftGraphBatchResponse } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface'; +import { MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message'; +import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util'; +import { isDefined } from 'src/utils/is-defined'; + +import { MicrosoftFetchByBatchService } from './microsoft-fetch-by-batch.service'; +import { MicrosoftHandleErrorService } from './microsoft-handle-error.service'; + +type ConnectedAccountType = Pick< + ConnectedAccountWorkspaceEntity, + 'refreshToken' | 'id' | 'provider' | 'handle' | 'handleAliases' +>; + +@Injectable() +export class MicrosoftGetMessagesService { + private readonly logger = new Logger(MicrosoftGetMessagesService.name); + + constructor( + private readonly microsoftFetchByBatchService: MicrosoftFetchByBatchService, + private readonly microsoftHandleErrorService: MicrosoftHandleErrorService, + ) {} + + async getMessages( + messageIds: string[], + connectedAccount: ConnectedAccountType, + workspaceId: string, + ): Promise { + const startTime = Date.now(); + + try { + const { batchResponses } = + await this.microsoftFetchByBatchService.fetchAllByBatches( + messageIds, + connectedAccount, + ); + + const messages = this.formatBatchResponsesAsMessages( + batchResponses, + connectedAccount, + ); + + const endTime = Date.now(); + + this.logger.log( + `Messaging import for workspace ${workspaceId} and account ${ + connectedAccount.id + } fetched ${messages.length} messages in ${endTime - startTime}ms`, + ); + + return messages; + } catch (error) { + this.microsoftHandleErrorService.handleMicrosoftMessageListFetchError( + error, + ); + + return []; + } + } + + public formatBatchResponsesAsMessages( + batchResponses: MicrosoftGraphBatchResponse[], + connectedAccount: ConnectedAccountType, + ): MessageWithParticipants[] { + return batchResponses.flatMap((batchResponse) => { + return this.formatBatchResponseAsMessages( + batchResponse, + connectedAccount, + ); + }); + } + + private formatBatchResponseAsMessages( + batchResponse: MicrosoftGraphBatchResponse, + connectedAccount: ConnectedAccountType, + ): MessageWithParticipants[] { + const parsedResponses = this.parseBatchResponse(batchResponse); + + const messages = parsedResponses.map((response) => { + if ('error' in response) { + this.microsoftHandleErrorService.handleMicrosoftMessageListFetchError( + response.error, + ); + } + + const participants = [ + ...formatAddressObjectAsParticipants( + response?.from?.emailAddress, + 'from', + ), + ...formatAddressObjectAsParticipants( + response?.toRecipients?.map((recipient) => recipient.emailAddress), + 'to', + ), + ...formatAddressObjectAsParticipants( + response?.ccRecipients?.map((recipient) => recipient.emailAddress), + 'cc', + ), + ...formatAddressObjectAsParticipants( + response?.bccRecipients?.map((recipient) => recipient.emailAddress), + 'bcc', + ), + ]; + + return { + externalId: response.id, + subject: response.subject || '', + receivedAt: new Date(response.receivedDateTime), + text: + response.body?.contentType === 'text' ? response.body?.content : '', + headerMessageId: response.internetMessageId, + messageThreadExternalId: response.conversationId, + direction: computeMessageDirection( + response.from.emailAddress.address, + connectedAccount, + ), + participants: participants, + attachments: [], + }; + }); + + return messages.filter(isDefined); + } + + private parseBatchResponse(batchResponse: MicrosoftGraphBatchResponse) { + if (!batchResponse?.responses) { + return []; + } + + return batchResponse.responses.map((response: any) => { + if (response.status === 200) { + return response.body; + } + + return { error: response.error }; + }); + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts new file mode 100644 index 000000000..4a8db7921 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts @@ -0,0 +1,32 @@ +import { Injectable } from '@nestjs/common'; + +import { GraphError } from '@microsoft/microsoft-graph-client'; + +import { + MessageImportDriverException, + MessageImportDriverExceptionCode, +} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; + +@Injectable() +export class MicrosoftHandleErrorService { + public handleMicrosoftMessageListFetchError(error: GraphError): void { + if (error.statusCode === 401) { + throw new MessageImportDriverException( + 'Unauthorized access to Microsoft Graph API', + MessageImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS, + ); + } + + if (error.statusCode === 403) { + throw new MessageImportDriverException( + 'Forbidden access to Microsoft Graph API', + MessageImportDriverExceptionCode.INSUFFICIENT_PERMISSIONS, + ); + } + + throw new MessageImportDriverException( + `Microsoft Graph API error: ${error.message}`, + MessageImportDriverExceptionCode.UNKNOWN, + ); + } +} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-messages.service.ts index bbd2e980d..1542a69dd 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-messages.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-messages.service.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; import { GmailGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-messages.service'; +import { MicrosoftGetMessagesService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service'; import { MessageImportException, MessageImportExceptionCode, @@ -14,6 +15,7 @@ export type GetMessagesResponse = MessageWithParticipants[]; export class MessagingGetMessagesService { constructor( private readonly gmailGetMessagesService: GmailGetMessagesService, + private readonly microsoftGetMessagesService: MicrosoftGetMessagesService, ) {} public async getMessages( @@ -36,6 +38,12 @@ export class MessagingGetMessagesService { connectedAccount, workspaceId, ); + case 'microsoft': + return this.microsoftGetMessagesService.getMessages( + messageIds, + connectedAccount, + workspaceId, + ); default: throw new MessageImportException( `Provider ${connectedAccount.provider} is not supported`, diff --git a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx index cbdfe2f81..e7edb4588 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx @@ -61,6 +61,10 @@ yarn command:prod cron:calendar:ongoing-stale ## For Outlook and Outlook Calendar (Microsoft 365) + +Users must have a [Microsoft 365 Licence](https://admin.microsoft.com/Adminportal/Home) to be able to use the Calendar and Messaging API. They will not be able to sync their account on Twenty without one. + + ### Create a project in Microsoft Azure You will need to create a project in [Microsoft Azure](https://portal.azure.com/#view/Microsoft_AAD_IAM/AppGalleryBladeV2) and get the credentials. diff --git a/packages/twenty-website/src/content/developers/self-hosting/troubleshooting.mdx b/packages/twenty-website/src/content/developers/self-hosting/troubleshooting.mdx index bff94051c..bceabfc49 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/troubleshooting.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/troubleshooting.mdx @@ -70,6 +70,10 @@ Most of the time, it's because the `worker` is not running in the background. Tr npx nx worker twenty-server ``` +#### Cannot connect my Microsoft 365 account + +Most of the time, it's because your admin has not enabled the Microsoft 365 Licence for your account. Check [https://admin.microsoft.com/](https://admin.microsoft.com/Adminportal/Home). + #### While running `yarn` warnings appear in console Warnings are informing about pulling additional dependencies which aren't explicitly stated in `package.json`, so as long as no breaking error appears, everything should work as expected.