From f077efd171af678173abea8e92cc226d0db6f122 Mon Sep 17 00:00:00 2001 From: Guillim Date: Thu, 16 Jan 2025 09:50:01 +0100 Subject: [PATCH] Outlook integration (#9631) Get Partial messages --- .../workspace/message-channels.ts | 3 + .../auth/services/microsoft-apis.service.ts | 44 +++++++------- .../message-import-driver.exception.ts | 1 + .../gmail-get-message-list.service.ts | 2 +- ...osoft-get-message-list.service.dev.spec.ts | 24 ++++++++ .../microsoft-get-message-list.service.ts | 59 ++++++++++++++++++- ...essage-import-exception-handler.service.ts | 23 ++++++++ .../messaging-get-message-list.service.ts | 11 ++-- .../services/messaging-message.service.ts | 3 +- .../display/icon/assets/microsoft-outlook.svg | 8 +++ .../icon/components/IconMicrosoftOutlook.tsx | 14 +++++ packages/twenty-ui/src/display/index.ts | 1 + .../content/developers/self-hosting/setup.mdx | 4 +- 13 files changed, 165 insertions(+), 32 deletions(-) create mode 100644 packages/twenty-ui/src/display/icon/assets/microsoft-outlook.svg create mode 100644 packages/twenty-ui/src/display/icon/components/IconMicrosoftOutlook.tsx diff --git a/packages/twenty-server/src/database/typeorm-seeds/workspace/message-channels.ts b/packages/twenty-server/src/database/typeorm-seeds/workspace/message-channels.ts index e27909ec5..f948b5d9d 100644 --- a/packages/twenty-server/src/database/typeorm-seeds/workspace/message-channels.ts +++ b/packages/twenty-server/src/database/typeorm-seeds/workspace/message-channels.ts @@ -44,6 +44,7 @@ export const seedMessageChannel = async ( type: 'email', connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.TIM, handle: 'tim@apple.dev', + isSyncEnabled: false, visibility: MessageChannelVisibility.SHARE_EVERYTHING, syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING, }, @@ -56,6 +57,7 @@ export const seedMessageChannel = async ( type: 'email', connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.JONY, handle: 'jony.ive@apple.dev', + isSyncEnabled: false, visibility: MessageChannelVisibility.SHARE_EVERYTHING, syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING, }, @@ -68,6 +70,7 @@ export const seedMessageChannel = async ( type: 'email', connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.PHIL, handle: 'phil.schiler@apple.dev', + isSyncEnabled: false, visibility: MessageChannelVisibility.SHARE_EVERYTHING, syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING, }, diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts index b8f619722..c25cf99d0 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts @@ -28,12 +28,18 @@ import { MessageChannelVisibility, MessageChannelWorkspaceEntity, } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { + MessagingMessageListFetchJob, + MessagingMessageListFetchJobData, +} from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @Injectable() export class MicrosoftAPIsService { constructor( private readonly twentyORMGlobalManager: TwentyORMGlobalManager, + @InjectMessageQueue(MessageQueue.messagingQueue) + private readonly messageQueueService: MessageQueueService, @InjectMessageQueue(MessageQueue.calendarQueue) private readonly calendarQueueService: MessageQueueService, private readonly accountsToReconnectService: AccountsToReconnectService, @@ -102,7 +108,6 @@ export class MicrosoftAPIsService { manager, ); - // TODO: Modify this when the email sync is implemented await messageChannelRepository.save( { id: v4(), @@ -111,8 +116,7 @@ export class MicrosoftAPIsService { handle, visibility: messageVisibility || MessageChannelVisibility.SHARE_EVERYTHING, - syncStatus: MessageChannelSyncStatus.NOT_SYNCED, - syncStage: MessageChannelSyncStage.FAILED, + syncStatus: MessageChannelSyncStatus.ONGOING, }, {}, manager, @@ -160,14 +164,13 @@ export class MicrosoftAPIsService { newOrExistingConnectedAccountId, ); - // TODO: Modify this when the email sync is implemented await messageChannelRepository.update( { connectedAccountId: newOrExistingConnectedAccountId, }, { - syncStage: MessageChannelSyncStage.FAILED, - syncStatus: MessageChannelSyncStatus.NOT_SYNCED, + syncStage: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING, + syncStatus: MessageChannelSyncStatus.ONGOING, syncCursor: '', syncStageStartedAt: null, }, @@ -176,22 +179,21 @@ export class MicrosoftAPIsService { } }); - // TODO: Uncomment this when the email sync is implemented - // const messageChannels = await messageChannelRepository.find({ - // where: { - // connectedAccountId: newOrExistingConnectedAccountId, - // }, - // }); + const messageChannels = await messageChannelRepository.find({ + where: { + connectedAccountId: newOrExistingConnectedAccountId, + }, + }); - // for (const messageChannel of messageChannels) { - // await this.messageQueueService.add( - // MessagingMessageListFetchJob.name, - // { - // workspaceId, - // messageChannelId: messageChannel.id, - // }, - // ); - // } + for (const messageChannel of messageChannels) { + await this.messageQueueService.add( + MessagingMessageListFetchJob.name, + { + workspaceId, + messageChannelId: messageChannel.id, + }, + ); + } const calendarChannels = await calendarChannelRepository.find({ where: { diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts index 600b66853..a0fc4cb04 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts @@ -14,4 +14,5 @@ export enum MessageImportDriverExceptionCode { UNKNOWN = 'UNKNOWN', UNKNOWN_NETWORK_ERROR = 'UNKNOWN_NETWORK_ERROR', NO_NEXT_SYNC_CURSOR = 'NO_NEXT_SYNC_CURSOR', + SYNC_CURSOR_ERROR = 'SYNC_CURSOR_ERROR', } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts index 1dfa2bc0a..ebbb11a42 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts @@ -32,7 +32,7 @@ export class GmailGetMessageListService { public async getFullMessageList( connectedAccount: Pick< ConnectedAccountWorkspaceEntity, - 'provider' | 'refreshToken' | 'id' + 'provider' | 'refreshToken' | 'id' | 'handle' >, ): Promise { const gmailClient = diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts index 76f98385a..0d506b4d6 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts @@ -10,6 +10,7 @@ import { MicrosoftGetMessageListService } from './microsoft-get-message-list.ser import { MicrosoftHandleErrorService } from './microsoft-handle-error.service'; const refreshToken = 'replace-with-your-refresh-token'; +const syncCursor = 'replace-with-your-sync-cursor'; xdescribe('Microsoft dev tests : get message list service', () => { let service: MicrosoftGetMessageListService; @@ -54,4 +55,27 @@ xdescribe('Microsoft dev tests : get message list service', () => { service.getFullMessageList(mockConnectedAccountUnvalid), ).rejects.toThrowError('Access token is undefined or empty'); }); + + it('Should fetch and return partial message list successfully', async () => { + const result = await service.getPartialMessageList( + mockConnectedAccount, + syncCursor, + ); + + expect(result.nextSyncCursor).toBeTruthy(); + }); + + it('Should fail partial message if syncCursor is invalid', async () => { + await expect( + service.getPartialMessageList(mockConnectedAccount, 'invalid-syncCursor'), + ).rejects.toThrowError( + /Resource not found for the segment|Badly formed content/g, + ); + }); + + it('Should fail partial message if syncCursor is missing', async () => { + await expect( + service.getPartialMessageList(mockConnectedAccount, ''), + ).rejects.toThrowError(/Missing SyncCursor/g); + }); }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts index b271270e2..4c82f0c7c 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -7,8 +7,15 @@ import { } from '@microsoft/microsoft-graph-client'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { + MessageImportDriverException, + MessageImportDriverExceptionCode, +} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; -import { GetFullMessageListResponse } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service'; +import { + GetFullMessageListResponse, + GetPartialMessageListResponse, +} from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service'; // Microsoft API limit is 1000 messages per request on this endpoint const MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT = 1000; @@ -54,4 +61,54 @@ export class MicrosoftGetMessageListService { nextSyncCursor: pageIterator.getDeltaLink() || '', }; } + + public async getPartialMessageList( + connectedAccount: Pick< + ConnectedAccountWorkspaceEntity, + 'provider' | 'refreshToken' | 'id' + >, + syncCursor: string, + ): Promise { + // important: otherwise tries to get the full message list + if (!syncCursor) { + throw new MessageImportDriverException( + 'Missing SyncCursor', + MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR, + ); + } + + const messageExternalIds: string[] = []; + const messageExternalIdsToDelete: string[] = []; + + const microsoftClient = + await this.microsoftClientProvider.getMicrosoftClient(connectedAccount); + + const response: PageCollection = await microsoftClient + .api(syncCursor) + .version('beta') + .headers({ + Prefer: `odata.maxpagesize=${MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT}, IdType="ImmutableId"`, + }) + .get(); + + const callback: PageIteratorCallback = (data) => { + if (data['@removed']) { + messageExternalIdsToDelete.push(data.id); + } else { + messageExternalIds.push(data.id); + } + + return true; + }; + + const pageIterator = new PageIterator(microsoftClient, response, callback); + + await pageIterator.iterate(); + + return { + messageExternalIds, + messageExternalIdsToDelete, + nextSyncCursor: pageIterator.getDeltaLink() || '', + }; + } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/message-import-exception-handler.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/message-import-exception-handler.service.ts index f67b03eb6..faaccee96 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/message-import-exception-handler.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/message-import-exception-handler.service.ts @@ -64,6 +64,13 @@ export class MessageImportExceptionHandlerService { workspaceId, ); break; + case MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR: + await this.handlePermanentException( + exception, + messageChannel, + workspaceId, + ); + break; default: throw exception; } @@ -149,6 +156,22 @@ export class MessageImportExceptionHandlerService { ); } + private async handlePermanentException( + exception: MessageImportDriverException, + messageChannel: Pick, + workspaceId: string, + ): Promise { + await this.messageChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport( + [messageChannel.id], + workspaceId, + ); + + throw new MessageImportException( + `Permanent error occurred while importing messages for message channel ${messageChannel.id} in workspace ${workspaceId}: ${exception.message}`, + MessageImportExceptionCode.UNKNOWN, + ); + } + private async handleNotFoundException( syncStep: MessageImportSyncStep, messageChannel: Pick, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts index 6fe1e5924..a0308572f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts @@ -29,7 +29,7 @@ export class MessagingGetMessageListService { public async getFullMessageList( connectedAccount: Pick< ConnectedAccountWorkspaceEntity, - 'provider' | 'refreshToken' | 'id' + 'provider' | 'refreshToken' | 'id' | 'handle' >, ): Promise { switch (connectedAccount.provider) { @@ -63,11 +63,10 @@ export class MessagingGetMessageListService { syncCursor, ); case 'microsoft': - return { - messageExternalIds: [], - messageExternalIdsToDelete: [], - nextSyncCursor: '', - }; + return this.microsoftGetMessageListService.getPartialMessageList( + connectedAccount, + syncCursor, + ); default: throw new MessageImportException( `Provider ${connectedAccount.provider} is not supported`, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts index f8e5e02c2..8e26eec36 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts @@ -58,13 +58,14 @@ export class MessagingMessageService { }); if (existingMessage) { - await messageChannelMessageAssociationRepository.insert( + await messageChannelMessageAssociationRepository.upsert( { messageChannelId, messageId: existingMessage.id, messageExternalId: message.externalId, messageThreadExternalId: message.messageThreadExternalId, }, + ['messageChannelId', 'messageExternalId'], transactionManager, ); diff --git a/packages/twenty-ui/src/display/icon/assets/microsoft-outlook.svg b/packages/twenty-ui/src/display/icon/assets/microsoft-outlook.svg new file mode 100644 index 000000000..8e0cdce35 --- /dev/null +++ b/packages/twenty-ui/src/display/icon/assets/microsoft-outlook.svg @@ -0,0 +1,8 @@ + + +file_type_outlook + + + + + \ No newline at end of file diff --git a/packages/twenty-ui/src/display/icon/components/IconMicrosoftOutlook.tsx b/packages/twenty-ui/src/display/icon/components/IconMicrosoftOutlook.tsx new file mode 100644 index 000000000..153d0af09 --- /dev/null +++ b/packages/twenty-ui/src/display/icon/components/IconMicrosoftOutlook.tsx @@ -0,0 +1,14 @@ +import { useTheme } from '@emotion/react'; + +import IconMicrosoftOutlookRaw from '../assets/microsoft-outlook.svg?react'; + +interface IconMicrosoftOutlookProps { + size?: number; +} + +export const IconMicrosoftOutlook = (props: IconMicrosoftOutlookProps) => { + const theme = useTheme(); + const size = props.size ?? theme.icon.size.lg; + + return ; +}; diff --git a/packages/twenty-ui/src/display/index.ts b/packages/twenty-ui/src/display/index.ts index 3b009a351..85aea3e8b 100644 --- a/packages/twenty-ui/src/display/index.ts +++ b/packages/twenty-ui/src/display/index.ts @@ -16,6 +16,7 @@ export * from './icon/components/IconGoogle'; export * from './icon/components/IconGoogleCalendar'; export * from './icon/components/IconLock'; export * from './icon/components/IconMicrosoft'; +export * from './icon/components/IconMicrosoftOutlook'; export * from './icon/components/IconRelationManyToOne'; export * from './icon/components/IconTwentyStar'; export * from './icon/components/IconTwentyStarFilled'; diff --git a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx index 434c5f875..20c81da72 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/setup.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/setup.mdx @@ -54,7 +54,7 @@ Register the following recurring jobs: yarn command:prod cron:messaging:messages-import yarn command:prod cron:messaging:message-list-fetch yarn command:prod cron:calendar:calendar-event-list-fetch -yarn command:prod cron:calendar:calendar-event-import +yarn command:prod cron:calendar:calendar-events-import yarn command:prod cron:messaging:ongoing-stale yarn command:prod cron:calendar:ongoing-stale ``` @@ -110,7 +110,7 @@ Register the following recurring jobs: yarn command:prod cron:messaging:messages-import yarn command:prod cron:messaging:message-list-fetch yarn command:prod cron:calendar:calendar-event-list-fetch -yarn command:prod cron:calendar:calendar-event-import +yarn command:prod cron:calendar:calendar-events-import yarn command:prod cron:messaging:ongoing-stale yarn command:prod cron:calendar:ongoing-stale ```