Outlook integration (#9631)

Get Partial messages

packages/twenty-server/src/database/typeorm-seeds/workspace/message-channels.ts

@@ -44,6 +44,7 @@ export const seedMessageChannel = async (
         type: 'email',
         connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.TIM,
         handle: 'tim@apple.dev',
+        isSyncEnabled: false,
         visibility: MessageChannelVisibility.SHARE_EVERYTHING,
         syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
       },
@@ -56,6 +57,7 @@ export const seedMessageChannel = async (
         type: 'email',
         connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.JONY,
         handle: 'jony.ive@apple.dev',
+        isSyncEnabled: false,
         visibility: MessageChannelVisibility.SHARE_EVERYTHING,
         syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
       },
@@ -68,6 +70,7 @@ export const seedMessageChannel = async (
         type: 'email',
         connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.PHIL,
         handle: 'phil.schiler@apple.dev',
+        isSyncEnabled: false,
         visibility: MessageChannelVisibility.SHARE_EVERYTHING,
         syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
       },

packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts

@@ -28,12 +28,18 @@ import {
   MessageChannelVisibility,
   MessageChannelWorkspaceEntity,
 } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
+import {
+  MessagingMessageListFetchJob,
+  MessagingMessageListFetchJobData,
+} from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job';
 import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
 
 @Injectable()
 export class MicrosoftAPIsService {
   constructor(
     private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    @InjectMessageQueue(MessageQueue.messagingQueue)
+    private readonly messageQueueService: MessageQueueService,
     @InjectMessageQueue(MessageQueue.calendarQueue)
     private readonly calendarQueueService: MessageQueueService,
     private readonly accountsToReconnectService: AccountsToReconnectService,
@@ -102,7 +108,6 @@ export class MicrosoftAPIsService {
           manager,
         );
 
-        // TODO: Modify this when the email sync is implemented
         await messageChannelRepository.save(
           {
             id: v4(),
@@ -111,8 +116,7 @@ export class MicrosoftAPIsService {
             handle,
             visibility:
               messageVisibility || MessageChannelVisibility.SHARE_EVERYTHING,
-            syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
+            syncStatus: MessageChannelSyncStatus.ONGOING,
-            syncStage: MessageChannelSyncStage.FAILED,
           },
           {},
           manager,
@@ -160,14 +164,13 @@ export class MicrosoftAPIsService {
           newOrExistingConnectedAccountId,
         );
 
-        // TODO: Modify this when the email sync is implemented
         await messageChannelRepository.update(
           {
             connectedAccountId: newOrExistingConnectedAccountId,
           },
           {
-            syncStage: MessageChannelSyncStage.FAILED,
+            syncStage: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
-            syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
+            syncStatus: MessageChannelSyncStatus.ONGOING,
             syncCursor: '',
             syncStageStartedAt: null,
           },
@@ -176,22 +179,21 @@ export class MicrosoftAPIsService {
       }
     });
 
-    // TODO: Uncomment this when the email sync is implemented
+    const messageChannels = await messageChannelRepository.find({
-    // const messageChannels = await messageChannelRepository.find({
+      where: {
-    //   where: {
+        connectedAccountId: newOrExistingConnectedAccountId,
-    //     connectedAccountId: newOrExistingConnectedAccountId,
+      },
-    //   },
+    });
-    // });
 
-    // for (const messageChannel of messageChannels) {
+    for (const messageChannel of messageChannels) {
-    //   await this.messageQueueService.add<MessagingMessageListFetchJobData>(
+      await this.messageQueueService.add<MessagingMessageListFetchJobData>(
-    //     MessagingMessageListFetchJob.name,
+        MessagingMessageListFetchJob.name,
-    //     {
+        {
-    //       workspaceId,
+          workspaceId,
-    //       messageChannelId: messageChannel.id,
+          messageChannelId: messageChannel.id,
-    //     },
+        },
-    //   );
+      );
-    // }
+    }
 
     const calendarChannels = await calendarChannelRepository.find({
       where: {

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts

@@ -14,4 +14,5 @@ export enum MessageImportDriverExceptionCode {
   UNKNOWN = 'UNKNOWN',
   UNKNOWN_NETWORK_ERROR = 'UNKNOWN_NETWORK_ERROR',
   NO_NEXT_SYNC_CURSOR = 'NO_NEXT_SYNC_CURSOR',
+  SYNC_CURSOR_ERROR = 'SYNC_CURSOR_ERROR',
 }

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts

@@ -32,7 +32,7 @@ export class GmailGetMessageListService {
   public async getFullMessageList(
     connectedAccount: Pick<
       ConnectedAccountWorkspaceEntity,
-      'provider' | 'refreshToken' | 'id'
+      'provider' | 'refreshToken' | 'id' | 'handle'
     >,
   ): Promise<GetFullMessageListResponse> {
     const gmailClient =

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts

@@ -10,6 +10,7 @@ import { MicrosoftGetMessageListService } from './microsoft-get-message-list.ser
 import { MicrosoftHandleErrorService } from './microsoft-handle-error.service';
 
 const refreshToken = 'replace-with-your-refresh-token';
+const syncCursor = 'replace-with-your-sync-cursor';
 
 xdescribe('Microsoft dev tests : get message list service', () => {
   let service: MicrosoftGetMessageListService;
@@ -54,4 +55,27 @@ xdescribe('Microsoft dev tests : get message list service', () => {
       service.getFullMessageList(mockConnectedAccountUnvalid),
     ).rejects.toThrowError('Access token is undefined or empty');
   });
+
+  it('Should fetch and return partial message list successfully', async () => {
+    const result = await service.getPartialMessageList(
+      mockConnectedAccount,
+      syncCursor,
+    );
+
+    expect(result.nextSyncCursor).toBeTruthy();
+  });
+
+  it('Should fail partial message if syncCursor is invalid', async () => {
+    await expect(
+      service.getPartialMessageList(mockConnectedAccount, 'invalid-syncCursor'),
+    ).rejects.toThrowError(
+      /Resource not found for the segment|Badly formed content/g,
+    );
+  });
+
+  it('Should fail partial message if syncCursor is missing', async () => {
+    await expect(
+      service.getPartialMessageList(mockConnectedAccount, ''),
+    ).rejects.toThrowError(/Missing SyncCursor/g);
+  });
 });

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts

@@ -7,8 +7,15 @@ import {
 } from '@microsoft/microsoft-graph-client';
 
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
+import {
+  MessageImportDriverException,
+  MessageImportDriverExceptionCode,
+} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
 import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider';
-import { GetFullMessageListResponse } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
+import {
+  GetFullMessageListResponse,
+  GetPartialMessageListResponse,
+} from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
 
 // Microsoft API limit is 1000 messages per request on this endpoint
 const MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT = 1000;
@@ -54,4 +61,54 @@ export class MicrosoftGetMessageListService {
       nextSyncCursor: pageIterator.getDeltaLink() || '',
     };
   }
+
+  public async getPartialMessageList(
+    connectedAccount: Pick<
+      ConnectedAccountWorkspaceEntity,
+      'provider' | 'refreshToken' | 'id'
+    >,
+    syncCursor: string,
+  ): Promise<GetPartialMessageListResponse> {
+    // important: otherwise tries to get the full message list
+    if (!syncCursor) {
+      throw new MessageImportDriverException(
+        'Missing SyncCursor',
+        MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR,
+      );
+    }
+
+    const messageExternalIds: string[] = [];
+    const messageExternalIdsToDelete: string[] = [];
+
+    const microsoftClient =
+      await this.microsoftClientProvider.getMicrosoftClient(connectedAccount);
+
+    const response: PageCollection = await microsoftClient
+      .api(syncCursor)
+      .version('beta')
+      .headers({
+        Prefer: `odata.maxpagesize=${MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT}, IdType="ImmutableId"`,
+      })
+      .get();
+
+    const callback: PageIteratorCallback = (data) => {
+      if (data['@removed']) {
+        messageExternalIdsToDelete.push(data.id);
+      } else {
+        messageExternalIds.push(data.id);
+      }
+
+      return true;
+    };
+
+    const pageIterator = new PageIterator(microsoftClient, response, callback);
+
+    await pageIterator.iterate();
+
+    return {
+      messageExternalIds,
+      messageExternalIdsToDelete,
+      nextSyncCursor: pageIterator.getDeltaLink() || '',
+    };
+  }
 }

packages/twenty-server/src/modules/messaging/message-import-manager/services/message-import-exception-handler.service.ts

@@ -64,6 +64,13 @@ export class MessageImportExceptionHandlerService {
           workspaceId,
         );
         break;
+      case MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR:
+        await this.handlePermanentException(
+          exception,
+          messageChannel,
+          workspaceId,
+        );
+        break;
       default:
         throw exception;
     }
@@ -149,6 +156,22 @@ export class MessageImportExceptionHandlerService {
     );
   }
 
+  private async handlePermanentException(
+    exception: MessageImportDriverException,
+    messageChannel: Pick<MessageChannelWorkspaceEntity, 'id'>,
+    workspaceId: string,
+  ): Promise<void> {
+    await this.messageChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
+      [messageChannel.id],
+      workspaceId,
+    );
+
+    throw new MessageImportException(
+      `Permanent error occurred while importing messages for message channel ${messageChannel.id} in workspace ${workspaceId}: ${exception.message}`,
+      MessageImportExceptionCode.UNKNOWN,
+    );
+  }
+
   private async handleNotFoundException(
     syncStep: MessageImportSyncStep,
     messageChannel: Pick<MessageChannelWorkspaceEntity, 'id'>,

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts

@@ -29,7 +29,7 @@ export class MessagingGetMessageListService {
   public async getFullMessageList(
     connectedAccount: Pick<
       ConnectedAccountWorkspaceEntity,
-      'provider' | 'refreshToken' | 'id'
+      'provider' | 'refreshToken' | 'id' | 'handle'
     >,
   ): Promise<GetFullMessageListResponse> {
     switch (connectedAccount.provider) {
@@ -63,11 +63,10 @@ export class MessagingGetMessageListService {
           syncCursor,
         );
       case 'microsoft':
-        return {
+        return this.microsoftGetMessageListService.getPartialMessageList(
-          messageExternalIds: [],
+          connectedAccount,
-          messageExternalIdsToDelete: [],
+          syncCursor,
-          nextSyncCursor: '',
+        );
-        };
       default:
         throw new MessageImportException(
           `Provider ${connectedAccount.provider} is not supported`,

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts

@@ -58,13 +58,14 @@ export class MessagingMessageService {
       });
 
       if (existingMessage) {
-        await messageChannelMessageAssociationRepository.insert(
+        await messageChannelMessageAssociationRepository.upsert(
           {
             messageChannelId,
             messageId: existingMessage.id,
             messageExternalId: message.externalId,
             messageThreadExternalId: message.messageThreadExternalId,
           },
+          ['messageChannelId', 'messageExternalId'],
           transactionManager,
         );
 

packages/twenty-ui/src/display/icon/assets/microsoft-outlook.svg

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg">
+<title>file_type_outlook</title>
+<path d="M19.484,7.937v5.477L21.4,14.619a.489.489,0,0,0,.21,0l8.238-5.554a1.174,1.174,0,0,0-.959-1.128Z" style="fill:#0072c6"/>
+<path d="M19.484,15.457l1.747,1.2a.522.522,0,0,0,.543,0c-.3.181,8.073-5.378,8.073-5.378V21.345a1.408,1.408,0,0,1-1.49,1.555H19.483V15.457Z" style="fill:#0072c6"/>
+<path d="M10.44,12.932a1.609,1.609,0,0,0-1.42.838,4.131,4.131,0,0,0-.526,2.218A4.05,4.05,0,0,0,9.02,18.2a1.6,1.6,0,0,0,2.771.022,4.014,4.014,0,0,0,.515-2.2,4.369,4.369,0,0,0-.5-2.281A1.536,1.536,0,0,0,10.44,12.932Z" style="fill:#0072c6"/>
+<path d="M2.153,5.155V26.582L18.453,30V2ZM13.061,19.491a3.231,3.231,0,0,1-2.7,1.361,3.19,3.19,0,0,1-2.64-1.318A5.459,5.459,0,0,1,6.706,16.1a5.868,5.868,0,0,1,1.036-3.616A3.267,3.267,0,0,1,10.486,11.1a3.116,3.116,0,0,1,2.61,1.321,5.639,5.639,0,0,1,1,3.484A5.763,5.763,0,0,1,13.061,19.491Z" style="fill:#0072c6"/>
+</svg>

packages/twenty-ui/src/display/icon/components/IconMicrosoftOutlook.tsx

@@ -0,0 +1,14 @@
+import { useTheme } from '@emotion/react';
+
+import IconMicrosoftOutlookRaw from '../assets/microsoft-outlook.svg?react';
+
+interface IconMicrosoftOutlookProps {
+  size?: number;
+}
+
+export const IconMicrosoftOutlook = (props: IconMicrosoftOutlookProps) => {
+  const theme = useTheme();
+  const size = props.size ?? theme.icon.size.lg;
+
+  return <IconMicrosoftOutlookRaw height={size} width={size} />;
+};

packages/twenty-ui/src/display/index.ts

@@ -16,6 +16,7 @@ export * from './icon/components/IconGoogle';
 export * from './icon/components/IconGoogleCalendar';
 export * from './icon/components/IconLock';
 export * from './icon/components/IconMicrosoft';
+export * from './icon/components/IconMicrosoftOutlook';
 export * from './icon/components/IconRelationManyToOne';
 export * from './icon/components/IconTwentyStar';
 export * from './icon/components/IconTwentyStarFilled';

packages/twenty-website/src/content/developers/self-hosting/setup.mdx

@@ -54,7 +54,7 @@ Register the following recurring jobs:
 yarn command:prod cron:messaging:messages-import
 yarn command:prod cron:messaging:message-list-fetch
 yarn command:prod cron:calendar:calendar-event-list-fetch
-yarn command:prod cron:calendar:calendar-event-import
+yarn command:prod cron:calendar:calendar-events-import
 yarn command:prod cron:messaging:ongoing-stale
 yarn command:prod cron:calendar:ongoing-stale
 ```
@@ -110,7 +110,7 @@ Register the following recurring jobs:
 yarn command:prod cron:messaging:messages-import
 yarn command:prod cron:messaging:message-list-fetch
 yarn command:prod cron:calendar:calendar-event-list-fetch
-yarn command:prod cron:calendar:calendar-event-import
+yarn command:prod cron:calendar:calendar-events-import
 yarn command:prod cron:messaging:ongoing-stale
 yarn command:prod cron:calendar:ongoing-stale
 ```