admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Outlook integration (#9631)

Browse Source 
Get Partial messages
This commit is contained in:
Guillim
2025-01-16 09:50:01 +01:00
committed by GitHub
parent 789ff30dc7
commit f077efd171
13 changed files with 165 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/twenty-server/src/database/typeorm-seeds/workspace/message-channels.ts
View File

@ -44,6 +44,7 @@ export const seedMessageChannel = async (
type: 'email',
connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.TIM,
handle: 'tim@apple.dev',
isSyncEnabled: false,
visibility: MessageChannelVisibility.SHARE_EVERYTHING,
syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
},
@ -56,6 +57,7 @@ export const seedMessageChannel = async (
type: 'email',
connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.JONY,
handle: 'jony.ive@apple.dev',
isSyncEnabled: false,
visibility: MessageChannelVisibility.SHARE_EVERYTHING,
syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
},
@ -68,6 +70,7 @@ export const seedMessageChannel = async (
type: 'email',
connectedAccountId: DEV_SEED_CONNECTED_ACCOUNT_IDS.PHIL,
handle: 'phil.schiler@apple.dev',
isSyncEnabled: false,
visibility: MessageChannelVisibility.SHARE_EVERYTHING,
syncSubStatus: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
},

44
packages/twenty-server/src/engine/core-modules/auth/services/microsoft-apis.service.ts
View File

@ -28,12 +28,18 @@ import {
MessageChannelVisibility,
MessageChannelWorkspaceEntity,
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import {
MessagingMessageListFetchJob,
MessagingMessageListFetchJobData,
} from 'src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@Injectable()
export class MicrosoftAPIsService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@InjectMessageQueue(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
@InjectMessageQueue(MessageQueue.calendarQueue)
private readonly calendarQueueService: MessageQueueService,
private readonly accountsToReconnectService: AccountsToReconnectService,
@ -102,7 +108,6 @@ export class MicrosoftAPIsService {
manager,
);
// TODO: Modify this when the email sync is implemented
await messageChannelRepository.save(
{
id: v4(),
@ -111,8 +116,7 @@ export class MicrosoftAPIsService {
handle,
visibility:
messageVisibility || MessageChannelVisibility.SHARE_EVERYTHING,
syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: MessageChannelSyncStatus.ONGOING,
},
{},
manager,
@ -160,14 +164,13 @@ export class MicrosoftAPIsService {
newOrExistingConnectedAccountId,
);
// TODO: Modify this when the email sync is implemented
await messageChannelRepository.update(
{
connectedAccountId: newOrExistingConnectedAccountId,
},
{
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: MessageChannelSyncStatus.NOT_SYNCED,
syncStage: MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncCursor: '',
syncStageStartedAt: null,
},
@ -176,22 +179,21 @@ export class MicrosoftAPIsService {
}
});
// TODO: Uncomment this when the email sync is implemented
// const messageChannels = await messageChannelRepository.find({
// where: {
// connectedAccountId: newOrExistingConnectedAccountId,
// },
// });
const messageChannels = await messageChannelRepository.find({
where: {
connectedAccountId: newOrExistingConnectedAccountId,
},
});
// for (const messageChannel of messageChannels) {
// await this.messageQueueService.add<MessagingMessageListFetchJobData>(
// MessagingMessageListFetchJob.name,
// {
// workspaceId,
// messageChannelId: messageChannel.id,
// },
// );
// }
for (const messageChannel of messageChannels) {
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
workspaceId,
messageChannelId: messageChannel.id,
},
);
}
const calendarChannels = await calendarChannelRepository.find({
where: {

1
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts
View File

@ -14,4 +14,5 @@ export enum MessageImportDriverExceptionCode {
UNKNOWN = 'UNKNOWN',
UNKNOWN_NETWORK_ERROR = 'UNKNOWN_NETWORK_ERROR',
NO_NEXT_SYNC_CURSOR = 'NO_NEXT_SYNC_CURSOR',
SYNC_CURSOR_ERROR = 'SYNC_CURSOR_ERROR',
}

2
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts
View File

@ -32,7 +32,7 @@ export class GmailGetMessageListService {
public async getFullMessageList(
connectedAccount: Pick<
ConnectedAccountWorkspaceEntity,
'provider' | 'refreshToken' | 'id'
'provider' | 'refreshToken' | 'id' | 'handle'
>,
): Promise<GetFullMessageListResponse> {
const gmailClient =

24
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts
View File

@ -10,6 +10,7 @@ import { MicrosoftGetMessageListService } from './microsoft-get-message-list.ser
import { MicrosoftHandleErrorService } from './microsoft-handle-error.service';
const refreshToken = 'replace-with-your-refresh-token';
const syncCursor = 'replace-with-your-sync-cursor';
xdescribe('Microsoft dev tests : get message list service', () => {
let service: MicrosoftGetMessageListService;
@ -54,4 +55,27 @@ xdescribe('Microsoft dev tests : get message list service', () => {
service.getFullMessageList(mockConnectedAccountUnvalid),
).rejects.toThrowError('Access token is undefined or empty');
});
it('Should fetch and return partial message list successfully', async () => {
const result = await service.getPartialMessageList(
mockConnectedAccount,
syncCursor,
);
expect(result.nextSyncCursor).toBeTruthy();
});
it('Should fail partial message if syncCursor is invalid', async () => {
await expect(
service.getPartialMessageList(mockConnectedAccount, 'invalid-syncCursor'),
).rejects.toThrowError(
/Resource not found for the segment|Badly formed content/g,
);
});
it('Should fail partial message if syncCursor is missing', async () => {
await expect(
service.getPartialMessageList(mockConnectedAccount, ''),
).rejects.toThrowError(/Missing SyncCursor/g);
});
});

59
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts
View File

@ -7,8 +7,15 @@ import {
} from '@microsoft/microsoft-graph-client';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import {
MessageImportDriverException,
MessageImportDriverExceptionCode,
} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider';
import { GetFullMessageListResponse } from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
import {
GetFullMessageListResponse,
GetPartialMessageListResponse,
} from 'src/modules/messaging/message-import-manager/services/messaging-get-message-list.service';
// Microsoft API limit is 1000 messages per request on this endpoint
const MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT = 1000;
@ -54,4 +61,54 @@ export class MicrosoftGetMessageListService {
nextSyncCursor: pageIterator.getDeltaLink() || '',
};
}
public async getPartialMessageList(
connectedAccount: Pick<
ConnectedAccountWorkspaceEntity,
'provider' | 'refreshToken' | 'id'
>,
syncCursor: string,
): Promise<GetPartialMessageListResponse> {
// important: otherwise tries to get the full message list
if (!syncCursor) {
throw new MessageImportDriverException(
'Missing SyncCursor',
MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR,
);
}
const messageExternalIds: string[] = [];
const messageExternalIdsToDelete: string[] = [];
const microsoftClient =
await this.microsoftClientProvider.getMicrosoftClient(connectedAccount);
const response: PageCollection = await microsoftClient
.api(syncCursor)
.version('beta')
.headers({
Prefer: `odata.maxpagesize=${MESSAGING_MICROSOFT_USERS_MESSAGES_LIST_MAX_RESULT}, IdType="ImmutableId"`,
})
.get();
const callback: PageIteratorCallback = (data) => {
if (data['@removed']) {
messageExternalIdsToDelete.push(data.id);
} else {
messageExternalIds.push(data.id);
}
return true;
};
const pageIterator = new PageIterator(microsoftClient, response, callback);
await pageIterator.iterate();
return {
messageExternalIds,
messageExternalIdsToDelete,
nextSyncCursor: pageIterator.getDeltaLink() || '',
};
}
}

23
packages/twenty-server/src/modules/messaging/message-import-manager/services/message-import-exception-handler.service.ts
View File

@ -64,6 +64,13 @@ export class MessageImportExceptionHandlerService {
workspaceId,
);
break;
case MessageImportDriverExceptionCode.SYNC_CURSOR_ERROR:
await this.handlePermanentException(
exception,
messageChannel,
workspaceId,
);
break;
default:
throw exception;
}
@ -149,6 +156,22 @@ export class MessageImportExceptionHandlerService {
);
}
private async handlePermanentException(
exception: MessageImportDriverException,
messageChannel: Pick<MessageChannelWorkspaceEntity, 'id'>,
workspaceId: string,
): Promise<void> {
await this.messageChannelSyncStatusService.markAsFailedUnknownAndFlushMessagesToImport(
[messageChannel.id],
workspaceId,
);
throw new MessageImportException(
`Permanent error occurred while importing messages for message channel ${messageChannel.id} in workspace ${workspaceId}: ${exception.message}`,
MessageImportExceptionCode.UNKNOWN,
);
}
private async handleNotFoundException(
syncStep: MessageImportSyncStep,
messageChannel: Pick<MessageChannelWorkspaceEntity, 'id'>,

11
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-get-message-list.service.ts
View File

@ -29,7 +29,7 @@ export class MessagingGetMessageListService {
public async getFullMessageList(
connectedAccount: Pick<
ConnectedAccountWorkspaceEntity,
'provider' | 'refreshToken' | 'id'
'provider' | 'refreshToken' | 'id' | 'handle'
>,
): Promise<GetFullMessageListResponse> {
switch (connectedAccount.provider) {
@ -63,11 +63,10 @@ export class MessagingGetMessageListService {
syncCursor,
);
case 'microsoft':
return {
messageExternalIds: [],
messageExternalIdsToDelete: [],
nextSyncCursor: '',
};
return this.microsoftGetMessageListService.getPartialMessageList(
connectedAccount,
syncCursor,
);
default:
throw new MessageImportException(
`Provider ${connectedAccount.provider} is not supported`,

3
packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message.service.ts
View File

@ -58,13 +58,14 @@ export class MessagingMessageService {
});
if (existingMessage) {
await messageChannelMessageAssociationRepository.insert(
await messageChannelMessageAssociationRepository.upsert(
{
messageChannelId,
messageId: existingMessage.id,
messageExternalId: message.externalId,
messageThreadExternalId: message.messageThreadExternalId,
},
['messageChannelId', 'messageExternalId'],
transactionManager,
);

8
packages/twenty-ui/src/display/icon/assets/microsoft-outlook.svg Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
<svg width="800px" height="800px" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg">
<title>file_type_outlook</title>
<path d="M19.484,7.937v5.477L21.4,14.619a.489.489,0,0,0,.21,0l8.238-5.554a1.174,1.174,0,0,0-.959-1.128Z" style="fill:#0072c6"/>
<path d="M19.484,15.457l1.747,1.2a.522.522,0,0,0,.543,0c-.3.181,8.073-5.378,8.073-5.378V21.345a1.408,1.408,0,0,1-1.49,1.555H19.483V15.457Z" style="fill:#0072c6"/>
<path d="M10.44,12.932a1.609,1.609,0,0,0-1.42.838,4.131,4.131,0,0,0-.526,2.218A4.05,4.05,0,0,0,9.02,18.2a1.6,1.6,0,0,0,2.771.022,4.014,4.014,0,0,0,.515-2.2,4.369,4.369,0,0,0-.5-2.281A1.536,1.536,0,0,0,10.44,12.932Z" style="fill:#0072c6"/>
<path d="M2.153,5.155V26.582L18.453,30V2ZM13.061,19.491a3.231,3.231,0,0,1-2.7,1.361,3.19,3.19,0,0,1-2.64-1.318A5.459,5.459,0,0,1,6.706,16.1a5.868,5.868,0,0,1,1.036-3.616A3.267,3.267,0,0,1,10.486,11.1a3.116,3.116,0,0,1,2.61,1.321,5.639,5.639,0,0,1,1,3.484A5.763,5.763,0,0,1,13.061,19.491Z" style="fill:#0072c6"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

14
packages/twenty-ui/src/display/icon/components/IconMicrosoftOutlook.tsx Normal file
View File

@ -0,0 +1,14 @@
import { useTheme } from '@emotion/react';
import IconMicrosoftOutlookRaw from '../assets/microsoft-outlook.svg?react';
interface IconMicrosoftOutlookProps {
size?: number;
}
export const IconMicrosoftOutlook = (props: IconMicrosoftOutlookProps) => {
const theme = useTheme();
const size = props.size ?? theme.icon.size.lg;
return <IconMicrosoftOutlookRaw height={size} width={size} />;
};

1
packages/twenty-ui/src/display/index.ts
View File

@ -16,6 +16,7 @@ export * from './icon/components/IconGoogle';
export * from './icon/components/IconGoogleCalendar';
export * from './icon/components/IconLock';
export * from './icon/components/IconMicrosoft';
export * from './icon/components/IconMicrosoftOutlook';
export * from './icon/components/IconRelationManyToOne';
export * from './icon/components/IconTwentyStar';
export * from './icon/components/IconTwentyStarFilled';

4
packages/twenty-website/src/content/developers/self-hosting/setup.mdx
View File

@ -54,7 +54,7 @@ Register the following recurring jobs:
yarn command:prod cron:messaging:messages-import
yarn command:prod cron:messaging:message-list-fetch
yarn command:prod cron:calendar:calendar-event-list-fetch
yarn command:prod cron:calendar:calendar-event-import
yarn command:prod cron:calendar:calendar-events-import
yarn command:prod cron:messaging:ongoing-stale
yarn command:prod cron:calendar:ongoing-stale
```
@ -110,7 +110,7 @@ Register the following recurring jobs:
yarn command:prod cron:messaging:messages-import
yarn command:prod cron:messaging:message-list-fetch
yarn command:prod cron:calendar:calendar-event-list-fetch
yarn command:prod cron:calendar:calendar-event-import
yarn command:prod cron:calendar:calendar-events-import
yarn command:prod cron:messaging:ongoing-stale
yarn command:prod cron:calendar:ongoing-stale
```