From 8cbb1aa71a37fe269e0e3325c5be3d483a00b855 Mon Sep 17 00:00:00 2001 From: Guillim Date: Wed, 2 Jul 2025 19:03:13 +0200 Subject: [PATCH] Miscrosoft Client errors when refreshing accessToken (#12884) # Context We had an error saying "Unknown error importing calendar events for [...]: Access token is undefined or empty. Please provide a valid token. For more help - https://github.com/microsoftgraph/msgraph-sdk-javascript/blob/dev/docs/CustomAuthenticationProvider.md " Reason is that the access token method for microsoft is a bit different than the one from google. And in microsoft case, we want to check the access token in the authProvider in case it fails. Currently it was not catched, so it broke services above that counted on the accesstoken to be valid. That ended in UNKNOWN failure for our calendar event fetch service. # Solution This PR should solve the issue since : 1. forcing the method to break if accesstoken renewal fails 2. logs will help to know what kind of errors will be sent in case we need to tackle this issue again 3. we now throw TEMPORARY error instead of unknown, allowing 3 getClientConfig failure before it is definitive Why so many changes while it should have been simple : The root cause is the `authProvider` from `'@microsoft/microsoft-graph-client'` npm package. It does not throw a custom error, and we cannot catch it on calling `Client.init`. Errors only occurs when the client from ``` const client = this.microsoftOAuth2ClientManagerService.getOAuth2Client(refreshtoken) ``` is used, as in `client.api('/messages').get().catch(err => [...])` So we need to go in every call using the client and catch errors, and rethrow whenver we need as a newly created message type `MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE` We discussed 1. and 2. with @bosiraphael already I added 3. to make our system more robust without waiting for more failures # Related Fixes : https://github.com/twentyhq/twenty/issues/12880 --------- Co-authored-by: Charles Bochet --- .../microsoft-calendar-get-events.service.ts | 11 +++++++ ...icrosoft-calendar-import-events.service.ts | 12 ++++++++ .../microsoft-email-alias-manager.service.ts | 11 +++++++ ...microsoft-oauth2-client-manager.service.ts | 24 ++++++++++++++- .../message-import-driver.exception.ts | 1 + .../providers/microsoft-client.provider.ts | 14 ++------- .../microsoft-fetch-by-batch.service.ts | 11 +++++++ .../microsoft-get-message-list.service.ts | 25 ++++++++++++++++ .../microsoft-handle-error.service.ts | 7 +++++ .../is-access-token-refreshing-error.utils.ts | 7 +++++ ...saging-import-exception-handler.service.ts | 1 + .../messaging-send-message.service.ts | 30 +++++++++++++++++-- 12 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils.ts diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts index ce9da8555..4ff4ae86c 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-get-events.service.ts @@ -6,10 +6,15 @@ import { PageIteratorCallback, } from '@microsoft/microsoft-graph-client'; +import { + CalendarEventImportDriverException, + CalendarEventImportDriverExceptionCode, +} from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception'; import { parseMicrosoftCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/parse-microsoft-calendar-error.util'; import { GetCalendarEventsResponse } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service'; import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils'; @Injectable() export class MicrosoftCalendarGetEventsService { @@ -56,6 +61,12 @@ export class MicrosoftCalendarGetEventsService { nextSyncCursor: pageIterator.getDeltaLink() || '', }; } catch (error) { + if (isAccessTokenRefreshingError(error?.body)) { + throw new CalendarEventImportDriverException( + error.message, + CalendarEventImportDriverExceptionCode.TEMPORARY_ERROR, + ); + } throw parseMicrosoftCalendarError(error); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts index 31aa76f33..39b3f8b66 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/services/microsoft-calendar-import-events.service.ts @@ -2,11 +2,16 @@ import { Injectable } from '@nestjs/common'; import { Event } from '@microsoft/microsoft-graph-types'; +import { + CalendarEventImportDriverException, + CalendarEventImportDriverExceptionCode, +} from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception'; import { formatMicrosoftCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util'; import { parseMicrosoftCalendarError } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/parse-microsoft-calendar-error.util'; import { FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event'; import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils'; @Injectable() export class MicrosoftCalendarImportEventsService { @@ -39,6 +44,13 @@ export class MicrosoftCalendarImportEventsService { return formatMicrosoftCalendarEvents(events); } catch (error) { + if (isAccessTokenRefreshingError(error?.body)) { + throw new CalendarEventImportDriverException( + error.message, + CalendarEventImportDriverExceptionCode.TEMPORARY_ERROR, + ); + } + throw parseMicrosoftCalendarError(error); } } diff --git a/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/microsoft/microsoft-email-alias-manager.service.ts b/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/microsoft/microsoft-email-alias-manager.service.ts index 4ad70db94..b0a484fcd 100644 --- a/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/microsoft/microsoft-email-alias-manager.service.ts +++ b/packages/twenty-server/src/modules/connected-account/email-alias-manager/drivers/microsoft/microsoft-email-alias-manager.service.ts @@ -1,7 +1,12 @@ import { Injectable } from '@nestjs/common'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { + MessageImportDriverException, + MessageImportDriverExceptionCode, +} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils'; @Injectable() export class MicrosoftEmailAliasManagerService { @@ -19,6 +24,12 @@ export class MicrosoftEmailAliasManagerService { .api('/me?$select=proxyAddresses') .get() .catch((error) => { + if (isAccessTokenRefreshingError(error?.message)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } throw new Error(`Failed to fetch email aliases: ${error.message}`); }); diff --git a/packages/twenty-server/src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service.ts b/packages/twenty-server/src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service.ts index 1a1c24b0a..a14640990 100644 --- a/packages/twenty-server/src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service.ts +++ b/packages/twenty-server/src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { AuthProvider, @@ -7,9 +7,13 @@ import { } from '@microsoft/microsoft-graph-client'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { ConnectedAccountRefreshAccessTokenExceptionCode } from 'src/modules/connected-account/refresh-tokens-manager/exceptions/connected-account-refresh-tokens.exception'; @Injectable() export class MicrosoftOAuth2ClientManagerService { + private readonly logger = new Logger( + MicrosoftOAuth2ClientManagerService.name, + ); constructor(private readonly twentyConfigService: TwentyConfigService) {} public async getOAuth2Client(refreshToken: string): Promise { @@ -41,6 +45,24 @@ export class MicrosoftOAuth2ClientManagerService { const data = await res.json(); + if (!res.ok) { + if (data) { + const accessTokenSliced = data?.access_token?.slice(0, 10); + const refreshTokenSliced = data?.refresh_token?.slice(0, 10); + + delete data.access_token; + delete data.refresh_token; + this.logger.error(data); + this.logger.error(`accessTokenSliced: ${accessTokenSliced}`); + this.logger.error(`refreshTokenSliced: ${refreshTokenSliced}`); + } + + this.logger.error(res); + throw new Error( + `${MicrosoftOAuth2ClientManagerService.name} error: ${ConnectedAccountRefreshAccessTokenExceptionCode.REFRESH_ACCESS_TOKEN_FAILED}`, + ); + } + callback(null, data.access_token); } catch (error) { callback(error, null); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts index d8ae8a258..01bb60343 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception.ts @@ -15,4 +15,5 @@ export enum MessageImportDriverExceptionCode { NO_NEXT_SYNC_CURSOR = 'NO_NEXT_SYNC_CURSOR', SYNC_CURSOR_ERROR = 'SYNC_CURSOR_ERROR', PROVIDER_NOT_SUPPORTED = 'PROVIDER_NOT_SUPPORTED', + CLIENT_NOT_AVAILABLE = 'CLIENT_NOT_AVAILABLE', } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts index 0dfc149b4..dea9403f0 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider.ts @@ -17,16 +17,8 @@ export class MicrosoftClientProvider { 'refreshToken' | 'id' >, ): Promise { - try { - return await this.microsoftOAuth2ClientManagerService.getOAuth2Client( - connectedAccount.refreshToken, - ); - } catch (error) { - throw new Error( - `Failed to get Microsoft client: ${ - error instanceof Error ? error.message : 'Unknown error' - }`, - ); - } + return await this.microsoftOAuth2ClientManagerService.getOAuth2Client( + connectedAccount.refreshToken, + ); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts index 5ff3c8854..8c723e03f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-fetch-by-batch.service.ts @@ -1,9 +1,14 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { + MessageImportDriverException, + MessageImportDriverExceptionCode, +} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; import { MicrosoftGraphBatchResponse } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface'; import { MicrosoftHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service'; +import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils'; @Injectable() export class MicrosoftFetchByBatchService { @@ -52,6 +57,12 @@ export class MicrosoftFetchByBatchService { batchResponses.push(batchResponse); } catch (error) { + if (isAccessTokenRefreshingError(error?.body)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } this.microsoftHandleErrorService.handleMicrosoftMessageFetchByBatchError( error, ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts index cc76d2d31..02888d833 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts @@ -18,6 +18,7 @@ import { import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; import { MicrosoftHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service'; import { MessageFolderName } from 'src/modules/messaging/message-import-manager/drivers/microsoft/types/folders'; +import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils'; import { GetFullMessageListForFoldersResponse, GetFullMessageListResponse, @@ -79,6 +80,12 @@ export class MicrosoftGetMessageListService { }) .get() .catch((error) => { + if (isAccessTokenRefreshingError(error?.body)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } this.microsoftHandleErrorService.handleMicrosoftGetMessageListError( error, ); @@ -97,6 +104,12 @@ export class MicrosoftGetMessageListService { }); await pageIterator.iterate().catch((error) => { + if (isAccessTokenRefreshingError(error?.body)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } this.microsoftHandleErrorService.handleMicrosoftGetMessageListError( error, ); @@ -204,6 +217,12 @@ export class MicrosoftGetMessageListService { }) .get() .catch((error) => { + if (isAccessTokenRefreshingError(error?.body)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } this.microsoftHandleErrorService.handleMicrosoftGetMessageListError( error, ); @@ -226,6 +245,12 @@ export class MicrosoftGetMessageListService { }); await pageIterator.iterate().catch((error) => { + if (isAccessTokenRefreshingError(error?.body)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } this.microsoftHandleErrorService.handleMicrosoftGetMessageListError( error, ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts index 26bfcf8c1..f45eb50cc 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-handle-error.service.ts @@ -70,6 +70,13 @@ export class MicrosoftHandleErrorService { // eslint-disable-next-line @typescript-eslint/no-explicit-any public handleMicrosoftGetMessagesError(error: any): void { + if ( + error instanceof MessageImportDriverException && + error.code === MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE + ) { + throw error; + } + if (!error.statusCode) { throw new MessageImportDriverException( `Microsoft Graph API unknown error: ${error}`, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils.ts new file mode 100644 index 000000000..396688773 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils.ts @@ -0,0 +1,7 @@ +import { ConnectedAccountRefreshAccessTokenExceptionCode } from 'src/modules/connected-account/refresh-tokens-manager/exceptions/connected-account-refresh-tokens.exception'; + +export const isAccessTokenRefreshingError = (body: string): boolean => { + return body.includes( + ConnectedAccountRefreshAccessTokenExceptionCode.REFRESH_ACCESS_TOKEN_FAILED, + ); +}; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts index 4c2dfa517..32b05fd4b 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts @@ -60,6 +60,7 @@ export class MessageImportExceptionHandlerService { case MessageNetworkExceptionCode.ECONNRESET: case MessageNetworkExceptionCode.ETIMEDOUT: case MessageNetworkExceptionCode.ERR_NETWORK: + case MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE: await this.handleTemporaryException( syncStep, messageChannel, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-send-message.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-send-message.service.ts index 731ee4769..7d6db35ca 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-send-message.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-send-message.service.ts @@ -5,9 +5,14 @@ import { assertUnreachable, isDefined } from 'twenty-shared/utils'; import { z } from 'zod'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { + MessageImportDriverException, + MessageImportDriverExceptionCode, +} from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception'; import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider'; import { OAuth2ClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/oauth2-client.provider'; import { MicrosoftClientProvider } from 'src/modules/messaging/message-import-manager/drivers/microsoft/providers/microsoft-client.provider'; +import { isAccessTokenRefreshingError } from 'src/modules/messaging/message-import-manager/drivers/microsoft/utils/is-access-token-refreshing-error.utils'; import { mimeEncode } from 'src/modules/messaging/message-import-manager/utils/mime-encode.util'; interface SendMessageInput { @@ -86,11 +91,32 @@ export class MessagingSendMessageService { const response = await microsoftClient .api(`/me/messages`) - .post(message); + .post(message) + .catch((error) => { + if (isAccessTokenRefreshingError(error?.body)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } + throw error; + }); z.string().parse(response.id); - await microsoftClient.api(`/me/messages/${response.id}/send`).post({}); + await microsoftClient + .api(`/me/messages/${response.id}/send`) + .post({}) + .catch((error) => { + if (isAccessTokenRefreshingError(error?.body)) { + throw new MessageImportDriverException( + error.message, + MessageImportDriverExceptionCode.CLIENT_NOT_AVAILABLE, + ); + } + throw error; + }); + break; } case ConnectedAccountProvider.IMAP_SMTP_CALDAV: {