From f60b704feb896991cb4c0cadb001ad146dd3a1c5 Mon Sep 17 00:00:00 2001 From: Matt Dvertola <64113801+mdvertola@users.noreply.github.com> Date: Tue, 3 Jun 2025 05:17:48 -0400 Subject: [PATCH] #12336 adding gmail email sync error handling (#12383) I believe that some emails with invalid characters are breaking the sync process. this PR attempts to create a "safeParseAddress" function. Hopefully this will change current behavior of a single email breaking the entire sync process to the sync process "skipping" an invalid email address and continuing on. I opened this because of issues explained in #12336 --------- Co-authored-by: guillim --- .../parse-and-format-gmail-message.util.ts | 21 ++++-- .../gmail/utils/parse-gmail-message.util.ts | 14 ++-- .../microsoft-get-messages.service.ts | 69 ++++++++++--------- .../types/email-address.ts | 4 ++ ...ddress-object-as-participants.util.spec.ts | 37 +++++++++- ...mat-address-object-as-participants.util.ts | 29 ++++---- .../utils/safe-parse.util.ts | 28 ++++++++ 7 files changed, 141 insertions(+), 61 deletions(-) create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/types/email-address.ts create mode 100644 packages/twenty-server/src/modules/messaging/message-import-manager/utils/safe-parse.util.ts diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts index 157b143d4..8aee992d1 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts @@ -40,11 +40,22 @@ export const parseAndFormatGmailMessage = ( return null; } + const toParticipants = to ?? deliveredTo; + const participants = [ - ...formatAddressObjectAsParticipants(from, 'from'), - ...formatAddressObjectAsParticipants(to ?? deliveredTo, 'to'), - ...formatAddressObjectAsParticipants(cc, 'cc'), - ...formatAddressObjectAsParticipants(bcc, 'bcc'), + ...(from + ? formatAddressObjectAsParticipants([{ address: from }], 'from') + : []), + ...(toParticipants + ? formatAddressObjectAsParticipants( + [{ address: toParticipants, name: '' }], + 'to', + ) + : []), + ...(cc ? formatAddressObjectAsParticipants([{ address: cc }], 'cc') : []), + ...(bcc + ? formatAddressObjectAsParticipants([{ address: bcc }], 'bcc') + : []), ]; const textWithoutReplyQuotations = text @@ -57,7 +68,7 @@ export const parseAndFormatGmailMessage = ( subject: subject || '', messageThreadExternalId: threadId, receivedAt: new Date(parseInt(internalDate)), - direction: computeMessageDirection(from[0].address || '', connectedAccount), + direction: computeMessageDirection(from || '', connectedAccount), participants, text: sanitizeString(textWithoutReplyQuotations), attachments, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts index 1ba116744..2ef5bda88 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts @@ -1,11 +1,11 @@ import assert from 'assert'; -import addressparser from 'addressparser'; import { gmail_v1 } from 'googleapis'; import { getAttachmentData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-attachment-data.util'; import { getBodyData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-body-data.util'; import { getPropertyFromHeaders } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-property-from-headers.util'; +import { safeParseEmailAddressAddress } from 'src/modules/messaging/message-import-manager/utils/safe-parse.util'; export const parseGmailMessage = (message: gmail_v1.Schema$Message) => { const subject = getPropertyFromHeaders(message, 'Subject'); @@ -36,11 +36,13 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => { historyId, internalDate, subject, - from: rawFrom ? addressparser(rawFrom) : undefined, - deliveredTo: rawDeliveredTo ? addressparser(rawDeliveredTo) : undefined, - to: rawTo ? addressparser(rawTo) : undefined, - cc: rawCc ? addressparser(rawCc) : undefined, - bcc: rawBcc ? addressparser(rawBcc) : undefined, + from: rawFrom ? safeParseEmailAddressAddress(rawFrom) : undefined, + deliveredTo: rawDeliveredTo + ? safeParseEmailAddressAddress(rawDeliveredTo) + : undefined, + to: rawTo ? safeParseEmailAddressAddress(rawTo) : undefined, + cc: rawCc ? safeParseEmailAddressAddress(rawCc) : undefined, + bcc: rawBcc ? safeParseEmailAddressAddress(rawBcc) : undefined, text, attachments, }; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts index 3acedb02a..8ec5bba46 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts @@ -1,5 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; +import { EmailAddress } from 'addressparser'; import { isDefined } from 'twenty-shared/utils'; import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; @@ -9,6 +10,7 @@ import { MicrosoftImportDriverException } from 'src/modules/messaging/message-im import { MicrosoftGraphBatchResponse } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface'; import { MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message'; import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util'; +import { safeParseEmailAddress } from 'src/modules/messaging/message-import-manager/utils/safe-parse.util'; import { MicrosoftFetchByBatchService } from './microsoft-fetch-by-batch.service'; import { MicrosoftHandleErrorService } from './microsoft-handle-error.service'; @@ -78,37 +80,42 @@ export class MicrosoftGetMessagesService { ); } - const participants = [ - ...formatAddressObjectAsParticipants( - response?.from?.emailAddress, - 'from', - ), - ...formatAddressObjectAsParticipants( - response?.toRecipients - ?.filter(isDefined) - // @ts-expect-error legacy noImplicitAny - .map((recipient) => recipient.emailAddress), - 'to', - ), - ...formatAddressObjectAsParticipants( - response?.ccRecipients - ?.filter(isDefined) - // @ts-expect-error legacy noImplicitAny - .map((recipient) => recipient.emailAddress), - 'cc', - ), - ...formatAddressObjectAsParticipants( - response?.bccRecipients - ?.filter(isDefined) - // @ts-expect-error legacy noImplicitAny - .map((recipient) => recipient.emailAddress), - 'bcc', - ), - ]; + const safeParseFrom = response?.from?.emailAddress + ? [safeParseEmailAddress(response.from.emailAddress)] + : []; - const safeParticipantsFormat = participants.filter((participant) => { - return participant.handle.includes('@'); - }); + const safeParseTo = response?.toRecipients + ?.filter(isDefined) + .map((recipient: { emailAddress: EmailAddress }) => + safeParseEmailAddress(recipient.emailAddress), + ); + + const safeParseCc = response?.ccRecipients + ?.filter(isDefined) + .map((recipient: { emailAddress: EmailAddress }) => + safeParseEmailAddress(recipient.emailAddress), + ); + + const safeParseBcc = response?.bccRecipients + ?.filter(isDefined) + .map((recipient: { emailAddress: EmailAddress }) => + safeParseEmailAddress(recipient.emailAddress), + ); + + const participants = [ + ...(safeParseFrom + ? formatAddressObjectAsParticipants(safeParseFrom, 'from') + : []), + ...(safeParseTo + ? formatAddressObjectAsParticipants(safeParseTo, 'to') + : []), + ...(safeParseCc + ? formatAddressObjectAsParticipants(safeParseCc, 'cc') + : []), + ...(safeParseBcc + ? formatAddressObjectAsParticipants(safeParseBcc, 'bcc') + : []), + ]; return { externalId: response.id, @@ -124,7 +131,7 @@ export class MicrosoftGetMessagesService { connectedAccount, ) : MessageDirection.INCOMING, - participants: safeParticipantsFormat, + participants, attachments: [], }; }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/types/email-address.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/types/email-address.ts new file mode 100644 index 000000000..1369ca821 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/types/email-address.ts @@ -0,0 +1,4 @@ +export type EmailAddress = { + address: string; + name?: string; +}; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/format-address-object-as-participants.util.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/format-address-object-as-participants.util.spec.ts index 84193a5ec..cff04d2da 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/format-address-object-as-participants.util.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/format-address-object-as-participants.util.spec.ts @@ -23,11 +23,42 @@ describe('formatAddressObjectAsParticipants', () => { ]); }); - it('should return an empty array if address object is undefined', () => { - const addressObject = undefined; + it('should return an empty array if address object handle has no @', () => { + const addressObject = { + name: 'John Doe', + address: 'john.doe', + }; - const result = formatAddressObjectAsParticipants(addressObject, 'to'); + const result = formatAddressObjectAsParticipants([addressObject], 'to'); expect(result).toEqual([]); }); + + it('should return an empty array if address object handle is empty', () => { + const addressObject = { + name: 'John Doe', + address: '', + }; + + const result = formatAddressObjectAsParticipants([addressObject], 'to'); + + expect(result).toEqual([]); + }); + + it('should return a lowewrcase handle if the handle is not lowercase', () => { + const addressObject = { + name: 'John Doe', + address: 'John.Doe@example.com', + }; + + const result = formatAddressObjectAsParticipants([addressObject], 'to'); + + expect(result).toEqual([ + { + role: 'to', + handle: 'john.doe@example.com', + displayName: 'John Doe', + }, + ]); + }); }); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util.ts index 03bed9460..4572d55d2 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util.ts @@ -1,36 +1,33 @@ -import addressparser from 'addressparser'; +import { isDefined } from 'twenty-shared/utils'; import { Participant } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message.type'; - -const formatAddressObjectAsArray = ( - addressObject: addressparser.EmailAddress | addressparser.EmailAddress[], -): addressparser.EmailAddress[] => { - return Array.isArray(addressObject) ? addressObject : [addressObject]; -}; +import { EmailAddress } from 'src/modules/messaging/message-import-manager/types/email-address'; const removeSpacesAndLowerCase = (email: string): string => { return email.replace(/\s/g, '').toLowerCase(); }; export const formatAddressObjectAsParticipants = ( - addressObject: - | addressparser.EmailAddress - | addressparser.EmailAddress[] - | undefined, + addressObjects: EmailAddress[], role: 'from' | 'to' | 'cc' | 'bcc', ): Participant[] => { - if (!addressObject) return []; - const addressObjects = formatAddressObjectAsArray(addressObject); - const participants = addressObjects.map((addressObject) => { const address = addressObject.address; + if (!isDefined(address)) { + return null; + } + + if (!address.includes('@')) { + return null; + } + return { role, - handle: address ? removeSpacesAndLowerCase(address) : '', + handle: removeSpacesAndLowerCase(address), displayName: addressObject.name || '', }; }); - return participants.flat(); + return participants.filter(isDefined) as Participant[]; }; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/safe-parse.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/safe-parse.util.ts new file mode 100644 index 000000000..0429aa1b3 --- /dev/null +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/safe-parse.util.ts @@ -0,0 +1,28 @@ +import { Logger } from '@nestjs/common'; + +import addressparser from 'addressparser'; + +import { EmailAddress } from 'src/modules/messaging/message-import-manager/types/email-address'; + +export const safeParseEmailAddressAddress = ( + address: string, +): string | undefined => { + const logger = new Logger(safeParseEmailAddressAddress.name); + + try { + return addressparser(address)[0].address; + } catch (error) { + logger.error(`Error parsing address: ${address}`, error); + + return undefined; + } +}; + +export const safeParseEmailAddress = ( + emailAddress: EmailAddress, +): EmailAddress => { + return { + address: safeParseEmailAddressAddress(emailAddress.address) || '', + name: emailAddress.name, + }; +};