admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

#12336 adding gmail email sync error handling (#12383)

Browse Source 
I believe that some emails with invalid characters are breaking the sync
process.

this PR attempts to create a "safeParseAddress" function. Hopefully this
will change current behavior of a single email breaking the entire sync
process to the sync process "skipping" an invalid email address and
continuing on.

I opened this because of issues explained in #12336

---------

Co-authored-by: guillim <guigloo@msn.com>
This commit is contained in:
Matt Dvertola
2025-06-03 05:17:48 -04:00
committed by GitHub
parent 7617dd76aa
commit f60b704feb
7 changed files with 141 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-and-format-gmail-message.util.ts
View File

@ -40,11 +40,22 @@ export const parseAndFormatGmailMessage = (
return null;
}
const toParticipants = to ?? deliveredTo;
const participants = [
...formatAddressObjectAsParticipants(from, 'from'),
...formatAddressObjectAsParticipants(to ?? deliveredTo, 'to'),
...formatAddressObjectAsParticipants(cc, 'cc'),
...formatAddressObjectAsParticipants(bcc, 'bcc'),
...(from
? formatAddressObjectAsParticipants([{ address: from }], 'from')
: []),
...(toParticipants
? formatAddressObjectAsParticipants(
[{ address: toParticipants, name: '' }],
'to',
)
: []),
...(cc ? formatAddressObjectAsParticipants([{ address: cc }], 'cc') : []),
...(bcc
? formatAddressObjectAsParticipants([{ address: bcc }], 'bcc')
: []),
];
const textWithoutReplyQuotations = text
@ -57,7 +68,7 @@ export const parseAndFormatGmailMessage = (
subject: subject || '',
messageThreadExternalId: threadId,
receivedAt: new Date(parseInt(internalDate)),
direction: computeMessageDirection(from[0].address || '', connectedAccount),
direction: computeMessageDirection(from || '', connectedAccount),
participants,
text: sanitizeString(textWithoutReplyQuotations),
attachments,

14
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/parse-gmail-message.util.ts
View File

@ -1,11 +1,11 @@
import assert from 'assert';
import addressparser from 'addressparser';
import { gmail_v1 } from 'googleapis';
import { getAttachmentData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-attachment-data.util';
import { getBodyData } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-body-data.util';
import { getPropertyFromHeaders } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/get-property-from-headers.util';
import { safeParseEmailAddressAddress } from 'src/modules/messaging/message-import-manager/utils/safe-parse.util';
export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {
const subject = getPropertyFromHeaders(message, 'Subject');
@ -36,11 +36,13 @@ export const parseGmailMessage = (message: gmail_v1.Schema$Message) => {
historyId,
internalDate,
subject,
from: rawFrom ? addressparser(rawFrom) : undefined,
deliveredTo: rawDeliveredTo ? addressparser(rawDeliveredTo) : undefined,
to: rawTo ? addressparser(rawTo) : undefined,
cc: rawCc ? addressparser(rawCc) : undefined,
bcc: rawBcc ? addressparser(rawBcc) : undefined,
from: rawFrom ? safeParseEmailAddressAddress(rawFrom) : undefined,
deliveredTo: rawDeliveredTo
? safeParseEmailAddressAddress(rawDeliveredTo)
: undefined,
to: rawTo ? safeParseEmailAddressAddress(rawTo) : undefined,
cc: rawCc ? safeParseEmailAddressAddress(rawCc) : undefined,
bcc: rawBcc ? safeParseEmailAddressAddress(rawBcc) : undefined,
text,
attachments,
};

69
packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.ts
View File

@ -1,5 +1,6 @@
import { Injectable, Logger } from '@nestjs/common';
import { EmailAddress } from 'addressparser';
import { isDefined } from 'twenty-shared/utils';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@ -9,6 +10,7 @@ import { MicrosoftImportDriverException } from 'src/modules/messaging/message-im
import { MicrosoftGraphBatchResponse } from 'src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.interface';
import { MessageWithParticipants } from 'src/modules/messaging/message-import-manager/types/message';
import { formatAddressObjectAsParticipants } from 'src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util';
import { safeParseEmailAddress } from 'src/modules/messaging/message-import-manager/utils/safe-parse.util';
import { MicrosoftFetchByBatchService } from './microsoft-fetch-by-batch.service';
import { MicrosoftHandleErrorService } from './microsoft-handle-error.service';
@ -78,37 +80,42 @@ export class MicrosoftGetMessagesService {
);
}
const participants = [
...formatAddressObjectAsParticipants(
response?.from?.emailAddress,
'from',
),
...formatAddressObjectAsParticipants(
response?.toRecipients
?.filter(isDefined)
// @ts-expect-error legacy noImplicitAny
.map((recipient) => recipient.emailAddress),
'to',
),
...formatAddressObjectAsParticipants(
response?.ccRecipients
?.filter(isDefined)
// @ts-expect-error legacy noImplicitAny
.map((recipient) => recipient.emailAddress),
'cc',
),
...formatAddressObjectAsParticipants(
response?.bccRecipients
?.filter(isDefined)
// @ts-expect-error legacy noImplicitAny
.map((recipient) => recipient.emailAddress),
'bcc',
),
];
const safeParseFrom = response?.from?.emailAddress
? [safeParseEmailAddress(response.from.emailAddress)]
: [];
const safeParticipantsFormat = participants.filter((participant) => {
return participant.handle.includes('@');
});
const safeParseTo = response?.toRecipients
?.filter(isDefined)
.map((recipient: { emailAddress: EmailAddress }) =>
safeParseEmailAddress(recipient.emailAddress),
);
const safeParseCc = response?.ccRecipients
?.filter(isDefined)
.map((recipient: { emailAddress: EmailAddress }) =>
safeParseEmailAddress(recipient.emailAddress),
);
const safeParseBcc = response?.bccRecipients
?.filter(isDefined)
.map((recipient: { emailAddress: EmailAddress }) =>
safeParseEmailAddress(recipient.emailAddress),
);
const participants = [
...(safeParseFrom
? formatAddressObjectAsParticipants(safeParseFrom, 'from')
: []),
...(safeParseTo
? formatAddressObjectAsParticipants(safeParseTo, 'to')
: []),
...(safeParseCc
? formatAddressObjectAsParticipants(safeParseCc, 'cc')
: []),
...(safeParseBcc
? formatAddressObjectAsParticipants(safeParseBcc, 'bcc')
: []),
];
return {
externalId: response.id,
@ -124,7 +131,7 @@ export class MicrosoftGetMessagesService {
connectedAccount,
)
: MessageDirection.INCOMING,
participants: safeParticipantsFormat,
participants,
attachments: [],
};
});

4
packages/twenty-server/src/modules/messaging/message-import-manager/types/email-address.ts Normal file
View File

@ -0,0 +1,4 @@
export type EmailAddress = {
address: string;
name?: string;
};

37
packages/twenty-server/src/modules/messaging/message-import-manager/utils/__tests__/format-address-object-as-participants.util.spec.ts
View File

@ -23,11 +23,42 @@ describe('formatAddressObjectAsParticipants', () => {
]);
});
it('should return an empty array if address object is undefined', () => {
const addressObject = undefined;
it('should return an empty array if address object handle has no @', () => {
const addressObject = {
name: 'John Doe',
address: 'john.doe',
};
const result = formatAddressObjectAsParticipants(addressObject, 'to');
const result = formatAddressObjectAsParticipants([addressObject], 'to');
expect(result).toEqual([]);
});
it('should return an empty array if address object handle is empty', () => {
const addressObject = {
name: 'John Doe',
address: '',
};
const result = formatAddressObjectAsParticipants([addressObject], 'to');
expect(result).toEqual([]);
});
it('should return a lowewrcase handle if the handle is not lowercase', () => {
const addressObject = {
name: 'John Doe',
address: 'John.Doe@example.com',
};
const result = formatAddressObjectAsParticipants([addressObject], 'to');
expect(result).toEqual([
{
role: 'to',
handle: 'john.doe@example.com',
displayName: 'John Doe',
},
]);
});
});

29
packages/twenty-server/src/modules/messaging/message-import-manager/utils/format-address-object-as-participants.util.ts
View File

@ -1,36 +1,33 @@
import addressparser from 'addressparser';
import { isDefined } from 'twenty-shared/utils';
import { Participant } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message.type';
const formatAddressObjectAsArray = (
addressObject: addressparser.EmailAddress | addressparser.EmailAddress[],
): addressparser.EmailAddress[] => {
return Array.isArray(addressObject) ? addressObject : [addressObject];
};
import { EmailAddress } from 'src/modules/messaging/message-import-manager/types/email-address';
const removeSpacesAndLowerCase = (email: string): string => {
return email.replace(/\s/g, '').toLowerCase();
};
export const formatAddressObjectAsParticipants = (
addressObject:
| addressparser.EmailAddress
| addressparser.EmailAddress[]
| undefined,
addressObjects: EmailAddress[],
role: 'from' | 'to' | 'cc' | 'bcc',
): Participant[] => {
if (!addressObject) return [];
const addressObjects = formatAddressObjectAsArray(addressObject);
const participants = addressObjects.map((addressObject) => {
const address = addressObject.address;
if (!isDefined(address)) {
return null;
}
if (!address.includes('@')) {
return null;
}
return {
role,
handle: address ? removeSpacesAndLowerCase(address) : '',
handle: removeSpacesAndLowerCase(address),
displayName: addressObject.name || '',
};
});
return participants.flat();
return participants.filter(isDefined) as Participant[];
};

28
packages/twenty-server/src/modules/messaging/message-import-manager/utils/safe-parse.util.ts Normal file
View File

@ -0,0 +1,28 @@
import { Logger } from '@nestjs/common';
import addressparser from 'addressparser';
import { EmailAddress } from 'src/modules/messaging/message-import-manager/types/email-address';
export const safeParseEmailAddressAddress = (
address: string,
): string | undefined => {
const logger = new Logger(safeParseEmailAddressAddress.name);
try {
return addressparser(address)[0].address;
} catch (error) {
logger.error(`Error parsing address: ${address}`, error);
return undefined;
}
};
export const safeParseEmailAddress = (
emailAddress: EmailAddress,
): EmailAddress => {
return {
address: safeParseEmailAddressAddress(emailAddress.address) || '',
name: emailAddress.name,
};
};