From e87c6236ca47da6f85b2af0e3d4682e32966d380 Mon Sep 17 00:00:00 2001 From: Guillim Date: Mon, 12 May 2025 14:09:57 +0200 Subject: [PATCH] calendar sync failed (#11970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # TLDR fix bug due to some event properties coming from the google calendar API containing weird characters like this "\u0000�4\u000b\u00042��K\u0001�z,\u001cm�", it made the Postgres select and insert operator fail ## Details We can have event properties (like cal UID below) encoded in a strange way. From my research, the character \u0000 comes from `C` language to signal end of line. It is wrongly interpreted by Postgres so must be escaped. I decided to remove all possibility of failure with this regex `[^\x20-\x7E]` basically "any character that is not a printable ASCII character" ``` [ "5foijj28qb8smqiafjablo17vd@google.com", "\u0011�\"�f�\\\u0019G_=��\u0005]x", "?}|��\f}l��+�弴�", "%���?t\u0007��n\u001e\u000eY�T<", ".\u0011�\u0016�!�\u000eIǹ� ��\u001f", "!h\u0004��DŽ�6���h�]E", "(�CX]�Q�7�^��n\u0006�", "_040000008200E00074C5B7101A82E0080000000070105B958DEFD801000000000000000010000000EA30DB22E888B943A8EE0AD483F8DB35", "\t�N�#�D��Ic�h", "+�)�H���jJ|Ժ�'�", "_040000008200E00074C5B7101A82E0080000000070A54736C6EED8010000000000000000100000006502334AFE61904595C2831FA4391034", "_040000008200E00074C5B7101A82E0080000000072C80C3590EFD8010000000000000000100000001BA9FD5B330C1A4D85462AC9D70B9D9D", "sg�fvUa:St>-<�d\u0006", "\u0017ڦ��_\u001e\u001fGm-1����", "_040000008200E00074C5B7101A82E00800000000F0F4F01F4EF0D8010000000000000000100000004C11CE0950C85549B79C456C13987AB8", "$�����\u0007V\u0007��\u001e�OLN", "_040000008200E00074C5B7101A82E00800000000341DA81151EDD8010000000000000000100000007453CEFB19AA2D4899B17F0BDB000493", "_01C756CA-98DC-4799-9F06-883A540A065C", "_040000008200E00074C5B7101A82E00800000000919548BBF1EDD80100000000000000001000000050AE1E41F3CD314CA9215F193EBE1D39", "_040000008200E00074C5B7101A82E0080000000039CF64D718EDD801000000000000000010000000B3824EF0711436488CB5459BE83733B1", "_040000008200E00074C5B7101A82E00800000000C50BDADCB5E4D8010000000000000000100000005B95FE762B2EF84B9C5AC53907B7E5E8", "�Spx�\u0003ve��ss�X��", "\b���>\u0013̈�ыh��0�", "_040000008200E00074C5B7101A82E008000000005BCD492230E8D801000000000000000010000000B243A9CE99E94C4DAD201129A8F2A2F7", "_040000008200E00074C5B7101A82E008000000009B540B82D1DCD801000000000000000010000000B4AFC9825D94994AA6C528C953BD3D96", "_040000008200E00074C5B7101A82E008000000000D39EABDB7E4D801000000000000000010000000CD07BA2D05E61B47A597EE538ECE8CA5", "\u0016���\u0003inv�=O����", "4?b�-���\u001c\u0013ת�E�p", ">\u0000\u0015-;_�^�W&�p\u001f�", "_040000008200E00074C5B7101A82E008000000006258818C85D9D80100000000000000001000000003346C625768FE43AF3F8D09917CA3C9", "+K��ٔ�\u0006�\u0018G\u000b\u0000�s\u000e", "/\f�\rj�IOD�g脅��", "_68d820f9-e2c1-4d9a-bf61-83e4957c8261", "xĠ�W4>���t�\u001d���", "_040000008200E00074C5B7101A82E008000000003F1F314328E3D8010000000000000000100000002C8DC60F5369F44DA2B066441F882B35", "_040000008200E00074C5B7101A82E008000000006DFCC204FDDED80100000000000000001000000046ECD444737D2E4992C349B2EA05637F", "_1FF7C1BB-4E1A-485F-ADD0-5C4768601179", "_040000008200E00074C5B7101A82E00800000000A0AA1223EFD3D801000000000000000010000000C83B72C2B424944199F3353D1442B27E", "_040000008200E00074C5B7101A82E0080000000063D101EB06E4D801000000000000000010000000E489F61C89B8914BA6F8C8A2606405FE", "\u0011��f�\"�b���rB�[�", ";t��\u001d\u001euDY+T\u001d��v", ] ``` Fixes https://github.com/twentyhq/core-team-issues/issues/946 Fixes https://twenty-v7.sentry.io/issues/6568530279/?environment=prod&project=4507072499810304&query=is%3Aunresolved%20issue.priority%3A%5Bhigh%2C%20medium%5D%20b09d7a84-e6bc-45cf-b3ca-1e6047dddeed&referrer=issue-stream&stream_index=0 ### Edit : Changed the regex to match the chars to remove from `[^\x20-\x7E]` to `replace('\u0000', '');` --- .../format-google-calendar-event.util.spec.ts | 99 ++++++++++++++++ .../format-google-calendar-event.util.ts | 23 +++- ...rmat-microsoft-calendar-event.util.spec.ts | 106 ++++++++++++++++++ .../format-microsoft-calendar-event.util.ts | 22 +++- .../drivers/utils/sanitizeCalendarEvent.ts | 26 +++++ 5 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/__tests__/format-google-calendar-event.util.spec.ts create mode 100644 packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/__tests__/format-microsoft-calendar-event.util.spec.ts create mode 100644 packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent.ts diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/__tests__/format-google-calendar-event.util.spec.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/__tests__/format-google-calendar-event.util.spec.ts new file mode 100644 index 000000000..7124f069d --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/__tests__/format-google-calendar-event.util.spec.ts @@ -0,0 +1,99 @@ +import { calendar_v3 as calendarV3 } from 'googleapis'; + +import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util'; +import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; + +describe('formatGoogleCalendarEvents', () => { + const mockGoogleEvent: calendarV3.Schema$Event = { + id: 'event123', + summary: 'Team Meeting', + description: 'Weekly team sync', + location: 'Conference Room A', + status: 'confirmed', + created: '2023-01-01T10:00:00Z', + updated: '2023-01-02T10:00:00Z', + iCalUID: 'event123@google.com', + start: { + dateTime: '2023-01-15T14:00:00Z', + }, + end: { + dateTime: '2023-01-15T15:00:00Z', + }, + attendees: [ + { + email: 'organizer@example.com', + displayName: 'Meeting Organizer', + organizer: true, + responseStatus: 'accepted', + }, + { + email: 'attendee@example.com', + displayName: 'Test Attendee', + responseStatus: 'tentative', + }, + ], + conferenceData: { + conferenceSolution: { + key: { + type: 'hangoutsMeet', + }, + }, + entryPoints: [ + { + uri: 'https://meet.google.com/abc-defg-hij', + }, + ], + }, + }; + + it('should correctly format a normal Google Calendar event', () => { + const result = formatGoogleCalendarEvents([mockGoogleEvent]); + + expect(result).toHaveLength(1); + const formattedEvent = result[0]; + + expect(formattedEvent.title).toBe('Team Meeting'); + expect(formattedEvent.description).toBe('Weekly team sync'); + expect(formattedEvent.location).toBe('Conference Room A'); + expect(formattedEvent.isCanceled).toBe(false); + expect(formattedEvent.isFullDay).toBe(false); + expect(formattedEvent.startsAt).toBe('2023-01-15T14:00:00Z'); + expect(formattedEvent.endsAt).toBe('2023-01-15T15:00:00Z'); + expect(formattedEvent.externalId).toBe('event123'); + expect(formattedEvent.conferenceSolution).toBe('hangoutsMeet'); + expect(formattedEvent.conferenceLinkUrl).toBe( + 'https://meet.google.com/abc-defg-hij', + ); + + expect(formattedEvent.participants).toHaveLength(2); + expect(formattedEvent.participants[0].handle).toBe('organizer@example.com'); + expect(formattedEvent.participants[0].isOrganizer).toBe(true); + expect(formattedEvent.participants[0].responseStatus).toBe( + CalendarEventParticipantResponseStatus.ACCEPTED, + ); + expect(formattedEvent.participants[1].handle).toBe('attendee@example.com'); + expect(formattedEvent.participants[1].responseStatus).toBe( + CalendarEventParticipantResponseStatus.TENTATIVE, + ); + }); + + it('should sanitize a UCALID with improper exit char 0x00', () => { + const mockGoogleEventWithImproperUcalid: calendarV3.Schema$Event = { + ...mockGoogleEvent, + iCalUID: '\u0000eventStrange@google.com', + }; + + const mockGoogleEventWithImproperUcalid2: calendarV3.Schema$Event = { + ...mockGoogleEvent, + iCalUID: '>\u0000\u0015-;_�^�W&�p\u001f�', + }; + + const result = formatGoogleCalendarEvents([ + mockGoogleEventWithImproperUcalid, + mockGoogleEventWithImproperUcalid2, + ]); + + expect(result[0].iCalUID).toBe('eventStrange@google.com'); + expect(result[1].iCalUID).toBe('>\u0015-;_�^�W&�p\u001f�'); + }); +}); diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util.ts index 06f97e3c5..eef52c0ca 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util.ts @@ -1,5 +1,6 @@ import { calendar_v3 as calendarV3 } from 'googleapis'; +import { sanitizeCalendarEvent } from 'src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent'; import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event'; @@ -25,7 +26,8 @@ const formatGoogleCalendarEvent = ( } }; - return { + // Create the event object + const calendarEvent: CalendarEventWithParticipants = { title: event.summary ?? '', isCanceled: event.status === 'cancelled', isFullDay: event.start?.dateTime == null, @@ -51,4 +53,23 @@ const formatGoogleCalendarEvent = ( })) ?? [], status: event.status ?? '', }; + + const propertiesToSanitize: (keyof CalendarEventWithParticipants)[] = [ + 'title', + 'startsAt', + 'endsAt', + 'externalId', + 'externalCreatedAt', + 'externalUpdatedAt', + 'description', + 'location', + 'iCalUID', + 'conferenceSolution', + 'conferenceLinkLabel', + 'conferenceLinkUrl', + 'recurringEventExternalId', + 'status', + ]; + + return sanitizeCalendarEvent(calendarEvent, propertiesToSanitize); }; diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/__tests__/format-microsoft-calendar-event.util.spec.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/__tests__/format-microsoft-calendar-event.util.spec.ts new file mode 100644 index 000000000..fd48a90e3 --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/__tests__/format-microsoft-calendar-event.util.spec.ts @@ -0,0 +1,106 @@ +import { Event } from '@microsoft/microsoft-graph-types'; + +import { formatMicrosoftCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util'; +import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; + +describe('formatMicrosoftCalendarEvents', () => { + const mockMicrosoftEvent: Event = { + id: 'event123', + subject: 'Team Meeting', + body: { + content: 'Weekly team sync', + contentType: 'text', + }, + location: { + displayName: 'Conference Room A', + }, + isCancelled: false, + isAllDay: false, + createdDateTime: '2023-01-01T10:00:00Z', + lastModifiedDateTime: '2023-01-02T10:00:00Z', + iCalUId: 'event123@microsoft.com', + start: { + dateTime: '2023-01-15T14:00:00Z', + timeZone: 'UTC', + }, + end: { + dateTime: '2023-01-15T15:00:00Z', + timeZone: 'UTC', + }, + attendees: [ + { + emailAddress: { + address: 'organizer@example.com', + name: 'Meeting Organizer', + }, + status: { + response: 'organizer', + }, + }, + { + emailAddress: { + address: 'attendee@example.com', + name: 'Test Attendee', + }, + status: { + response: 'tentativelyAccepted', + }, + }, + ], + onlineMeetingProvider: 'teamsForBusiness', + onlineMeeting: { + joinUrl: 'https://teams.microsoft.com/l/meetup-join/abc123', + }, + }; + + it('should correctly format a normal Microsoft Calendar event', () => { + const result = formatMicrosoftCalendarEvents([mockMicrosoftEvent]); + + expect(result).toHaveLength(1); + const formattedEvent = result[0]; + + expect(formattedEvent.title).toBe('Team Meeting'); + expect(formattedEvent.description).toBe('Weekly team sync'); + expect(formattedEvent.location).toBe('Conference Room A'); + expect(formattedEvent.isCanceled).toBe(false); + expect(formattedEvent.isFullDay).toBe(false); + expect(formattedEvent.startsAt).toBe('2023-01-15T14:00:00Z'); + expect(formattedEvent.endsAt).toBe('2023-01-15T15:00:00Z'); + expect(formattedEvent.externalId).toBe('event123'); + expect(formattedEvent.conferenceSolution).toBe('teamsForBusiness'); + expect(formattedEvent.conferenceLinkUrl).toBe( + 'https://teams.microsoft.com/l/meetup-join/abc123', + ); + + expect(formattedEvent.participants).toHaveLength(2); + expect(formattedEvent.participants[0].handle).toBe('organizer@example.com'); + expect(formattedEvent.participants[0].isOrganizer).toBe(true); + expect(formattedEvent.participants[0].responseStatus).toBe( + CalendarEventParticipantResponseStatus.ACCEPTED, + ); + expect(formattedEvent.participants[1].handle).toBe('attendee@example.com'); + expect(formattedEvent.participants[1].responseStatus).toBe( + CalendarEventParticipantResponseStatus.TENTATIVE, + ); + }); + + it('should sanitize a Microsoft Calendar event with improper exit char 0x00', () => { + const mockMicrosoftEventWithImproperData: Event = { + ...mockMicrosoftEvent, + iCalUId: '\u0000eventStrange@microsoft.com', + }; + + const mockMicrosoftEventWithImproperData2: Event = { + ...mockMicrosoftEvent, + iCalUId: '>\u0000\u0015-;_�^�W&�p\u001f�', + }; + + const result = formatMicrosoftCalendarEvents([ + mockMicrosoftEventWithImproperData, + mockMicrosoftEventWithImproperData2, + ]); + + expect(result[0].iCalUID).toBe('eventStrange@microsoft.com'); + expect(result[1].iCalUID).toBe('>\u0015-;_�^�W&�p\u001f�'); + }); +}); diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util.ts index a0085bff2..0beca6c24 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util.ts @@ -4,6 +4,7 @@ import { ResponseType, } from '@microsoft/microsoft-graph-types'; +import { sanitizeCalendarEvent } from 'src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent'; import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event'; @@ -32,7 +33,7 @@ const formatMicrosoftCalendarEvent = ( } }; - return { + const calendarEvent: CalendarEventWithParticipants = { title: event.subject ?? '', isCanceled: !!event.isCancelled, isFullDay: !!event.isAllDay, @@ -57,4 +58,23 @@ const formatMicrosoftCalendarEvent = ( })) ?? [], status: '', }; + + const propertiesToSanitize: (keyof CalendarEventWithParticipants)[] = [ + 'title', + 'startsAt', + 'endsAt', + 'externalId', + 'externalCreatedAt', + 'externalUpdatedAt', + 'description', + 'location', + 'iCalUID', + 'conferenceSolution', + 'conferenceLinkLabel', + 'conferenceLinkUrl', + 'recurringEventExternalId', + 'status', + ]; + + return sanitizeCalendarEvent(calendarEvent, propertiesToSanitize); }; diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent.ts new file mode 100644 index 000000000..29c076ae7 --- /dev/null +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent.ts @@ -0,0 +1,26 @@ +import { isDefined } from 'twenty-shared/utils'; + +export const sanitizeCalendarEvent = >( + event: T, + propertiesToSanitize: (keyof T)[], +): T => { + const sanitizedEvent = { ...event }; + + for (const property of propertiesToSanitize) { + if (!isDefined(sanitizedEvent[property])) { + continue; + } + if (typeof sanitizedEvent[property] !== 'string') { + continue; + } + sanitizedEvent[property] = sanitizeString( + sanitizedEvent[property], + ) as T[typeof property]; + } + + return sanitizedEvent; +}; + +const sanitizeString = (value: string): string => { + return value.replace('\u0000', ''); +};