admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

calendar sync failed (#11970)

Browse Source 
# TLDR 
fix bug due to some event properties coming from the google calendar API
containing weird characters like this
"\u0000�4\u000b\u00042��K\u0001�z,\u001cm�",

it made the Postgres select and insert operator fail



## Details
We can have event properties (like cal UID below) encoded in a strange
way. From my research, the character \u0000 comes from `C` language to
signal end of line. It is wrongly interpreted by Postgres so must be
escaped. I decided to remove all possibility of failure with this regex
`[^\x20-\x7E]` basically "any character that is not a printable ASCII
character"

```
[
  "5foijj28qb8smqiafjablo17vd@google.com",
  "\u0011�\"�f�\\\u0019G_=��\u0005]x",
  "?}|��\f}l��+�弴�",
  "%���?t\u0007��n\u001e\u000eY�T<",
  ".\u0011�\u0016�!�\u000eIǹ� ��\u001f",
  "!h\u0004��DŽ�6���h�]E",
  "(�CX]�Q�7�^��n\u0006�",
  "_040000008200E00074C5B7101A82E0080000000070105B958DEFD801000000000000000010000000EA30DB22E888B943A8EE0AD483F8DB35",
  "\t�N�#�D��Ic�h",
  "+�)�H���jJ|Ժ�'�",
  "_040000008200E00074C5B7101A82E0080000000070A54736C6EED8010000000000000000100000006502334AFE61904595C2831FA4391034",
  "_040000008200E00074C5B7101A82E0080000000072C80C3590EFD8010000000000000000100000001BA9FD5B330C1A4D85462AC9D70B9D9D",
  "sg�fvUa:St>-<�d\u0006",
  "\u0017ڦ��_\u001e\u001fGm-1����",
  "_040000008200E00074C5B7101A82E00800000000F0F4F01F4EF0D8010000000000000000100000004C11CE0950C85549B79C456C13987AB8",
  "$�����\u0007V\u0007��\u001e�OLN",
  "_040000008200E00074C5B7101A82E00800000000341DA81151EDD8010000000000000000100000007453CEFB19AA2D4899B17F0BDB000493",
  "_01C756CA-98DC-4799-9F06-883A540A065C",
  "_040000008200E00074C5B7101A82E00800000000919548BBF1EDD80100000000000000001000000050AE1E41F3CD314CA9215F193EBE1D39",
  "_040000008200E00074C5B7101A82E0080000000039CF64D718EDD801000000000000000010000000B3824EF0711436488CB5459BE83733B1",
  "_040000008200E00074C5B7101A82E00800000000C50BDADCB5E4D8010000000000000000100000005B95FE762B2EF84B9C5AC53907B7E5E8",
  "�Spx�\u0003ve��ss�X��",
  "\b���>\u0013̈�ыh��0�",
  "_040000008200E00074C5B7101A82E008000000005BCD492230E8D801000000000000000010000000B243A9CE99E94C4DAD201129A8F2A2F7",
  "_040000008200E00074C5B7101A82E008000000009B540B82D1DCD801000000000000000010000000B4AFC9825D94994AA6C528C953BD3D96",
  "_040000008200E00074C5B7101A82E008000000000D39EABDB7E4D801000000000000000010000000CD07BA2D05E61B47A597EE538ECE8CA5",
  "\u0016���\u0003inv�=O����",
  "4?b�-���\u001c\u0013ת�E�p",
  ">\u0000\u0015-;_�^�W&�p\u001f�",
  "_040000008200E00074C5B7101A82E008000000006258818C85D9D80100000000000000001000000003346C625768FE43AF3F8D09917CA3C9",
  "+K��ٔ�\u0006�\u0018G\u000b\u0000�s\u000e",
  "/\f�\rj�IOD�g脅��",
  "_68d820f9-e2c1-4d9a-bf61-83e4957c8261",
  "xĠ�W4>���t�\u001d���",
  "_040000008200E00074C5B7101A82E008000000003F1F314328E3D8010000000000000000100000002C8DC60F5369F44DA2B066441F882B35",
  "_040000008200E00074C5B7101A82E008000000006DFCC204FDDED80100000000000000001000000046ECD444737D2E4992C349B2EA05637F",
  "_1FF7C1BB-4E1A-485F-ADD0-5C4768601179",
  "_040000008200E00074C5B7101A82E00800000000A0AA1223EFD3D801000000000000000010000000C83B72C2B424944199F3353D1442B27E",
  "_040000008200E00074C5B7101A82E0080000000063D101EB06E4D801000000000000000010000000E489F61C89B8914BA6F8C8A2606405FE",
  "\u0011��f�\"�b���rB�[�",
  ";t��\u001d\u001euDY+T\u001d��v",
]
```



Fixes https://github.com/twentyhq/core-team-issues/issues/946

Fixes
https://twenty-v7.sentry.io/issues/6568530279/?environment=prod&project=4507072499810304&query=is%3Aunresolved%20issue.priority%3A%5Bhigh%2C%20medium%5D%20b09d7a84-e6bc-45cf-b3ca-1e6047dddeed&referrer=issue-stream&stream_index=0


### Edit :
Changed the regex to match the chars to remove from 
`[^\x20-\x7E]` to `replace('\u0000', '');`
This commit is contained in:
Guillim
2025-05-12 14:09:57 +02:00
committed by GitHub
parent a4656b415c
commit e87c6236ca
5 changed files with 274 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/__tests__/format-google-calendar-event.util.spec.ts Normal file
View File

@ -0,0 +1,99 @@
import { calendar_v3 as calendarV3 } from 'googleapis';
import { formatGoogleCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util';
import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
describe('formatGoogleCalendarEvents', () => {
const mockGoogleEvent: calendarV3.Schema$Event = {
id: 'event123',
summary: 'Team Meeting',
description: 'Weekly team sync',
location: 'Conference Room A',
status: 'confirmed',
created: '2023-01-01T10:00:00Z',
updated: '2023-01-02T10:00:00Z',
iCalUID: 'event123@google.com',
start: {
dateTime: '2023-01-15T14:00:00Z',
},
end: {
dateTime: '2023-01-15T15:00:00Z',
},
attendees: [
{
email: 'organizer@example.com',
displayName: 'Meeting Organizer',
organizer: true,
responseStatus: 'accepted',
},
{
email: 'attendee@example.com',
displayName: 'Test Attendee',
responseStatus: 'tentative',
},
],
conferenceData: {
conferenceSolution: {
key: {
type: 'hangoutsMeet',
},
},
entryPoints: [
{
uri: 'https://meet.google.com/abc-defg-hij',
},
],
},
};
it('should correctly format a normal Google Calendar event', () => {
const result = formatGoogleCalendarEvents([mockGoogleEvent]);
expect(result).toHaveLength(1);
const formattedEvent = result[0];
expect(formattedEvent.title).toBe('Team Meeting');
expect(formattedEvent.description).toBe('Weekly team sync');
expect(formattedEvent.location).toBe('Conference Room A');
expect(formattedEvent.isCanceled).toBe(false);
expect(formattedEvent.isFullDay).toBe(false);
expect(formattedEvent.startsAt).toBe('2023-01-15T14:00:00Z');
expect(formattedEvent.endsAt).toBe('2023-01-15T15:00:00Z');
expect(formattedEvent.externalId).toBe('event123');
expect(formattedEvent.conferenceSolution).toBe('hangoutsMeet');
expect(formattedEvent.conferenceLinkUrl).toBe(
'https://meet.google.com/abc-defg-hij',
);
expect(formattedEvent.participants).toHaveLength(2);
expect(formattedEvent.participants[0].handle).toBe('organizer@example.com');
expect(formattedEvent.participants[0].isOrganizer).toBe(true);
expect(formattedEvent.participants[0].responseStatus).toBe(
CalendarEventParticipantResponseStatus.ACCEPTED,
);
expect(formattedEvent.participants[1].handle).toBe('attendee@example.com');
expect(formattedEvent.participants[1].responseStatus).toBe(
CalendarEventParticipantResponseStatus.TENTATIVE,
);
});
it('should sanitize a UCALID with improper exit char 0x00', () => {
const mockGoogleEventWithImproperUcalid: calendarV3.Schema$Event = {
...mockGoogleEvent,
iCalUID: '\u0000eventStrange@google.com',
};
const mockGoogleEventWithImproperUcalid2: calendarV3.Schema$Event = {
...mockGoogleEvent,
iCalUID: '>\u0000\u0015-;_<>^<5E>W&<26>p\u001f<31>',
};
const result = formatGoogleCalendarEvents([
mockGoogleEventWithImproperUcalid,
mockGoogleEventWithImproperUcalid2,
]);
expect(result[0].iCalUID).toBe('eventStrange@google.com');
expect(result[1].iCalUID).toBe('>\u0015-;_<>^<5E>W&<26>p\u001f<31>');
});
});

23
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/utils/format-google-calendar-event.util.ts
View File

@ -1,5 +1,6 @@
import { calendar_v3 as calendarV3 } from 'googleapis';
import { sanitizeCalendarEvent } from 'src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent';
import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
@ -25,7 +26,8 @@ const formatGoogleCalendarEvent = (
}
};
return {
// Create the event object
const calendarEvent: CalendarEventWithParticipants = {
title: event.summary ?? '',
isCanceled: event.status === 'cancelled',
isFullDay: event.start?.dateTime == null,
@ -51,4 +53,23 @@ const formatGoogleCalendarEvent = (
})) ?? [],
status: event.status ?? '',
};
const propertiesToSanitize: (keyof CalendarEventWithParticipants)[] = [
'title',
'startsAt',
'endsAt',
'externalId',
'externalCreatedAt',
'externalUpdatedAt',
'description',
'location',
'iCalUID',
'conferenceSolution',
'conferenceLinkLabel',
'conferenceLinkUrl',
'recurringEventExternalId',
'status',
];
return sanitizeCalendarEvent(calendarEvent, propertiesToSanitize);
};

106
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/__tests__/format-microsoft-calendar-event.util.spec.ts Normal file
View File

@ -0,0 +1,106 @@
import { Event } from '@microsoft/microsoft-graph-types';
import { formatMicrosoftCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util';
import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
describe('formatMicrosoftCalendarEvents', () => {
const mockMicrosoftEvent: Event = {
id: 'event123',
subject: 'Team Meeting',
body: {
content: 'Weekly team sync',
contentType: 'text',
},
location: {
displayName: 'Conference Room A',
},
isCancelled: false,
isAllDay: false,
createdDateTime: '2023-01-01T10:00:00Z',
lastModifiedDateTime: '2023-01-02T10:00:00Z',
iCalUId: 'event123@microsoft.com',
start: {
dateTime: '2023-01-15T14:00:00Z',
timeZone: 'UTC',
},
end: {
dateTime: '2023-01-15T15:00:00Z',
timeZone: 'UTC',
},
attendees: [
{
emailAddress: {
address: 'organizer@example.com',
name: 'Meeting Organizer',
},
status: {
response: 'organizer',
},
},
{
emailAddress: {
address: 'attendee@example.com',
name: 'Test Attendee',
},
status: {
response: 'tentativelyAccepted',
},
},
],
onlineMeetingProvider: 'teamsForBusiness',
onlineMeeting: {
joinUrl: 'https://teams.microsoft.com/l/meetup-join/abc123',
},
};
it('should correctly format a normal Microsoft Calendar event', () => {
const result = formatMicrosoftCalendarEvents([mockMicrosoftEvent]);
expect(result).toHaveLength(1);
const formattedEvent = result[0];
expect(formattedEvent.title).toBe('Team Meeting');
expect(formattedEvent.description).toBe('Weekly team sync');
expect(formattedEvent.location).toBe('Conference Room A');
expect(formattedEvent.isCanceled).toBe(false);
expect(formattedEvent.isFullDay).toBe(false);
expect(formattedEvent.startsAt).toBe('2023-01-15T14:00:00Z');
expect(formattedEvent.endsAt).toBe('2023-01-15T15:00:00Z');
expect(formattedEvent.externalId).toBe('event123');
expect(formattedEvent.conferenceSolution).toBe('teamsForBusiness');
expect(formattedEvent.conferenceLinkUrl).toBe(
'https://teams.microsoft.com/l/meetup-join/abc123',
);
expect(formattedEvent.participants).toHaveLength(2);
expect(formattedEvent.participants[0].handle).toBe('organizer@example.com');
expect(formattedEvent.participants[0].isOrganizer).toBe(true);
expect(formattedEvent.participants[0].responseStatus).toBe(
CalendarEventParticipantResponseStatus.ACCEPTED,
);
expect(formattedEvent.participants[1].handle).toBe('attendee@example.com');
expect(formattedEvent.participants[1].responseStatus).toBe(
CalendarEventParticipantResponseStatus.TENTATIVE,
);
});
it('should sanitize a Microsoft Calendar event with improper exit char 0x00', () => {
const mockMicrosoftEventWithImproperData: Event = {
...mockMicrosoftEvent,
iCalUId: '\u0000eventStrange@microsoft.com',
};
const mockMicrosoftEventWithImproperData2: Event = {
...mockMicrosoftEvent,
iCalUId: '>\u0000\u0015-;_<>^<5E>W&<26>p\u001f<31>',
};
const result = formatMicrosoftCalendarEvents([
mockMicrosoftEventWithImproperData,
mockMicrosoftEventWithImproperData2,
]);
expect(result[0].iCalUID).toBe('eventStrange@microsoft.com');
expect(result[1].iCalUID).toBe('>\u0015-;_<>^<5E>W&<26>p\u001f<31>');
});
});

22
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/microsoft-calendar/utils/format-microsoft-calendar-event.util.ts
View File

@ -4,6 +4,7 @@ import {
ResponseType,
} from '@microsoft/microsoft-graph-types';
import { sanitizeCalendarEvent } from 'src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent';
import { CalendarEventParticipantResponseStatus } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
@ -32,7 +33,7 @@ const formatMicrosoftCalendarEvent = (
}
};
return {
const calendarEvent: CalendarEventWithParticipants = {
title: event.subject ?? '',
isCanceled: !!event.isCancelled,
isFullDay: !!event.isAllDay,
@ -57,4 +58,23 @@ const formatMicrosoftCalendarEvent = (
})) ?? [],
status: '',
};
const propertiesToSanitize: (keyof CalendarEventWithParticipants)[] = [
'title',
'startsAt',
'endsAt',
'externalId',
'externalCreatedAt',
'externalUpdatedAt',
'description',
'location',
'iCalUID',
'conferenceSolution',
'conferenceLinkLabel',
'conferenceLinkUrl',
'recurringEventExternalId',
'status',
];
return sanitizeCalendarEvent(calendarEvent, propertiesToSanitize);
};

26
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/utils/sanitizeCalendarEvent.ts Normal file
View File

@ -0,0 +1,26 @@
import { isDefined } from 'twenty-shared/utils';
export const sanitizeCalendarEvent = <T extends Record<string, any>>(
event: T,
propertiesToSanitize: (keyof T)[],
): T => {
const sanitizedEvent = { ...event };
for (const property of propertiesToSanitize) {
if (!isDefined(sanitizedEvent[property])) {
continue;
}
if (typeof sanitizedEvent[property] !== 'string') {
continue;
}
sanitizedEvent[property] = sanitizeString(
sanitizedEvent[property],
) as T[typeof property];
}
return sanitizedEvent;
};
const sanitizeString = (value: string): string => {
return value.replace('\u0000', '');
};