bosiraphael
@ -4,8 +4,9 @@ import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
import { CalendarEventCleanerModule } from 'src/modules/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
|
||||
import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module';
|
||||
import { GoogleCalendarFullSyncService } from 'src/modules/calendar/services/google-calendar-full-sync.service';
|
||||
import { GoogleCalendarSyncService } from 'src/modules/calendar/services/google-calendar-sync.service';
|
||||
import { CalendarProvidersModule } from 'src/modules/calendar/services/providers/calendar-providers.module';
|
||||
import { CalendarChannelEventAssociationObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel-event-association.object-metadata';
|
||||
import { CalendarChannelObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel.object-metadata';
|
||||
@ -32,8 +33,9 @@ import { WorkspaceMemberObjectMetadata } from 'src/modules/workspace-member/stan
|
||||
CalendarEventParticipantModule,
|
||||
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
CalendarEventCleanerModule,
|
||||
],
|
||||
providers: [GoogleCalendarFullSyncService],
|
||||
exports: [GoogleCalendarFullSyncService],
|
||||
providers: [GoogleCalendarSyncService],
|
||||
exports: [GoogleCalendarSyncService],
|
||||
})
|
||||
export class GoogleCalendarFullSyncModule {}
|
||||
export class GoogleCalendarSyncModule {}
|
||||
@ -1,8 +1,9 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
import { calendar_v3 as calendarV3 } from 'googleapis';
|
||||
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
|
||||
@ -14,12 +15,9 @@ import { GoogleCalendarClientProvider } from 'src/modules/calendar/services/prov
|
||||
import { googleCalendarSearchFilterExcludeEmails } from 'src/modules/calendar/utils/google-calendar-search-filter.util';
|
||||
import { CalendarChannelEventAssociationRepository } from 'src/modules/calendar/repositories/calendar-channel-event-association.repository';
|
||||
import { CalendarChannelRepository } from 'src/modules/calendar/repositories/calendar-channel.repository';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||
import { CalendarEventRepository } from 'src/modules/calendar/repositories/calendar-event.repository';
|
||||
import { formatGoogleCalendarEvent } from 'src/modules/calendar/utils/format-google-calendar-event.util';
|
||||
import { GoogleCalendarFullSyncJobData } from 'src/modules/calendar/jobs/google-calendar-full-sync.job';
|
||||
import { CalendarEventParticipantRepository } from 'src/modules/calendar/repositories/calendar-event-participant.repository';
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
@ -28,16 +26,16 @@ import { CalendarChannelObjectMetadata } from 'src/modules/calendar/standard-obj
|
||||
import { CalendarChannelEventAssociationObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel-event-association.object-metadata';
|
||||
import { CalendarEventParticipantObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-event-participant.object-metadata';
|
||||
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||
import { CalendarEventCleanerService } from 'src/modules/calendar/services/calendar-event-cleaner/calendar-event-cleaner.service';
|
||||
import { CalendarEventParticipantService } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.service';
|
||||
import { CalendarEventParticipant } from 'src/modules/calendar/types/calendar-event';
|
||||
|
||||
@Injectable()
|
||||
export class GoogleCalendarFullSyncService {
|
||||
private readonly logger = new Logger(GoogleCalendarFullSyncService.name);
|
||||
export class GoogleCalendarSyncService {
|
||||
private readonly logger = new Logger(GoogleCalendarSyncService.name);
|
||||
|
||||
constructor(
|
||||
private readonly googleCalendarClientProvider: GoogleCalendarClientProvider,
|
||||
@Inject(MessageQueue.calendarQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectObjectMetadataRepository(CalendarEventObjectMetadata)
|
||||
@ -56,13 +54,13 @@ export class GoogleCalendarFullSyncService {
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||
private readonly eventEmitter: EventEmitter2,
|
||||
private readonly calendarEventCleanerService: CalendarEventCleanerService,
|
||||
private readonly calendarEventParticipantsService: CalendarEventParticipantService,
|
||||
) {}
|
||||
|
||||
public async startGoogleCalendarFullSync(
|
||||
public async startGoogleCalendarSync(
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
pageToken?: string,
|
||||
): Promise<void> {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
@ -78,7 +76,7 @@ export class GoogleCalendarFullSyncService {
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during sync`,
|
||||
);
|
||||
}
|
||||
|
||||
@ -88,6 +86,8 @@ export class GoogleCalendarFullSyncService {
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const syncToken = calendarChannel?.syncCursor || undefined;
|
||||
|
||||
if (!calendarChannel) {
|
||||
return;
|
||||
}
|
||||
@ -120,30 +120,47 @@ export class GoogleCalendarFullSyncService {
|
||||
|
||||
let startTime = Date.now();
|
||||
|
||||
const googleCalendarEvents = await googleCalendarClient.events.list({
|
||||
calendarId: 'primary',
|
||||
maxResults: 500,
|
||||
pageToken: pageToken,
|
||||
q: googleCalendarSearchFilterExcludeEmails(blocklistedEmails),
|
||||
});
|
||||
let nextSyncToken: string | null | undefined;
|
||||
let nextPageToken: string | undefined;
|
||||
const events: calendarV3.Schema$Event[] = [];
|
||||
|
||||
while (true) {
|
||||
const googleCalendarEvents = await googleCalendarClient.events.list({
|
||||
calendarId: 'primary',
|
||||
maxResults: 500,
|
||||
syncToken,
|
||||
pageToken: nextPageToken,
|
||||
showDeleted: true,
|
||||
q: googleCalendarSearchFilterExcludeEmails(blocklistedEmails),
|
||||
});
|
||||
|
||||
nextSyncToken = googleCalendarEvents.data.nextSyncToken;
|
||||
nextPageToken = googleCalendarEvents.data.nextPageToken || undefined;
|
||||
|
||||
const { items } = googleCalendarEvents.data;
|
||||
|
||||
if (!items || items.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
events.push(...items);
|
||||
|
||||
if (!nextPageToken) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} getting events list in ${
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} getting events list in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
const {
|
||||
items: events,
|
||||
nextPageToken,
|
||||
nextSyncToken,
|
||||
} = googleCalendarEvents.data;
|
||||
|
||||
if (!events || events?.length === 0) {
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
);
|
||||
|
||||
return;
|
||||
@ -163,17 +180,11 @@ export class GoogleCalendarFullSyncService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: getting existing calendar channel event associations in ${
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: getting existing calendar channel event associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
// TODO: In V2, we will also import deleted events by doing batch GET queries on the canceled events
|
||||
// The canceled events start and end are not accessible in the list query
|
||||
const formattedEvents = events
|
||||
.filter((event) => event.status !== 'cancelled')
|
||||
.map((event) => formatGoogleCalendarEvent(event));
|
||||
|
||||
// TODO: When we will be able to add unicity contraint on iCalUID, we will do a INSERT ON CONFLICT DO UPDATE
|
||||
|
||||
const existingEventExternalIds =
|
||||
@ -181,6 +192,22 @@ export class GoogleCalendarFullSyncService {
|
||||
(association) => association.eventExternalId,
|
||||
);
|
||||
|
||||
const existingEventsIds = existingCalendarChannelEventAssociations.map(
|
||||
(association) => association.calendarEventId,
|
||||
);
|
||||
|
||||
const iCalUIDCalendarEventIdMap =
|
||||
await this.calendarEventRepository.getICalUIDCalendarEventIdMap(
|
||||
existingEventsIds,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const formattedEvents = events
|
||||
.filter((event) => event.status !== 'cancelled')
|
||||
.map((event) =>
|
||||
formatGoogleCalendarEvent(event, iCalUIDCalendarEventIdMap),
|
||||
);
|
||||
|
||||
const eventsToSave = formattedEvents.filter(
|
||||
(event) => !existingEventExternalIds.includes(event.externalId),
|
||||
);
|
||||
@ -189,6 +216,10 @@ export class GoogleCalendarFullSyncService {
|
||||
existingEventExternalIds.includes(event.externalId),
|
||||
);
|
||||
|
||||
const cancelledEventExternalIds = events
|
||||
.filter((event) => event.status === 'cancelled')
|
||||
.map((event) => event.id as string);
|
||||
|
||||
const calendarChannelEventAssociationsToSave = eventsToSave.map(
|
||||
(event) => ({
|
||||
calendarEventId: event.id,
|
||||
@ -205,11 +236,7 @@ export class GoogleCalendarFullSyncService {
|
||||
(event) => event.participants,
|
||||
);
|
||||
|
||||
const iCalUIDCalendarEventIdMap =
|
||||
await this.calendarEventRepository.getICalUIDCalendarEventIdMap(
|
||||
eventsToUpdate.map((event) => event.iCalUID),
|
||||
workspaceId,
|
||||
);
|
||||
let newCalendarEventParticipants: CalendarEventParticipant[] = [];
|
||||
|
||||
if (events.length > 0) {
|
||||
const dataSourceMetadata =
|
||||
@ -230,9 +257,9 @@ export class GoogleCalendarFullSyncService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: saving ${
|
||||
eventsToSave.length
|
||||
} events in ${endTime - startTime}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
@ -246,9 +273,9 @@ export class GoogleCalendarFullSyncService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: updating ${
|
||||
eventsToUpdate.length
|
||||
} events in ${endTime - startTime}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
@ -262,7 +289,27 @@ export class GoogleCalendarFullSyncService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving calendar channel event associations in ${
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: saving calendar channel event associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
newCalendarEventParticipants =
|
||||
await this.calendarEventParticipantsRepository.updateCalendarEventParticipantsAndReturnNewOnes(
|
||||
participantsToUpdate,
|
||||
iCalUIDCalendarEventIdMap,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
participantsToSave.push(...newCalendarEventParticipants);
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: updating participants in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
@ -278,16 +325,16 @@ export class GoogleCalendarFullSyncService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving participants in ${
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: saving participants in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarEventParticipantsRepository.updateCalendarEventParticipants(
|
||||
participantsToUpdate,
|
||||
iCalUIDCalendarEventIdMap,
|
||||
await this.calendarChannelEventAssociationRepository.deleteByEventExternalIdsAndCalendarChannelId(
|
||||
cancelledEventExternalIds,
|
||||
calendarChannelId,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
@ -295,35 +342,47 @@ export class GoogleCalendarFullSyncService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating participants in ${
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: deleting calendar channel event associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
});
|
||||
|
||||
if (calendarChannel.isContactAutoCreationEnabled) {
|
||||
const contactsToCreate = participantsToSave;
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: cleaning calendar events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
if (calendarChannel.isContactAutoCreationEnabled) {
|
||||
this.eventEmitter.emit(`createContacts`, {
|
||||
workspaceId,
|
||||
connectedAccountHandle: connectedAccount.handle,
|
||||
contactsToCreate,
|
||||
contactsToCreate: participantsToSave,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error during google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}`,
|
||||
`Error during google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!nextSyncToken) {
|
||||
throw new Error(
|
||||
`No next sync token found for connected account ${connectedAccountId} in workspace ${workspaceId} during full-sync`,
|
||||
`No next sync token found for connected account ${connectedAccountId} in workspace ${workspaceId} during sync`,
|
||||
);
|
||||
}
|
||||
|
||||
@ -338,29 +397,15 @@ export class GoogleCalendarFullSyncService {
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating sync cursor in ${
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: updating sync cursor in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} ${
|
||||
nextPageToken ? `and ${nextPageToken} pageToken` : ''
|
||||
} done.`,
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} ${
|
||||
syncToken ? `and ${syncToken} syncToken ` : ''
|
||||
}done.`,
|
||||
);
|
||||
|
||||
if (nextPageToken) {
|
||||
await this.messageQueueService.add<GoogleCalendarFullSyncJobData>(
|
||||
GoogleCalendarFullSyncService.name,
|
||||
{
|
||||
workspaceId,
|
||||
connectedAccountId,
|
||||
nextPageToken,
|
||||
},
|
||||
{
|
||||
retryLimit: 2,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user