Refactor calendar to use new sync statuses and stages (#6141)
- Refactor calendar modules and some messaging modules to better organize them by business rules and decouple them - Work toward a common architecture for the different calendar providers by introducing interfaces for the drivers - Modify cron job to use the new sync statuses and stages
This commit is contained in:
bosiraphael
@ -0,0 +1,111 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator';
|
||||
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import {
|
||||
CalendarChannelWorkspaceEntity,
|
||||
CalendarChannelSyncStage,
|
||||
CalendarChannelSyncStatus,
|
||||
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
|
||||
|
||||
@Injectable()
|
||||
export class CalendarChannelSyncStatusService {
|
||||
constructor(
|
||||
@InjectWorkspaceRepository(CalendarChannelWorkspaceEntity)
|
||||
private readonly calendarChannelRepository: WorkspaceRepository<CalendarChannelWorkspaceEntity>,
|
||||
@InjectCacheStorage(CacheStorageNamespace.Calendar)
|
||||
private readonly cacheStorage: CacheStorageService,
|
||||
) {}
|
||||
|
||||
public async scheduleFullCalendarEventListFetch(calendarChannelId: string) {
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStage:
|
||||
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
|
||||
});
|
||||
}
|
||||
|
||||
public async schedulePartialCalendarEventListFetch(
|
||||
calendarChannelId: string,
|
||||
) {
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStage:
|
||||
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
|
||||
});
|
||||
}
|
||||
|
||||
public async markAsCalendarEventListFetchOngoing(calendarChannelId: string) {
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
|
||||
syncStatus: CalendarChannelSyncStatus.ONGOING,
|
||||
syncStageStartedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
public async resetAndScheduleFullCalendarEventListFetch(
|
||||
calendarChannelId: string,
|
||||
workspaceId: string,
|
||||
) {
|
||||
await this.cacheStorage.del(
|
||||
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
|
||||
);
|
||||
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncCursor: '',
|
||||
syncStageStartedAt: null,
|
||||
throttleFailureCount: 0,
|
||||
});
|
||||
|
||||
await this.scheduleFullCalendarEventListFetch(calendarChannelId);
|
||||
}
|
||||
|
||||
public async scheduleCalendarEventsImport(calendarChannelId: string) {
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
|
||||
});
|
||||
}
|
||||
|
||||
public async markAsCalendarEventsImportOngoing(calendarChannelId: string) {
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
|
||||
syncStatus: CalendarChannelSyncStatus.ONGOING,
|
||||
});
|
||||
}
|
||||
|
||||
public async markAsCalendarEventsImportCompleted(calendarChannelId: string) {
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
|
||||
syncStatus: CalendarChannelSyncStatus.ACTIVE,
|
||||
});
|
||||
}
|
||||
|
||||
public async markAsFailedUnknownAndFlushCalendarEventsToImport(
|
||||
calendarChannelId: string,
|
||||
workspaceId: string,
|
||||
) {
|
||||
await this.cacheStorage.del(
|
||||
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
|
||||
);
|
||||
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
|
||||
syncStage: CalendarChannelSyncStage.FAILED,
|
||||
});
|
||||
}
|
||||
|
||||
public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
|
||||
calendarChannelId: string,
|
||||
workspaceId: string,
|
||||
) {
|
||||
await this.cacheStorage.del(
|
||||
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
|
||||
);
|
||||
|
||||
await this.calendarChannelRepository.update(calendarChannelId, {
|
||||
syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
|
||||
syncStage: CalendarChannelSyncStage.FAILED,
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -1,600 +1,111 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Any, Repository } from 'typeorm';
|
||||
import { calendar_v3 as calendarV3 } from 'googleapis';
|
||||
import { GaxiosError } from 'gaxios';
|
||||
import { Any } from 'typeorm';
|
||||
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
|
||||
import {
|
||||
FeatureFlagEntity,
|
||||
FeatureFlagKeys,
|
||||
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||
import { formatGoogleCalendarEvent } from 'src/modules/calendar/calendar-event-import-manager/utils/format-google-calendar-event.util';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
|
||||
import {
|
||||
CalendarEventParticipant,
|
||||
CalendarEventWithParticipants,
|
||||
} from 'src/modules/calendar/common/types/calendar-event';
|
||||
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import {
|
||||
CreateCompanyAndContactJob,
|
||||
CreateCompanyAndContactJobData,
|
||||
} from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
|
||||
import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator';
|
||||
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import { isDefined } from 'src/utils/is-defined';
|
||||
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
|
||||
import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
|
||||
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
|
||||
import { CalendarEventParticipantService } from 'src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service';
|
||||
import { GoogleCalendarClientProvider } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/providers/google-calendar.provider';
|
||||
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
|
||||
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
|
||||
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
|
||||
import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util';
|
||||
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
|
||||
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
|
||||
import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
|
||||
import { CalendarEventWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event.workspace-entity';
|
||||
import { filterOutBlocklistedEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-out-blocklisted-events.util';
|
||||
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
|
||||
import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
|
||||
@Injectable()
|
||||
export class CalendarEventsImportService {
|
||||
private readonly logger = new Logger(CalendarEventsImportService.name);
|
||||
|
||||
constructor(
|
||||
private readonly googleCalendarClientProvider: GoogleCalendarClientProvider,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
@InjectWorkspaceRepository(CalendarEventWorkspaceEntity)
|
||||
private readonly calendarEventRepository: WorkspaceRepository<CalendarEventWorkspaceEntity>,
|
||||
@InjectWorkspaceRepository(CalendarChannelWorkspaceEntity)
|
||||
private readonly calendarChannelRepository: WorkspaceRepository<CalendarChannelWorkspaceEntity>,
|
||||
@InjectWorkspaceRepository(CalendarChannelEventAssociationWorkspaceEntity)
|
||||
private readonly calendarChannelEventAssociationRepository: WorkspaceRepository<CalendarChannelEventAssociationWorkspaceEntity>,
|
||||
@InjectWorkspaceRepository(CalendarEventParticipantWorkspaceEntity)
|
||||
private readonly calendarEventParticipantsRepository: WorkspaceRepository<CalendarEventParticipantWorkspaceEntity>,
|
||||
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
@InjectWorkspaceDatasource()
|
||||
private readonly workspaceDataSource: WorkspaceDataSource,
|
||||
private readonly calendarEventCleanerService: CalendarEventCleanerService,
|
||||
private readonly calendarEventParticipantsService: CalendarEventParticipantService,
|
||||
@InjectMessageQueue(MessageQueue.contactCreationQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
private readonly eventEmitter: EventEmitter2,
|
||||
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
|
||||
private readonly getCalendarEventsService: CalendarGetCalendarEventsService,
|
||||
private readonly calendarSaveEventsService: CalendarSaveEventsService,
|
||||
) {}
|
||||
|
||||
public async processCalendarEventsImport(
|
||||
calendarChannel: CalendarChannelWorkspaceEntity,
|
||||
connectedAccount: ConnectedAccountWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
connectedAccountId: string,
|
||||
emailOrDomainToReimport?: string,
|
||||
): Promise<void> {
|
||||
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||
connectedAccountId,
|
||||
workspaceId,
|
||||
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing(
|
||||
calendarChannel.id,
|
||||
);
|
||||
|
||||
if (!connectedAccount) {
|
||||
return;
|
||||
}
|
||||
const { calendarEvents, nextSyncCursor } =
|
||||
await this.getCalendarEventsService.getCalendarEvents(
|
||||
connectedAccount,
|
||||
calendarChannel.syncCursor,
|
||||
);
|
||||
|
||||
const refreshToken = connectedAccount.refreshToken;
|
||||
const workspaceMemberId = connectedAccount.accountOwnerId;
|
||||
if (!calendarEvents || calendarEvents?.length === 0) {
|
||||
await this.calendarChannelRepository.update(
|
||||
{
|
||||
id: calendarChannel.id,
|
||||
},
|
||||
{
|
||||
syncCursor: nextSyncCursor,
|
||||
},
|
||||
);
|
||||
|
||||
if (!refreshToken) {
|
||||
throw new Error(
|
||||
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId} during sync`,
|
||||
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
|
||||
calendarChannel.id,
|
||||
);
|
||||
}
|
||||
|
||||
const calendarChannel = await this.calendarChannelRepository.findOneBy({
|
||||
connectedAccount: {
|
||||
id: connectedAccountId,
|
||||
},
|
||||
});
|
||||
const blocklist = await this.blocklistRepository.getByWorkspaceMemberId(
|
||||
connectedAccount.accountOwnerId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const syncToken = calendarChannel?.syncCursor || undefined;
|
||||
const { filteredEvents, cancelledEvents } =
|
||||
filterEventsAndReturnCancelledEvents(
|
||||
calendarChannel,
|
||||
calendarEvents,
|
||||
blocklist.map((blocklist) => blocklist.handle),
|
||||
);
|
||||
|
||||
if (!calendarChannel) {
|
||||
return;
|
||||
}
|
||||
const cancelledEventExternalIds = cancelledEvents.map(
|
||||
(event) => event.externalId,
|
||||
);
|
||||
|
||||
const calendarChannelId = calendarChannel.id;
|
||||
|
||||
const { events, nextSyncToken } = await this.getEventsFromGoogleCalendar(
|
||||
await this.calendarSaveEventsService.saveCalendarEventsAndEnqueueContactCreationJob(
|
||||
filteredEvents,
|
||||
calendarChannel,
|
||||
connectedAccount,
|
||||
workspaceId,
|
||||
emailOrDomainToReimport,
|
||||
syncToken,
|
||||
);
|
||||
|
||||
if (!events || events?.length === 0) {
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!workspaceMemberId) {
|
||||
throw new Error(
|
||||
`Workspace member ID is undefined for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||
);
|
||||
}
|
||||
|
||||
const blocklist = await this.getBlocklist(workspaceMemberId, workspaceId);
|
||||
|
||||
let filteredEvents = filterOutBlocklistedEvents(
|
||||
calendarChannel.handle,
|
||||
events,
|
||||
blocklist,
|
||||
).filter((event) => event.status !== 'cancelled');
|
||||
|
||||
if (emailOrDomainToReimport) {
|
||||
filteredEvents = filteredEvents.filter(
|
||||
(event) =>
|
||||
event.attendees?.some(
|
||||
(attendee) => attendee.email?.endsWith(emailOrDomainToReimport),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const cancelledEventExternalIds = filteredEvents
|
||||
.filter((event) => event.status === 'cancelled')
|
||||
.map((event) => event.id as string);
|
||||
|
||||
const existingCalendarEvents = await this.calendarEventRepository.find({
|
||||
where: {
|
||||
iCalUID: Any(filteredEvents.map((event) => event.iCalUID as string)),
|
||||
await this.calendarChannelEventAssociationRepository.delete({
|
||||
eventExternalId: Any(cancelledEventExternalIds),
|
||||
calendarChannel: {
|
||||
id: calendarChannel.id,
|
||||
},
|
||||
});
|
||||
|
||||
const iCalUIDCalendarEventIdMap = new Map(
|
||||
existingCalendarEvents.map((calendarEvent) => [
|
||||
calendarEvent.iCalUID,
|
||||
calendarEvent.id,
|
||||
]),
|
||||
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const formattedEvents = filteredEvents.map((event) =>
|
||||
formatGoogleCalendarEvent(event, iCalUIDCalendarEventIdMap),
|
||||
);
|
||||
|
||||
// TODO: When we will be able to add unicity contraint on iCalUID, we will do a INSERT ON CONFLICT DO UPDATE
|
||||
|
||||
let startTime = Date.now();
|
||||
|
||||
const existingEventsICalUIDs = existingCalendarEvents.map(
|
||||
(calendarEvent) => calendarEvent.iCalUID,
|
||||
);
|
||||
|
||||
let endTime = Date.now();
|
||||
|
||||
const eventsToSave = formattedEvents.filter(
|
||||
(calendarEvent) =>
|
||||
!existingEventsICalUIDs.includes(calendarEvent.iCalUID),
|
||||
);
|
||||
|
||||
const eventsToUpdate = formattedEvents.filter((calendarEvent) =>
|
||||
existingEventsICalUIDs.includes(calendarEvent.iCalUID),
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
const existingCalendarChannelEventAssociations =
|
||||
await this.calendarChannelEventAssociationRepository.find({
|
||||
where: {
|
||||
eventExternalId: Any(
|
||||
formattedEvents.map((calendarEvent) => calendarEvent.id),
|
||||
),
|
||||
calendarChannel: {
|
||||
id: calendarChannelId,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: getting existing calendar channel event associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
const calendarChannelEventAssociationsToSave = formattedEvents
|
||||
.filter(
|
||||
(calendarEvent) =>
|
||||
!existingCalendarChannelEventAssociations.some(
|
||||
(association) => association.eventExternalId === calendarEvent.id,
|
||||
),
|
||||
)
|
||||
.map((calendarEvent) => ({
|
||||
calendarEventId: calendarEvent.id,
|
||||
eventExternalId: calendarEvent.externalId,
|
||||
calendarChannelId,
|
||||
}));
|
||||
|
||||
if (events.length > 0) {
|
||||
await this.saveGoogleCalendarEvents(
|
||||
eventsToSave,
|
||||
eventsToUpdate,
|
||||
calendarChannelEventAssociationsToSave,
|
||||
connectedAccount,
|
||||
calendarChannel,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarChannelEventAssociationRepository.delete({
|
||||
eventExternalId: Any(cancelledEventExternalIds),
|
||||
calendarChannel: {
|
||||
id: calendarChannelId,
|
||||
},
|
||||
});
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: deleting calendar channel event associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: cleaning calendar events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!nextSyncToken) {
|
||||
throw new Error(
|
||||
`No next sync token found for connected account ${connectedAccountId} in workspace ${workspaceId} during sync`,
|
||||
);
|
||||
}
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarChannelRepository.update(
|
||||
{
|
||||
id: calendarChannel.id,
|
||||
},
|
||||
{
|
||||
syncCursor: nextSyncToken,
|
||||
syncCursor: nextSyncCursor,
|
||||
},
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId}: updating sync cursor in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
|
||||
calendarChannel.id,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${connectedAccountId} ${
|
||||
syncToken ? `and ${syncToken} syncToken ` : ''
|
||||
}done.`,
|
||||
);
|
||||
}
|
||||
|
||||
public async getBlocklist(workspaceMemberId: string, workspaceId: string) {
|
||||
const isBlocklistEnabledFeatureFlag =
|
||||
await this.featureFlagRepository.findOneBy({
|
||||
workspaceId,
|
||||
key: FeatureFlagKeys.IsBlocklistEnabled,
|
||||
value: true,
|
||||
});
|
||||
|
||||
const isBlocklistEnabled =
|
||||
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
|
||||
|
||||
const blocklist = isBlocklistEnabled
|
||||
? await this.blocklistRepository.getByWorkspaceMemberId(
|
||||
workspaceMemberId,
|
||||
workspaceId,
|
||||
)
|
||||
: [];
|
||||
|
||||
return blocklist.map((blocklist) => blocklist.handle);
|
||||
}
|
||||
|
||||
public async getEventsFromGoogleCalendar(
|
||||
connectedAccount: ConnectedAccountWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
emailOrDomainToReimport?: string,
|
||||
syncToken?: string,
|
||||
): Promise<{
|
||||
events: calendarV3.Schema$Event[];
|
||||
nextSyncToken: string | null | undefined;
|
||||
}> {
|
||||
const googleCalendarClient =
|
||||
await this.googleCalendarClientProvider.getGoogleCalendarClient(
|
||||
connectedAccount,
|
||||
);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
let nextSyncToken: string | null | undefined;
|
||||
let nextPageToken: string | undefined;
|
||||
const events: calendarV3.Schema$Event[] = [];
|
||||
|
||||
let hasMoreEvents = true;
|
||||
|
||||
while (hasMoreEvents) {
|
||||
const googleCalendarEvents = await googleCalendarClient.events
|
||||
.list({
|
||||
calendarId: 'primary',
|
||||
maxResults: 500,
|
||||
syncToken: emailOrDomainToReimport ? undefined : syncToken,
|
||||
pageToken: nextPageToken,
|
||||
q: emailOrDomainToReimport,
|
||||
showDeleted: true,
|
||||
})
|
||||
.catch(async (error: GaxiosError) => {
|
||||
if (error.response?.status !== 410) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await this.calendarChannelRepository.update(
|
||||
{
|
||||
id: connectedAccount.id,
|
||||
},
|
||||
{
|
||||
syncCursor: '',
|
||||
},
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Sync token is no longer valid for connected account ${connectedAccount.id} in workspace ${workspaceId}, resetting sync cursor.`,
|
||||
);
|
||||
|
||||
return {
|
||||
data: {
|
||||
items: [],
|
||||
nextSyncToken: undefined,
|
||||
nextPageToken: undefined,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
nextSyncToken = googleCalendarEvents.data.nextSyncToken;
|
||||
nextPageToken = googleCalendarEvents.data.nextPageToken || undefined;
|
||||
|
||||
const { items } = googleCalendarEvents.data;
|
||||
|
||||
if (!items || items.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
events.push(...items);
|
||||
|
||||
if (!nextPageToken) {
|
||||
hasMoreEvents = false;
|
||||
}
|
||||
}
|
||||
|
||||
const endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
} getting events list in ${endTime - startTime}ms.`,
|
||||
);
|
||||
|
||||
return { events, nextSyncToken };
|
||||
}
|
||||
|
||||
public async saveGoogleCalendarEvents(
|
||||
eventsToSave: CalendarEventWithParticipants[],
|
||||
eventsToUpdate: CalendarEventWithParticipants[],
|
||||
calendarChannelEventAssociationsToSave: {
|
||||
calendarEventId: string;
|
||||
eventExternalId: string;
|
||||
calendarChannelId: string;
|
||||
}[],
|
||||
connectedAccount: ConnectedAccountWorkspaceEntity,
|
||||
calendarChannel: CalendarChannelWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
const participantsToSave = eventsToSave.flatMap(
|
||||
(event) => event.participants,
|
||||
);
|
||||
|
||||
const participantsToUpdate = eventsToUpdate.flatMap(
|
||||
(event) => event.participants,
|
||||
);
|
||||
|
||||
let startTime: number;
|
||||
let endTime: number;
|
||||
|
||||
const savedCalendarEventParticipantsToEmit: CalendarEventParticipantWorkspaceEntity[] =
|
||||
[];
|
||||
|
||||
try {
|
||||
await this.workspaceDataSource?.transaction(
|
||||
async (transactionManager) => {
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarEventRepository.save(
|
||||
eventsToSave,
|
||||
{},
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
}: saving ${eventsToSave.length} events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarChannelRepository.save(
|
||||
eventsToUpdate,
|
||||
{},
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
}: updating ${eventsToUpdate.length} events in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
await this.calendarChannelEventAssociationRepository.save(
|
||||
calendarChannelEventAssociationsToSave,
|
||||
{},
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
}: saving calendar channel event associations in ${
|
||||
endTime - startTime
|
||||
}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
const existingCalendarEventParticipants =
|
||||
await this.calendarEventParticipantsRepository.find({
|
||||
where: {
|
||||
calendarEventId: Any(
|
||||
participantsToUpdate
|
||||
.map((participant) => participant.calendarEventId)
|
||||
.filter(isDefined),
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
const {
|
||||
calendarEventParticipantsToDelete,
|
||||
newCalendarEventParticipants,
|
||||
} = participantsToUpdate.reduce(
|
||||
(acc, calendarEventParticipant) => {
|
||||
const existingCalendarEventParticipant =
|
||||
existingCalendarEventParticipants.find(
|
||||
(existingCalendarEventParticipant) =>
|
||||
existingCalendarEventParticipant.handle ===
|
||||
calendarEventParticipant.handle,
|
||||
);
|
||||
|
||||
if (existingCalendarEventParticipant) {
|
||||
acc.calendarEventParticipantsToDelete.push(
|
||||
existingCalendarEventParticipant,
|
||||
);
|
||||
} else {
|
||||
acc.newCalendarEventParticipants.push(calendarEventParticipant);
|
||||
}
|
||||
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
calendarEventParticipantsToDelete:
|
||||
[] as CalendarEventParticipantWorkspaceEntity[],
|
||||
newCalendarEventParticipants: [] as CalendarEventParticipant[],
|
||||
},
|
||||
);
|
||||
|
||||
await this.calendarEventParticipantsRepository.delete({
|
||||
id: Any(
|
||||
calendarEventParticipantsToDelete.map(
|
||||
(calendarEventParticipant) => calendarEventParticipant.id,
|
||||
),
|
||||
),
|
||||
});
|
||||
|
||||
await this.calendarEventParticipantsRepository.save(
|
||||
participantsToUpdate,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
participantsToSave.push(...newCalendarEventParticipants);
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
}: updating participants in ${endTime - startTime}ms.`,
|
||||
);
|
||||
|
||||
startTime = Date.now();
|
||||
|
||||
const savedCalendarEventParticipants =
|
||||
await this.calendarEventParticipantsService.saveCalendarEventParticipants(
|
||||
participantsToSave,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
savedCalendarEventParticipantsToEmit.push(
|
||||
...savedCalendarEventParticipants,
|
||||
);
|
||||
|
||||
endTime = Date.now();
|
||||
|
||||
this.logger.log(
|
||||
`google calendar sync for workspace ${workspaceId} and account ${
|
||||
connectedAccount.id
|
||||
}: saving participants in ${endTime - startTime}ms.`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
this.eventEmitter.emit(`calendarEventParticipant.matched`, {
|
||||
workspaceId,
|
||||
workspaceMemberId: connectedAccount.accountOwnerId,
|
||||
calendarEventParticipants: savedCalendarEventParticipantsToEmit,
|
||||
});
|
||||
|
||||
if (calendarChannel.isContactAutoCreationEnabled) {
|
||||
await this.messageQueueService.add<CreateCompanyAndContactJobData>(
|
||||
CreateCompanyAndContactJob.name,
|
||||
{
|
||||
workspaceId,
|
||||
connectedAccount,
|
||||
contactsToCreate: participantsToSave,
|
||||
},
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Error during google calendar sync for workspace ${workspaceId} and account ${connectedAccount.id}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,37 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { GoogleCalendarGetEventsService as GoogleCalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/services/google-calendar-get-events.service';
|
||||
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
|
||||
export type GetCalendarEventsResponse = {
|
||||
calendarEvents: CalendarEventWithParticipants[];
|
||||
nextSyncCursor: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CalendarGetCalendarEventsService {
|
||||
constructor(
|
||||
private readonly googleCalendarGetCalendarEventsService: GoogleCalendarGetCalendarEventsService,
|
||||
) {}
|
||||
|
||||
public async getCalendarEvents(
|
||||
connectedAccount: Pick<
|
||||
ConnectedAccountWorkspaceEntity,
|
||||
'provider' | 'refreshToken' | 'id'
|
||||
>,
|
||||
syncCursor?: string,
|
||||
): Promise<GetCalendarEventsResponse> {
|
||||
switch (connectedAccount.provider) {
|
||||
case 'google':
|
||||
return this.googleCalendarGetCalendarEventsService.getCalendarEvents(
|
||||
connectedAccount,
|
||||
syncCursor,
|
||||
);
|
||||
default:
|
||||
throw new Error(
|
||||
`Provider ${connectedAccount.provider} is not supported.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,160 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
|
||||
import { Any } from 'typeorm';
|
||||
|
||||
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import {
|
||||
CreateCompanyAndContactJob,
|
||||
CreateCompanyAndContactJobData,
|
||||
} from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
|
||||
import { InjectWorkspaceRepository } from 'src/engine/twenty-orm/decorators/inject-workspace-repository.decorator';
|
||||
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
|
||||
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
|
||||
import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
|
||||
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
|
||||
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
|
||||
import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity';
|
||||
import { CalendarEventWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event.workspace-entity';
|
||||
import { injectIdsInCalendarEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/inject-ids-in-calendar-events.util';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
|
||||
import { CalendarEventParticipantService } from 'src/modules/calendar/calendar-event-participant-manager/services/calendar-event-participant.service';
|
||||
|
||||
@Injectable()
|
||||
export class CalendarSaveEventsService {
|
||||
constructor(
|
||||
@InjectWorkspaceRepository(CalendarEventWorkspaceEntity)
|
||||
private readonly calendarEventRepository: WorkspaceRepository<CalendarEventWorkspaceEntity>,
|
||||
@InjectWorkspaceRepository(CalendarChannelEventAssociationWorkspaceEntity)
|
||||
private readonly calendarChannelEventAssociationRepository: WorkspaceRepository<CalendarChannelEventAssociationWorkspaceEntity>,
|
||||
@InjectWorkspaceDatasource()
|
||||
private readonly workspaceDataSource: WorkspaceDataSource,
|
||||
private readonly calendarEventParticipantService: CalendarEventParticipantService,
|
||||
@InjectMessageQueue(MessageQueue.contactCreationQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
private readonly eventEmitter: EventEmitter2,
|
||||
) {}
|
||||
|
||||
public async saveCalendarEventsAndEnqueueContactCreationJob(
|
||||
filteredEvents: CalendarEventWithParticipants[],
|
||||
calendarChannel: CalendarChannelWorkspaceEntity,
|
||||
connectedAccount: ConnectedAccountWorkspaceEntity,
|
||||
workspaceId: string,
|
||||
): Promise<void> {
|
||||
const existingCalendarEvents = await this.calendarEventRepository.find({
|
||||
where: {
|
||||
iCalUID: Any(filteredEvents.map((event) => event.iCalUID as string)),
|
||||
},
|
||||
});
|
||||
|
||||
const iCalUIDCalendarEventIdMap = new Map(
|
||||
existingCalendarEvents.map((calendarEvent) => [
|
||||
calendarEvent.iCalUID,
|
||||
calendarEvent.id,
|
||||
]),
|
||||
);
|
||||
|
||||
const calendarEventsWithIds = injectIdsInCalendarEvents(
|
||||
filteredEvents,
|
||||
iCalUIDCalendarEventIdMap,
|
||||
);
|
||||
|
||||
// TODO: When we will be able to add unicity contraint on iCalUID, we will do a INSERT ON CONFLICT DO UPDATE
|
||||
|
||||
const existingEventsICalUIDs = existingCalendarEvents.map(
|
||||
(calendarEvent) => calendarEvent.iCalUID,
|
||||
);
|
||||
|
||||
const eventsToSave = calendarEventsWithIds.filter(
|
||||
(calendarEvent) =>
|
||||
!existingEventsICalUIDs.includes(calendarEvent.iCalUID),
|
||||
);
|
||||
|
||||
const eventsToUpdate = calendarEventsWithIds.filter((calendarEvent) =>
|
||||
existingEventsICalUIDs.includes(calendarEvent.iCalUID),
|
||||
);
|
||||
|
||||
const existingCalendarChannelEventAssociations =
|
||||
await this.calendarChannelEventAssociationRepository.find({
|
||||
where: {
|
||||
eventExternalId: Any(
|
||||
calendarEventsWithIds.map((calendarEvent) => calendarEvent.id),
|
||||
),
|
||||
calendarChannel: {
|
||||
id: calendarChannel.id,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const calendarChannelEventAssociationsToSave = calendarEventsWithIds
|
||||
.filter(
|
||||
(calendarEvent) =>
|
||||
!existingCalendarChannelEventAssociations.some(
|
||||
(association) => association.eventExternalId === calendarEvent.id,
|
||||
),
|
||||
)
|
||||
.map((calendarEvent) => ({
|
||||
calendarEventId: calendarEvent.id,
|
||||
eventExternalId: calendarEvent.externalId,
|
||||
calendarChannelId: calendarChannel.id,
|
||||
}));
|
||||
|
||||
const participantsToSave = eventsToSave.flatMap(
|
||||
(event) => event.participants,
|
||||
);
|
||||
|
||||
const participantsToUpdate = eventsToUpdate.flatMap(
|
||||
(event) => event.participants,
|
||||
);
|
||||
|
||||
const savedCalendarEventParticipantsToEmit: CalendarEventParticipantWorkspaceEntity[] =
|
||||
[];
|
||||
|
||||
await this.workspaceDataSource?.transaction(async (transactionManager) => {
|
||||
await this.calendarEventRepository.save(
|
||||
eventsToSave,
|
||||
{},
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.calendarEventRepository.save(
|
||||
eventsToUpdate,
|
||||
{},
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.calendarChannelEventAssociationRepository.save(
|
||||
calendarChannelEventAssociationsToSave,
|
||||
{},
|
||||
transactionManager,
|
||||
);
|
||||
|
||||
await this.calendarEventParticipantService.upsertAndDeleteCalendarEventParticipants(
|
||||
participantsToSave,
|
||||
participantsToUpdate,
|
||||
workspaceId,
|
||||
transactionManager,
|
||||
);
|
||||
});
|
||||
|
||||
this.eventEmitter.emit(`calendarEventParticipant.matched`, {
|
||||
workspaceId,
|
||||
workspaceMemberId: connectedAccount.accountOwnerId,
|
||||
calendarEventParticipants: savedCalendarEventParticipantsToEmit,
|
||||
});
|
||||
|
||||
if (calendarChannel.isContactAutoCreationEnabled) {
|
||||
await this.messageQueueService.add<CreateCompanyAndContactJobData>(
|
||||
CreateCompanyAndContactJob.name,
|
||||
{
|
||||
workspaceId,
|
||||
connectedAccount,
|
||||
contactsToCreate: participantsToSave,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user