6657 Refactor and fix blocklist (#6803)

Closes #6657
- Fix listeners
- Refactor jobs to take array of events
- Fix calendar events and messages deletion

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Raphaël Bosi
2024-08-31 16:38:47 +02:00
committed by GitHub
parent d9650fd5cf
commit cd66ea74a2
37 changed files with 799 additions and 699 deletions

View File

@ -4,10 +4,10 @@ import { BlocklistItemDeleteCalendarEventsJob } from 'src/modules/calendar/block
import { BlocklistReimportCalendarEventsJob } from 'src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job';
import { CalendarBlocklistListener } from 'src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener';
import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module';
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
@Module({
imports: [CalendarEventCleanerModule, CalendarEventImportManagerModule],
imports: [CalendarEventCleanerModule, CalendarCommonModule],
providers: [
CalendarBlocklistListener,
BlocklistItemDeleteCalendarEventsJob,

View File

@ -1,20 +1,21 @@
import { Logger, Scope } from '@nestjs/common';
import { Any, ILike } from 'typeorm';
import { And, Any, ILike, In, Not, Or } from 'typeorm';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export type BlocklistItemDeleteCalendarEventsJobData = {
workspaceId: string;
blocklistItemId: string;
};
export type BlocklistItemDeleteCalendarEventsJobData = WorkspaceEventBatch<
ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
>;
@Processor({
queueName: MessageQueue.calendarQueue,
@ -27,77 +28,133 @@ export class BlocklistItemDeleteCalendarEventsJob {
constructor(
private readonly twentyORMManager: TwentyORMManager,
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly calendarEventCleanerService: CalendarEventCleanerService,
) {}
@Process(BlocklistItemDeleteCalendarEventsJob.name)
async handle(data: BlocklistItemDeleteCalendarEventsJobData): Promise<void> {
const { workspaceId, blocklistItemId } = data;
const workspaceId = data.workspaceId;
const blocklistItem = await this.blocklistRepository.getById(
blocklistItemId,
workspaceId,
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
if (!blocklistItem) {
this.logger.log(
`Blocklist item with id ${blocklistItemId} not found in workspace ${workspaceId}`,
const blocklistRepository =
await this.twentyORMManager.getRepository<BlocklistWorkspaceEntity>(
'blocklist',
);
return;
}
const { handle, workspaceMemberId } = blocklistItem;
this.logger.log(
`Deleting calendar events from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
if (!workspaceMemberId) {
throw new Error(
`Workspace member ID is undefined for blocklist item ${blocklistItemId} in workspace ${workspaceId}`,
);
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository('calendarChannel');
const calendarChannels = await calendarChannelRepository.find({
const blocklist = await blocklistRepository.find({
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
id: Any(blocklistItemIds),
},
});
const calendarChannelIds = calendarChannels.map(({ id }) => id);
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
const isHandleDomain = handle.startsWith('@');
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
const calendarChannelEventAssociationRepository =
await this.twentyORMManager.getRepository(
await this.twentyORMManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
'calendarChannelEventAssociation',
);
await calendarChannelEventAssociationRepository.delete({
calendarEvent: {
calendarEventParticipants: {
handle: isHandleDomain ? ILike(`%${handle}`) : handle,
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
this.logger.log(
`Deleting calendar events from ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
const calendarChannels = await calendarChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
calendarChannelEventAssociations: {
calendarChannelId: Any(calendarChannelIds),
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
},
},
});
relations: ['connectedAccount'],
});
for (const calendarChannel of calendarChannels) {
const calendarChannelHandles = [calendarChannel.handle];
if (calendarChannel.connectedAccount.handleAliases) {
calendarChannelHandles.push(
...calendarChannel.connectedAccount.handleAliases.split(','),
);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(calendarChannelHandles)),
),
}
: { handle };
});
const calendarEventsAssociationsToDelete =
await calendarChannelEventAssociationRepository.find({
where: {
calendarChannelId: calendarChannel.id,
calendarEvent: {
calendarEventParticipants: handleConditions,
},
},
});
if (calendarEventsAssociationsToDelete.length === 0) {
continue;
}
await calendarChannelEventAssociationRepository.delete(
calendarEventsAssociationsToDelete.map(({ id }) => id),
);
}
this.logger.log(
`Deleted calendar events from handle ${handles.join(
', ',
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
}
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
this.logger.log(
`Deleted calendar events from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
);
}
}

View File

@ -1,23 +1,23 @@
import { Scope } from '@nestjs/common';
import { Any } from 'typeorm';
import { Not } from 'typeorm';
import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
} from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
export type BlocklistReimportCalendarEventsJobData = {
workspaceId: string;
workspaceMemberId: string;
};
export type BlocklistReimportCalendarEventsJobData = WorkspaceEventBatch<
ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
>;
@Processor({
queueName: MessageQueue.calendarQueue,
@ -26,39 +26,38 @@ export type BlocklistReimportCalendarEventsJobData = {
export class BlocklistReimportCalendarEventsJob {
constructor(
private readonly twentyORMManager: TwentyORMManager,
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
private readonly connectedAccountRepository: ConnectedAccountRepository,
private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
) {}
@Process(BlocklistReimportCalendarEventsJob.name)
async handle(data: BlocklistReimportCalendarEventsJobData): Promise<void> {
const { workspaceId, workspaceMemberId } = data;
const connectedAccounts =
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
workspaceMemberId,
workspaceId,
);
if (!connectedAccounts || connectedAccounts.length === 0) {
return;
}
const workspaceId = data.workspaceId;
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(
{
connectedAccountId: Any(
connectedAccounts.map((connectedAccount) => connectedAccount.id),
),
},
{
syncStage:
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
},
);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const calendarChannels = await calendarChannelRepository.find({
select: ['id'],
where: {
connectedAccount: {
accountOwnerId: workspaceMemberId,
},
syncStage: Not(
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
),
},
});
await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}
}
}

View File

@ -31,16 +31,9 @@ export class CalendarBlocklistListener {
ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
>,
) {
await Promise.all(
payload.events.map((eventPayload) =>
this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
BlocklistItemDeleteCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
blocklistItemId: eventPayload.recordId,
},
),
),
await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
BlocklistItemDeleteCalendarEventsJob.name,
payload,
);
}
@ -50,17 +43,9 @@ export class CalendarBlocklistListener {
ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
>,
) {
await Promise.all(
payload.events.map((eventPayload) =>
this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
BlocklistReimportCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
workspaceMemberId:
eventPayload.properties.before.workspaceMember.id,
},
),
),
await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
BlocklistReimportCalendarEventsJob.name,
payload,
);
}
@ -70,31 +55,14 @@ export class CalendarBlocklistListener {
ObjectRecordUpdateEvent<BlocklistWorkspaceEntity>
>,
) {
await Promise.all(
payload.events.reduce((acc: Promise<void>[], eventPayload) => {
acc.push(
this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
BlocklistItemDeleteCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
blocklistItemId: eventPayload.recordId,
},
),
);
await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
BlocklistItemDeleteCalendarEventsJob.name,
payload,
);
acc.push(
this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
BlocklistReimportCalendarEventsJob.name,
{
workspaceId: payload.workspaceId,
workspaceMemberId:
eventPayload.properties.after.workspaceMember.id,
},
),
);
return acc;
}, []),
await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
BlocklistReimportCalendarEventsJob.name,
payload,
);
}
}

View File

@ -16,12 +16,13 @@ import { CalendarOngoingStaleCronJob } from 'src/modules/calendar/calendar-event
import { GoogleCalendarDriverModule } from 'src/modules/calendar/calendar-event-import-manager/drivers/google-calendar/google-calendar-driver.module';
import { CalendarEventListFetchJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarOngoingStaleJob } from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-ongoing-stale.job';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarEventImportErrorHandlerService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service';
import { CalendarEventsImportService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-events-import.service';
import { CalendarGetCalendarEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
import { RefreshAccessTokenManagerModule } from 'src/modules/connected-account/refresh-access-token-manager/refresh-access-token-manager.module';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@ -44,6 +45,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
RefreshAccessTokenManagerModule,
CalendarEventParticipantManagerModule,
ConnectedAccountModule,
CalendarCommonModule,
],
providers: [
CalendarChannelSyncStatusService,

View File

@ -6,8 +6,8 @@ import { Process } from 'src/engine/integrations/message-queue/decorators/proces
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { isSyncStale } from 'src/modules/calendar/calendar-event-import-manager/utils/is-sync-stale.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import {
CalendarChannelSyncStage,
CalendarChannelWorkspaceEntity,
@ -54,19 +54,19 @@ export class CalendarOngoingStaleJob {
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to pending`,
);
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt(
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt([
calendarChannel.id,
);
]);
switch (calendarChannel.syncStage) {
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING:
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
break;
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING:
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
calendarChannel.id,
[calendarChannel.id],
);
break;
default:

View File

@ -10,7 +10,7 @@ import {
CalendarEventImportException,
CalendarEventImportExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
export enum CalendarEventImportSyncStep {
@ -81,7 +81,7 @@ export class CalendarEventImportErrorHandlerService {
calendarChannel.throttleFailureCount >= CALENDAR_THROTTLE_MAX_ATTEMPTS
) {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
@ -104,19 +104,19 @@ export class CalendarEventImportErrorHandlerService {
switch (syncStep) {
case CalendarEventImportSyncStep.FULL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.scheduleFullCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
break;
case CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH:
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
break;
case CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT:
await this.calendarChannelSyncStatusService.scheduleCalendarEventsImport(
calendarChannel.id,
[calendarChannel.id],
);
break;
@ -130,7 +130,7 @@ export class CalendarEventImportErrorHandlerService {
workspaceId: string,
): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
}
@ -141,7 +141,7 @@ export class CalendarEventImportErrorHandlerService {
workspaceId: string,
): Promise<void> {
await this.calendarChannelSyncStatusService.markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
@ -163,7 +163,7 @@ export class CalendarEventImportErrorHandlerService {
}
await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
workspaceId,
);
}

View File

@ -7,7 +7,6 @@ import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service';
import {
CalendarEventImportErrorHandlerService,
CalendarEventImportSyncStep,
@ -18,6 +17,7 @@ import {
} from 'src/modules/calendar/calendar-event-import-manager/services/calendar-get-events.service';
import { CalendarSaveEventsService } from 'src/modules/calendar/calendar-event-import-manager/services/calendar-save-events.service';
import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-events.util';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import {
CalendarChannelSyncStage,
@ -50,7 +50,7 @@ export class CalendarEventsImportService {
: CalendarEventImportSyncStep.PARTIAL_CALENDAR_EVENT_LIST_FETCH;
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchOngoing(
calendarChannel.id,
[calendarChannel.id],
);
let calendarEvents: GetCalendarEventsResponse['calendarEvents'] = [];
let nextSyncCursor: GetCalendarEventsResponse['nextSyncCursor'] = '';
@ -81,7 +81,7 @@ export class CalendarEventsImportService {
);
await this.calendarChannelSyncStatusService.schedulePartialCalendarEventListFetch(
calendarChannel.id,
[calendarChannel.id],
);
}
@ -92,7 +92,10 @@ export class CalendarEventsImportService {
const { filteredEvents, cancelledEvents } =
filterEventsAndReturnCancelledEvents(
calendarChannel,
[
calendarChannel.handle,
...connectedAccount.handleAliases.split(','),
],
calendarEvents,
blocklist.map((blocklist) => blocklist.handle),
);
@ -133,8 +136,8 @@ export class CalendarEventsImportService {
},
);
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialMessageListFetch(
calendarChannel.id,
await this.calendarChannelSyncStatusService.markAsCompletedAndSchedulePartialCalendarEventListFetch(
[calendarChannel.id],
);
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(

View File

@ -1,9 +1,8 @@
import { filterOutBlocklistedEvents } from 'src/modules/calendar/calendar-event-import-manager/utils/filter-out-blocklisted-events.util';
import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
export const filterEventsAndReturnCancelledEvents = (
calendarChannel: Pick<CalendarChannelWorkspaceEntity, 'handle'>,
calendarChannelHandles: string[],
events: CalendarEventWithParticipants[],
blocklist: string[],
): {
@ -11,7 +10,7 @@ export const filterEventsAndReturnCancelledEvents = (
cancelledEvents: CalendarEventWithParticipants[];
} => {
const filteredEvents = filterOutBlocklistedEvents(
calendarChannel.handle,
calendarChannelHandles,
events,
blocklist,
);

View File

@ -2,7 +2,7 @@ import { isEmailBlocklisted } from 'src/modules/blocklist/utils/is-email-blockli
import { CalendarEventWithParticipants } from 'src/modules/calendar/common/types/calendar-event';
export const filterOutBlocklistedEvents = (
calendarChannelHandle: string,
calendarChannelHandles: string[],
events: CalendarEventWithParticipants[],
blocklist: string[],
) => {
@ -13,7 +13,7 @@ export const filterOutBlocklistedEvents = (
return event.participants.every(
(attendee) =>
!isEmailBlocklisted(calendarChannelHandle, attendee.handle, blocklist),
!isEmailBlocklisted(calendarChannelHandles, attendee.handle, blocklist),
);
});
};

View File

@ -4,6 +4,7 @@ import { CalendarBlocklistManagerModule } from 'src/modules/calendar/blocklist-m
import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module';
import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/calendar-event-participant-manager/calendar-event-participant-manager.module';
import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';
@Module({
imports: [
@ -11,6 +12,7 @@ import { CalendarEventParticipantManagerModule } from 'src/modules/calendar/cale
CalendarEventCleanerModule,
CalendarEventImportManagerModule,
CalendarEventParticipantManagerModule,
CalendarCommonModule,
],
providers: [],
exports: [],

View File

@ -0,0 +1,18 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
@Module({
imports: [
WorkspaceDataSourceModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
ConnectedAccountModule,
],
providers: [CalendarChannelSyncStatusService],
exports: [CalendarChannelSyncStatusService],
})
export class CalendarCommonModule {}

View File

@ -1,13 +1,11 @@
import { Injectable } from '@nestjs/common';
import { Any } from 'typeorm';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {
CalendarEventImportException,
CalendarEventImportExceptionCode,
} from 'src/modules/calendar/calendar-event-import-manager/exceptions/calendar-event-import.exception';
import {
CalendarChannelSyncStage,
CalendarChannelSyncStatus,
@ -26,39 +24,55 @@ export class CalendarChannelSyncStatusService {
private readonly accountsToReconnectService: AccountsToReconnectService,
) {}
public async scheduleFullCalendarEventListFetch(calendarChannelId: string) {
public async scheduleFullCalendarEventListFetch(
calendarChannelIds: string[],
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncStage:
CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
});
}
public async schedulePartialCalendarEventListFetch(
calendarChannelId: string,
calendarChannelIds: string[],
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncStage:
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
});
}
public async markAsCalendarEventListFetchOngoing(calendarChannelId: string) {
public async markAsCalendarEventListFetchOngoing(
calendarChannelIds: string[],
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
@ -66,58 +80,92 @@ export class CalendarChannelSyncStatusService {
}
public async resetAndScheduleFullCalendarEventListFetch(
calendarChannelId: string,
calendarChannelIds: string[],
workspaceId: string,
) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
if (!calendarChannelIds.length) {
return;
}
for (const calendarChannelId of calendarChannelIds) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
});
await this.scheduleFullCalendarEventListFetch(calendarChannelId);
await this.scheduleFullCalendarEventListFetch(calendarChannelIds);
}
public async resetSyncStageStartedAt(calendarChannelId: string) {
public async resetSyncStageStartedAt(calendarChannelIds: string[]) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncStageStartedAt: null,
});
}
public async scheduleCalendarEventsImport(calendarChannelId: string) {
public async scheduleCalendarEventsImport(calendarChannelIds: string[]) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
});
}
public async markAsCompletedAndSchedulePartialMessageListFetch(
calendarChannelId: string,
) {
public async markAsCalendarEventsImportOngoing(calendarChannelIds: string[]) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
});
}
public async markAsCompletedAndSchedulePartialCalendarEventListFetch(
calendarChannelIds: string[],
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await calendarChannelRepository.update(calendarChannelIds, {
syncStage:
CalendarChannelSyncStage.PARTIAL_CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
@ -125,42 +173,53 @@ export class CalendarChannelSyncStatusService {
syncStageStartedAt: null,
});
await this.schedulePartialCalendarEventListFetch(calendarChannelId);
await this.schedulePartialCalendarEventListFetch(calendarChannelIds);
}
public async markAsFailedUnknownAndFlushCalendarEventsToImport(
calendarChannelId: string,
calendarChannelIds: string[],
workspaceId: string,
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
for (const calendarChannelId of calendarChannelIds) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
}
await calendarChannelRepository.update(calendarChannelId, {
await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED,
});
}
public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
calendarChannelId: string,
calendarChannelIds: string[],
workspaceId: string,
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
await calendarChannelRepository.update(calendarChannelId, {
for (const calendarChannelId of calendarChannelIds) {
await this.cacheStorage.del(
`calendar-events-to-import:${workspaceId}:google-calendar:${calendarChannelId}`,
);
}
await calendarChannelRepository.update(calendarChannelIds, {
syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: CalendarChannelSyncStage.FAILED,
});
@ -170,41 +229,44 @@ export class CalendarChannelSyncStatusService {
'connectedAccount',
);
const calendarChannel = await calendarChannelRepository.findOne({
where: { id: calendarChannelId },
const calendarChannels = await calendarChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(calendarChannelIds) },
});
if (!calendarChannel) {
throw new CalendarEventImportException(
`Calendar channel ${calendarChannelId} not found in workspace ${workspaceId}`,
CalendarEventImportExceptionCode.CALENDAR_CHANNEL_NOT_FOUND,
);
}
const connectedAccountId = calendarChannel.connectedAccountId;
const connectedAccountIds = calendarChannels.map(
(calendarChannel) => calendarChannel.connectedAccountId,
);
await connectedAccountRepository.update(
{ id: connectedAccountId },
{ id: Any(connectedAccountIds) },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(calendarChannelId, workspaceId);
await this.addToAccountsToReconnect(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}
private async addToAccountsToReconnect(
calendarChannelId: string,
calendarChannelIds: string[],
workspaceId: string,
) {
if (!calendarChannelIds.length) {
return;
}
const calendarChannelRepository =
await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
'calendarChannel',
);
const calendarChannel = await calendarChannelRepository.findOne({
const calendarChannels = await calendarChannelRepository.find({
where: {
id: calendarChannelId,
id: Any(calendarChannelIds),
},
relations: {
connectedAccount: {
@ -213,18 +275,16 @@ export class CalendarChannelSyncStatusService {
},
});
if (!calendarChannel) {
return;
for (const calendarChannel of calendarChannels) {
const userId = calendarChannel.connectedAccount.accountOwner.userId;
const connectedAccountId = calendarChannel.connectedAccount.id;
await this.accountsToReconnectService.addAccountToReconnectByKey(
AccountsToReconnectKeys.ACCOUNTS_TO_RECONNECT_INSUFFICIENT_PERMISSIONS,
userId,
workspaceId,
connectedAccountId,
);
}
const userId = calendarChannel.connectedAccount.accountOwner.userId;
const connectedAccountId = calendarChannel.connectedAccount.id;
await this.accountsToReconnectService.addAccountToReconnectByKey(
AccountsToReconnectKeys.ACCOUNTS_TO_RECONNECT_INSUFFICIENT_PERMISSIONS,
userId,
workspaceId,
connectedAccountId,
);
}
}