6657 Refactor and fix blocklist (#6803)

Closes #6657 - Fix listeners - Refactor jobs to take array of events - Fix calendar events and messages deletion --------- Co-authored-by: Charles Bochet <charles@twenty.com>
2024-08-31 16:38:47 +02:00
parent d9650fd5cf
commit cd66ea74a2
37 changed files with 799 additions and 699 deletions
--- a/packages/twenty-server/src/modules/calendar/blocklist-manager/calendar-blocklist-manager.module.ts
+++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/calendar-blocklist-manager.module.ts
@ -4,10 +4,10 @@ import { BlocklistItemDeleteCalendarEventsJob } from 'src/modules/calendar/block
 import { BlocklistReimportCalendarEventsJob } from 'src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job';
 import { CalendarBlocklistListener } from 'src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener';
 import { CalendarEventCleanerModule } from 'src/modules/calendar/calendar-event-cleaner/calendar-event-cleaner.module';
-import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module';
+import { CalendarCommonModule } from 'src/modules/calendar/common/calendar-common.module';

@Module({
-  imports: [CalendarEventCleanerModule, CalendarEventImportManagerModule],
+  imports: [CalendarEventCleanerModule, CalendarCommonModule],
  providers: [
    CalendarBlocklistListener,
    BlocklistItemDeleteCalendarEventsJob,
--- a/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-item-delete-calendar-events.job.ts
+++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-item-delete-calendar-events.job.ts
@ -1,20 +1,21 @@
 import { Logger, Scope } from '@nestjs/common';

-import { Any, ILike } from 'typeorm';
+import { And, Any, ILike, In, Not, Or } from 'typeorm';

+import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
 import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
 import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
-import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
 import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
-import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
+import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
 import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
 import { CalendarEventCleanerService } from 'src/modules/calendar/calendar-event-cleaner/services/calendar-event-cleaner.service';
+import { CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
+import { CalendarChannelWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';

-export type BlocklistItemDeleteCalendarEventsJobData = {
-  workspaceId: string;
-  blocklistItemId: string;
-};
+export type BlocklistItemDeleteCalendarEventsJobData = WorkspaceEventBatch<
+  ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
+>;

@Processor({
  queueName: MessageQueue.calendarQueue,
@ -27,77 +28,133 @@ export class BlocklistItemDeleteCalendarEventsJob {

  constructor(
    private readonly twentyORMManager: TwentyORMManager,
-    @InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
-    private readonly blocklistRepository: BlocklistRepository,
    private readonly calendarEventCleanerService: CalendarEventCleanerService,
  ) {}

  @Process(BlocklistItemDeleteCalendarEventsJob.name)
  async handle(data: BlocklistItemDeleteCalendarEventsJobData): Promise<void> {
-    const { workspaceId, blocklistItemId } = data;
+    const workspaceId = data.workspaceId;

-    const blocklistItem = await this.blocklistRepository.getById(
-      blocklistItemId,
-      workspaceId,
+    const blocklistItemIds = data.events.map(
+      (eventPayload) => eventPayload.recordId,
    );

-    if (!blocklistItem) {
-      this.logger.log(
-        `Blocklist item with id ${blocklistItemId} not found in workspace ${workspaceId}`,
+    const blocklistRepository =
+      await this.twentyORMManager.getRepository<BlocklistWorkspaceEntity>(
+        'blocklist',
      );

-      return;
-    }
-
-    const { handle, workspaceMemberId } = blocklistItem;
-
-    this.logger.log(
-      `Deleting calendar events from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
-    );
-
-    if (!workspaceMemberId) {
-      throw new Error(
-        `Workspace member ID is undefined for blocklist item ${blocklistItemId} in workspace ${workspaceId}`,
-      );
-    }
-
-    const calendarChannelRepository =
-      await this.twentyORMManager.getRepository('calendarChannel');
-
-    const calendarChannels = await calendarChannelRepository.find({
+    const blocklist = await blocklistRepository.find({
      where: {
-        connectedAccount: {
-          accountOwnerId: workspaceMemberId,
-        },
+        id: Any(blocklistItemIds),
      },
    });

-    const calendarChannelIds = calendarChannels.map(({ id }) => id);
+    const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
+      (acc, blocklistItem) => {
+        const { handle, workspaceMemberId } = blocklistItem;

-    const isHandleDomain = handle.startsWith('@');
+        if (!acc.has(workspaceMemberId)) {
+          acc.set(workspaceMemberId, []);
+        }
+
+        acc.get(workspaceMemberId)?.push(handle);
+
+        return acc;
+      },
+      new Map<string, string[]>(),
+    );
+
+    const calendarChannelRepository =
+      await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
+        'calendarChannel',
+      );

    const calendarChannelEventAssociationRepository =
-      await this.twentyORMManager.getRepository(
+      await this.twentyORMManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
        'calendarChannelEventAssociation',
      );

-    await calendarChannelEventAssociationRepository.delete({
-      calendarEvent: {
-        calendarEventParticipants: {
-          handle: isHandleDomain ? ILike(`%${handle}`) : handle,
+    for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
+      const handles =
+        handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
+
+      if (!handles) {
+        continue;
+      }
+
+      this.logger.log(
+        `Deleting calendar events from ${handles.join(
+          ', ',
+        )} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
+      );
+
+      const calendarChannels = await calendarChannelRepository.find({
+        select: {
+          id: true,
+          handle: true,
+          connectedAccount: {
+            handleAliases: true,
+          },
        },
-        calendarChannelEventAssociations: {
-          calendarChannelId: Any(calendarChannelIds),
+        where: {
+          connectedAccount: {
+            accountOwnerId: workspaceMemberId,
+          },
        },
-      },
-    });
+        relations: ['connectedAccount'],
+      });
+
+      for (const calendarChannel of calendarChannels) {
+        const calendarChannelHandles = [calendarChannel.handle];
+
+        if (calendarChannel.connectedAccount.handleAliases) {
+          calendarChannelHandles.push(
+            ...calendarChannel.connectedAccount.handleAliases.split(','),
+          );
+        }
+
+        const handleConditions = handles.map((handle) => {
+          const isHandleDomain = handle.startsWith('@');
+
+          return isHandleDomain
+            ? {
+                handle: And(
+                  Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
+                  Not(In(calendarChannelHandles)),
+                ),
+              }
+            : { handle };
+        });
+
+        const calendarEventsAssociationsToDelete =
+          await calendarChannelEventAssociationRepository.find({
+            where: {
+              calendarChannelId: calendarChannel.id,
+              calendarEvent: {
+                calendarEventParticipants: handleConditions,
+              },
+            },
+          });
+
+        if (calendarEventsAssociationsToDelete.length === 0) {
+          continue;
+        }
+
+        await calendarChannelEventAssociationRepository.delete(
+          calendarEventsAssociationsToDelete.map(({ id }) => id),
+        );
+      }
+
+      this.logger.log(
+        `Deleted calendar events from handle ${handles.join(
+          ', ',
+        )} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
+      );
+    }

    await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
      workspaceId,
    );
-
-    this.logger.log(
-      `Deleted calendar events from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
-    );
  }
 }
--- a/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job.ts
+++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/jobs/blocklist-reimport-calendar-events.job.ts
@ -1,23 +1,23 @@
 import { Scope } from '@nestjs/common';

-import { Any } from 'typeorm';
+import { Not } from 'typeorm';

+import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
 import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
 import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
 import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
-import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
 import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
+import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
+import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
+import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
 import {
  CalendarChannelSyncStage,
  CalendarChannelWorkspaceEntity,
 } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
-import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
-import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';

-export type BlocklistReimportCalendarEventsJobData = {
-  workspaceId: string;
-  workspaceMemberId: string;
-};
+export type BlocklistReimportCalendarEventsJobData = WorkspaceEventBatch<
+  ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
+>;

@Processor({
  queueName: MessageQueue.calendarQueue,
@ -26,39 +26,38 @@ export type BlocklistReimportCalendarEventsJobData = {
 export class BlocklistReimportCalendarEventsJob {
  constructor(
    private readonly twentyORMManager: TwentyORMManager,
-    @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
-    private readonly connectedAccountRepository: ConnectedAccountRepository,
+    private readonly calendarChannelSyncStatusService: CalendarChannelSyncStatusService,
  ) {}

  @Process(BlocklistReimportCalendarEventsJob.name)
  async handle(data: BlocklistReimportCalendarEventsJobData): Promise<void> {
-    const { workspaceId, workspaceMemberId } = data;
-
-    const connectedAccounts =
-      await this.connectedAccountRepository.getAllByWorkspaceMemberId(
-        workspaceMemberId,
-        workspaceId,
-      );
-
-    if (!connectedAccounts || connectedAccounts.length === 0) {
-      return;
-    }
+    const workspaceId = data.workspaceId;

    const calendarChannelRepository =
      await this.twentyORMManager.getRepository<CalendarChannelWorkspaceEntity>(
        'calendarChannel',
      );

-    await calendarChannelRepository.update(
-      {
-        connectedAccountId: Any(
-          connectedAccounts.map((connectedAccount) => connectedAccount.id),
-        ),
-      },
-      {
-        syncStage:
-          CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
-      },
-    );
+    for (const eventPayload of data.events) {
+      const workspaceMemberId =
+        eventPayload.properties.before.workspaceMemberId;
+
+      const calendarChannels = await calendarChannelRepository.find({
+        select: ['id'],
+        where: {
+          connectedAccount: {
+            accountOwnerId: workspaceMemberId,
+          },
+          syncStage: Not(
+            CalendarChannelSyncStage.FULL_CALENDAR_EVENT_LIST_FETCH_PENDING,
+          ),
+        },
+      });
+
+      await this.calendarChannelSyncStatusService.resetAndScheduleFullCalendarEventListFetch(
+        calendarChannels.map((calendarChannel) => calendarChannel.id),
+        workspaceId,
+      );
+    }
  }
 }
--- a/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts
+++ b/packages/twenty-server/src/modules/calendar/blocklist-manager/listeners/calendar-blocklist.listener.ts
@ -31,16 +31,9 @@ export class CalendarBlocklistListener {
      ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
    >,
  ) {
-    await Promise.all(
-      payload.events.map((eventPayload) =>
-        this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
-          BlocklistItemDeleteCalendarEventsJob.name,
-          {
-            workspaceId: payload.workspaceId,
-            blocklistItemId: eventPayload.recordId,
-          },
-        ),
-      ),
+    await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
+      BlocklistItemDeleteCalendarEventsJob.name,
+      payload,
    );
  }

@ -50,17 +43,9 @@ export class CalendarBlocklistListener {
      ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
    >,
  ) {
-    await Promise.all(
-      payload.events.map((eventPayload) =>
-        this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
-          BlocklistReimportCalendarEventsJob.name,
-          {
-            workspaceId: payload.workspaceId,
-            workspaceMemberId:
-              eventPayload.properties.before.workspaceMember.id,
-          },
-        ),
-      ),
+    await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
+      BlocklistReimportCalendarEventsJob.name,
+      payload,
    );
  }

@ -70,31 +55,14 @@ export class CalendarBlocklistListener {
      ObjectRecordUpdateEvent<BlocklistWorkspaceEntity>
    >,
  ) {
-    await Promise.all(
-      payload.events.reduce((acc: Promise<void>[], eventPayload) => {
-        acc.push(
-          this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
-            BlocklistItemDeleteCalendarEventsJob.name,
-            {
-              workspaceId: payload.workspaceId,
-              blocklistItemId: eventPayload.recordId,
-            },
-          ),
-        );
+    await this.messageQueueService.add<BlocklistItemDeleteCalendarEventsJobData>(
+      BlocklistItemDeleteCalendarEventsJob.name,
+      payload,
+    );

-        acc.push(
-          this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
-            BlocklistReimportCalendarEventsJob.name,
-            {
-              workspaceId: payload.workspaceId,
-              workspaceMemberId:
-                eventPayload.properties.after.workspaceMember.id,
-            },
-          ),
-        );
-
-        return acc;
-      }, []),
+    await this.messageQueueService.add<BlocklistReimportCalendarEventsJobData>(
+      BlocklistReimportCalendarEventsJob.name,
+      payload,
    );
  }
 }