diff --git a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts index 0ea2eacb3..c7c6af34c 100644 --- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts +++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts @@ -2,38 +2,68 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u describe('isEmailBlocklisted', () => { it('should return true if email is blocklisted', () => { - const email = 'hello@example.com'; - const blocklist = ['hello@example.com', 'hey@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const channelHandle = 'abc@example.com'; + const email = 'hello@twenty.com'; + const blocklist = ['hello@twenty.com', 'hey@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(true); }); it('should return false if email is not blocklisted', () => { + const channelHandle = 'abc@example.com'; const email = 'hello@twenty.com'; const blocklist = ['hey@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(false); }); it('should return false if email is null', () => { + const channelHandle = 'abc@twenty.com'; const email = null; const blocklist = ['@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const result = isEmailBlocklisted(channelHandle, email, blocklist); + + expect(result).toBe(false); + }); + it('should return true for subdomains', () => { + const channelHandle = 'abc@example.com'; + const email = 'hello@twenty.twenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); + + expect(result).toBe(true); + }); + it('should return false for domains which end with blocklisted domain but are not subdomains', () => { + const channelHandle = 'abc@example.com'; + const email = 'hello@twentytwenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(false); }); it('should return false if email is undefined', () => { + const channelHandle = 'abc@example.com'; const email = undefined; - const blocklist = ['@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(false); }); it('should return true if email ends with blocklisted domain', () => { - const email = 'hello@example.com'; - const blocklist = ['@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const channelHandle = 'abc@example.com'; + const email = 'hello@twenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(true); }); + + it('should return false if email is same as channel handle', () => { + const channelHandle = 'hello@twenty.com'; + const email = 'hello@twenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); + + expect(result).toBe(false); + }); }); diff --git a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts index 4231863bf..af21bbe1e 100644 --- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts +++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts @@ -1,14 +1,17 @@ export const isEmailBlocklisted = ( + channelHandle: string, email: string | null | undefined, blocklist: string[], ): boolean => { - if (!email) { + if (!email || email === channelHandle) { return false; } return blocklist.some((item) => { if (item.startsWith('@')) { - return email.endsWith(item); + const domain = email.split('@')[1]; + + return domain === item.slice(1) || domain.endsWith(`.${item.slice(1)}`); } return email === item; diff --git a/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts b/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts index 1fdb9f8fb..d0c8663d8 100644 --- a/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts +++ b/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts @@ -124,14 +124,13 @@ export class GoogleCalendarSyncService { const blocklist = await this.getBlocklist(workspaceMemberId, workspaceId); - let filteredEvents = filterOutBlocklistedEvents(events, blocklist).filter( - (event) => event.status !== 'cancelled', - ); + let filteredEvents = filterOutBlocklistedEvents( + calendarChannel.handle, + events, + blocklist, + ).filter((event) => event.status !== 'cancelled'); if (emailOrDomainToReimport) { - // We still need to filter the events to only keep the ones that have the email or domain we want to reimport - // because the q parameter in the list method also filters the events that have the email or domain in their summary, description ... - // The q parameter allows us to narrow down the events filteredEvents = filteredEvents.filter( (event) => event.attendees?.some( diff --git a/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts b/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts index eafc669b1..067e49ea5 100644 --- a/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts +++ b/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts @@ -3,6 +3,7 @@ import { calendar_v3 as calendarV3 } from 'googleapis'; import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util'; export const filterOutBlocklistedEvents = ( + calendarChannelHandle: string, events: calendarV3.Schema$Event[], blocklist: string[], ) => { @@ -12,7 +13,8 @@ export const filterOutBlocklistedEvents = ( } return event.attendees.every( - (attendee) => !isEmailBlocklisted(attendee.email, blocklist), + (attendee) => + !isEmailBlocklisted(calendarChannelHandle, attendee.email, blocklist), ); }); }; diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts index ac2f6defa..3a52e42de 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts @@ -66,12 +66,14 @@ export class BlocklistItemDeleteMessagesJob const rolesToDelete: ('from' | 'to')[] = ['from', 'to']; - await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles( - handle, - messageChannelIds, - rolesToDelete, - workspaceId, - ); + for (const messageChannelId of messageChannelIds) { + await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdAndRoles( + handle, + messageChannelId, + rolesToDelete, + workspaceId, + ); + } await this.threadCleanerService.cleanWorkspaceThreads(workspaceId); diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts deleted file mode 100644 index 12952b6ea..000000000 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; - -import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface'; - -import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; -import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository'; -import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; - -export type BlocklistReimportMessagesJobData = { - workspaceId: string; - workspaceMemberId: string; - handle: string; -}; - -@Injectable() -export class BlocklistReimportMessagesJob - implements MessageQueueJob -{ - private readonly logger = new Logger(BlocklistReimportMessagesJob.name); - - constructor( - @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity) - private readonly connectedAccountRepository: ConnectedAccountRepository, - ) {} - - async handle(data: BlocklistReimportMessagesJobData): Promise { - const { workspaceId, workspaceMemberId, handle } = data; - - this.logger.log( - `Reimporting messages from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`, - ); - - const connectedAccount = - await this.connectedAccountRepository.getAllByWorkspaceMemberId( - workspaceMemberId, - workspaceId, - ); - - if (!connectedAccount || connectedAccount.length === 0) { - this.logger.error( - `No connected account found for workspace member ${workspaceMemberId} in workspace ${workspaceId}`, - ); - - return; - } - - // TODO: reimplement that - // await this.gmailMessageListFetchJob.fetchConnectedAccountThreads( - // workspaceId, - // connectedAccount[0].id, - // [handle], - // ); - - this.logger.log( - `Reimporting messages from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId} done`, - ); - } -} diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts index 8d2ff5e1f..c852bb9ed 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts @@ -6,21 +6,28 @@ import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/t import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity'; -import { - BlocklistReimportMessagesJob, - BlocklistReimportMessagesJobData, -} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job'; import { BlocklistItemDeleteMessagesJobData, BlocklistItemDeleteMessagesJob, } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job'; import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event'; +import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; +import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository'; +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository'; +import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service'; +import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @Injectable() export class MessagingBlocklistListener { constructor( @Inject(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, + @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity) + private readonly connectedAccountRepository: ConnectedAccountRepository, + private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService, + @InjectObjectMetadataRepository(MessageChannelWorkspaceEntity) + private readonly messageChannelRepository: MessageChannelRepository, ) {} @OnEvent('blocklist.created') @@ -40,13 +47,28 @@ export class MessagingBlocklistListener { async handleDeletedEvent( payload: ObjectRecordDeleteEvent, ) { - await this.messageQueueService.add( - BlocklistReimportMessagesJob.name, - { - workspaceId: payload.workspaceId, - workspaceMemberId: payload.properties.before.workspaceMember.id, - handle: payload.properties.before.handle, - }, + const workspaceMemberId = payload.properties.before.workspaceMember.id; + const workspaceId = payload.workspaceId; + + const connectedAccount = + await this.connectedAccountRepository.getAllByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ); + + if (!connectedAccount || connectedAccount.length === 0) { + return; + } + + const messageChannel = + await this.messageChannelRepository.getByConnectedAccountId( + connectedAccount[0].id, + workspaceId, + ); + + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( + messageChannel[0].id, + workspaceId, ); } @@ -54,21 +76,36 @@ export class MessagingBlocklistListener { async handleUpdatedEvent( payload: ObjectRecordUpdateEvent, ) { + const workspaceMemberId = payload.properties.before.workspaceMember.id; + const workspaceId = payload.workspaceId; + await this.messageQueueService.add( BlocklistItemDeleteMessagesJob.name, { - workspaceId: payload.workspaceId, + workspaceId, blocklistItemId: payload.recordId, }, ); - await this.messageQueueService.add( - BlocklistReimportMessagesJob.name, - { - workspaceId: payload.workspaceId, - workspaceMemberId: payload.properties.after.workspaceMember.id, - handle: payload.properties.before.handle, - }, + const connectedAccount = + await this.connectedAccountRepository.getAllByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ); + + if (!connectedAccount || connectedAccount.length === 0) { + return; + } + + const messageChannel = + await this.messageChannelRepository.getByConnectedAccountId( + connectedAccount[0].id, + workspaceId, + ); + + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( + messageChannel[0].id, + workspaceId, ); } } diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts index 0d59ad486..0ed9ad1fe 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts @@ -1,8 +1,19 @@ import { Module } from '@nestjs/common'; +import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job'; +import { MessagingBlocklistListener } from 'src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener'; +import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; +import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module'; + @Module({ - imports: [], - providers: [], + imports: [MessagingCommonModule, MessagingMessageCleanerModule], + providers: [ + MessagingBlocklistListener, + { + provide: BlocklistItemDeleteMessagesJob.name, + useClass: BlocklistItemDeleteMessagesJob, + }, + ], exports: [], }) export class MessagingBlocklistManagerModule {} diff --git a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts index fddbf24c6..8699a7670 100644 --- a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts +++ b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts @@ -67,9 +67,9 @@ export class MessageChannelMessageAssociationRepository { ); } - public async deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles( + public async deleteByMessageParticipantHandleAndMessageChannelIdAndRoles( messageParticipantHandle: string, - messageChannelIds: string[], + messageChannelId: string, rolesToDelete: ('from' | 'to' | 'cc' | 'bcc')[], workspaceId: string, transactionManager?: EntityManager, @@ -79,21 +79,35 @@ export class MessageChannelMessageAssociationRepository { const isHandleDomain = messageParticipantHandle.startsWith('@'); + const messageChannel = + await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."messageChannel" + WHERE "id" = $1`, + [messageChannelId], + workspaceId, + transactionManager, + ); + + const messageChannelHandle = messageChannel[0].handle; + const messageChannelMessageAssociationIdsToDelete = await this.workspaceDataSourceService.executeRawQuery( `SELECT "messageChannelMessageAssociation".id FROM ${dataSourceSchema}."messageChannelMessageAssociation" "messageChannelMessageAssociation" JOIN ${dataSourceSchema}."message" ON "messageChannelMessageAssociation"."messageId" = ${dataSourceSchema}."message"."id" JOIN ${dataSourceSchema}."messageParticipant" "messageParticipant" ON ${dataSourceSchema}."message"."id" = "messageParticipant"."messageId" - WHERE "messageParticipant"."handle" ${ - isHandleDomain ? 'ILIKE' : '=' - } $1 AND "messageParticipant"."role" = ANY($2) AND "messageChannelMessageAssociation"."messageChannelId" = ANY($3)`, + WHERE "messageParticipant"."handle" != $1 + AND "messageParticipant"."handle" ${isHandleDomain ? '~*' : '='} $2 + AND "messageParticipant"."role" = ANY($3) + AND "messageChannelMessageAssociation"."messageChannelId" = $4`, [ + messageChannelHandle, isHandleDomain - ? `%${messageParticipantHandle}` + ? // eslint-disable-next-line no-useless-escape + `.+@(.+\.)?${messageParticipantHandle.slice(1)}` : messageParticipantHandle, rolesToDelete, - messageChannelIds, + messageChannelId, ], workspaceId, transactionManager, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts index 46a0a4c28..8b6e46242 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts @@ -111,6 +111,7 @@ export class MessagingGmailMessagesImportService { ); const messagesToSave = filterEmails( + messageChannel.handle, allMessages, blocklist.map((blocklistItem) => blocklistItem.handle), ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts index 9e1ebecfe..3bd9819f2 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts @@ -2,14 +2,20 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message'; // Todo: refactor this into several utils -export const filterEmails = (messages: GmailMessage[], blocklist: string[]) => { +export const filterEmails = ( + messageChannelHandle: string, + messages: GmailMessage[], + blocklist: string[], +) => { return filterOutBlocklistedMessages( + messageChannelHandle, filterOutIcsAttachments(filterOutNonPersonalEmails(messages)), blocklist, ); }; const filterOutBlocklistedMessages = ( + messageChannelHandle: string, messages: GmailMessage[], blocklist: string[], ) => { @@ -19,7 +25,12 @@ const filterOutBlocklistedMessages = ( } return message.participants.every( - (participant) => !isEmailBlocklisted(participant.handle, blocklist), + (participant) => + !isEmailBlocklisted( + messageChannelHandle, + participant.handle, + blocklist, + ), ); }); };