6657 Refactor and fix blocklist (#6803)
Closes #6657 - Fix listeners - Refactor jobs to take array of events - Fix calendar events and messages deletion --------- Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Raphaël Bosi
@ -1,21 +1,21 @@
|
||||
import { Logger, Scope } from '@nestjs/common';
|
||||
|
||||
import { Any } from 'typeorm';
|
||||
import { And, Any, ILike, In, Not, Or } from 'typeorm';
|
||||
|
||||
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
|
||||
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
|
||||
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
|
||||
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
|
||||
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
|
||||
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
|
||||
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
|
||||
|
||||
export type BlocklistItemDeleteMessagesJobData = {
|
||||
workspaceId: string;
|
||||
blocklistItemId: string;
|
||||
};
|
||||
export type BlocklistItemDeleteMessagesJobData = WorkspaceEventBatch<
|
||||
ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
|
||||
>;
|
||||
|
||||
@Processor({
|
||||
queueName: MessageQueue.messagingQueue,
|
||||
@ -25,66 +25,135 @@ export class BlocklistItemDeleteMessagesJob {
|
||||
private readonly logger = new Logger(BlocklistItemDeleteMessagesJob.name);
|
||||
|
||||
constructor(
|
||||
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
|
||||
private readonly blocklistRepository: BlocklistRepository,
|
||||
private readonly threadCleanerService: MessagingMessageCleanerService,
|
||||
private readonly twentyORMManager: TwentyORMManager,
|
||||
) {}
|
||||
|
||||
@Process(BlocklistItemDeleteMessagesJob.name)
|
||||
async handle(data: BlocklistItemDeleteMessagesJobData): Promise<void> {
|
||||
const { workspaceId, blocklistItemId } = data;
|
||||
const workspaceId = data.workspaceId;
|
||||
|
||||
const blocklistItem = await this.blocklistRepository.getById(
|
||||
blocklistItemId,
|
||||
workspaceId,
|
||||
const blocklistItemIds = data.events.map(
|
||||
(eventPayload) => eventPayload.recordId,
|
||||
);
|
||||
|
||||
if (!blocklistItem) {
|
||||
this.logger.log(
|
||||
`Blocklist item with id ${blocklistItemId} not found in workspace ${workspaceId}`,
|
||||
const blocklistRepository =
|
||||
await this.twentyORMManager.getRepository<BlocklistWorkspaceEntity>(
|
||||
'blocklist',
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
const blocklist = await blocklistRepository.find({
|
||||
where: {
|
||||
id: Any(blocklistItemIds),
|
||||
},
|
||||
});
|
||||
|
||||
const { handle, workspaceMemberId } = blocklistItem;
|
||||
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
|
||||
(acc, blocklistItem) => {
|
||||
const { handle, workspaceMemberId } = blocklistItem;
|
||||
|
||||
this.logger.log(
|
||||
`Deleting messages from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
|
||||
if (!acc.has(workspaceMemberId)) {
|
||||
acc.set(workspaceMemberId, []);
|
||||
}
|
||||
|
||||
acc.get(workspaceMemberId)?.push(handle);
|
||||
|
||||
return acc;
|
||||
},
|
||||
new Map<string, string[]>(),
|
||||
);
|
||||
|
||||
if (!workspaceMemberId) {
|
||||
throw new Error(
|
||||
`Workspace member ID is not defined for blocklist item ${blocklistItemId} in workspace ${workspaceId}`,
|
||||
const messageChannelRepository =
|
||||
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
|
||||
'messageChannel',
|
||||
);
|
||||
}
|
||||
|
||||
const messageChannelMessageAssociationRepository =
|
||||
await this.twentyORMManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
|
||||
'messageChannelMessageAssociation',
|
||||
);
|
||||
|
||||
const rolesToDelete: ('from' | 'to')[] = ['from', 'to'];
|
||||
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
|
||||
const handles =
|
||||
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
|
||||
|
||||
await messageChannelMessageAssociationRepository.delete({
|
||||
messageChannel: {
|
||||
connectedAccount: {
|
||||
accountOwnerId: workspaceMemberId,
|
||||
if (!handles) {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Deleting messages from ${handles.join(
|
||||
', ',
|
||||
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
|
||||
);
|
||||
|
||||
const rolesToDelete: ('from' | 'to')[] = ['from', 'to'];
|
||||
|
||||
const messageChannels = await messageChannelRepository.find({
|
||||
select: {
|
||||
id: true,
|
||||
handle: true,
|
||||
connectedAccount: {
|
||||
handleAliases: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
message: {
|
||||
messageParticipants: {
|
||||
handle,
|
||||
role: Any(rolesToDelete),
|
||||
where: {
|
||||
connectedAccount: {
|
||||
accountOwnerId: workspaceMemberId,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
relations: ['connectedAccount'],
|
||||
});
|
||||
|
||||
for (const messageChannel of messageChannels) {
|
||||
const messageChannelHandles = [messageChannel.handle];
|
||||
|
||||
if (messageChannel.connectedAccount.handleAliases) {
|
||||
messageChannelHandles.push(
|
||||
...messageChannel.connectedAccount.handleAliases.split(','),
|
||||
);
|
||||
}
|
||||
|
||||
const handleConditions = handles.map((handle) => {
|
||||
const isHandleDomain = handle.startsWith('@');
|
||||
|
||||
return isHandleDomain
|
||||
? {
|
||||
handle: And(
|
||||
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
|
||||
Not(In(messageChannelHandles)),
|
||||
),
|
||||
role: In(rolesToDelete),
|
||||
}
|
||||
: { handle, role: In(rolesToDelete) };
|
||||
});
|
||||
|
||||
const messageChannelMessageAssociationsToDelete =
|
||||
await messageChannelMessageAssociationRepository.find({
|
||||
where: {
|
||||
messageChannelId: messageChannel.id,
|
||||
message: {
|
||||
messageParticipants: handleConditions,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (messageChannelMessageAssociationsToDelete.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await messageChannelMessageAssociationRepository.delete(
|
||||
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Deleted messages from handle ${handles.join(
|
||||
', ',
|
||||
)} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
|
||||
);
|
||||
}
|
||||
|
||||
await this.threadCleanerService.cleanWorkspaceThreads(workspaceId);
|
||||
|
||||
this.logger.log(
|
||||
`Deleted messages from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,63 @@
|
||||
import { Scope } from '@nestjs/common';
|
||||
|
||||
import { Not } from 'typeorm';
|
||||
|
||||
import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
|
||||
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
|
||||
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
|
||||
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
|
||||
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
|
||||
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
|
||||
import {
|
||||
MessageChannelSyncStage,
|
||||
MessageChannelWorkspaceEntity,
|
||||
} from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
|
||||
export type BlocklistReimportMessagesJobData = WorkspaceEventBatch<
|
||||
ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
|
||||
>;
|
||||
|
||||
@Processor({
|
||||
queueName: MessageQueue.messagingQueue,
|
||||
scope: Scope.REQUEST,
|
||||
})
|
||||
export class BlocklistReimportMessagesJob {
|
||||
constructor(
|
||||
private readonly twentyORMManager: TwentyORMManager,
|
||||
private readonly messagingChannelSyncStatusService: MessageChannelSyncStatusService,
|
||||
) {}
|
||||
|
||||
@Process(BlocklistReimportMessagesJob.name)
|
||||
async handle(data: BlocklistReimportMessagesJobData): Promise<void> {
|
||||
const workspaceId = data.workspaceId;
|
||||
|
||||
const messageChannelRepository =
|
||||
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
|
||||
'messageChannel',
|
||||
);
|
||||
|
||||
for (const eventPayload of data.events) {
|
||||
const workspaceMemberId =
|
||||
eventPayload.properties.before.workspaceMemberId;
|
||||
|
||||
const messageChannels = await messageChannelRepository.find({
|
||||
select: ['id'],
|
||||
where: {
|
||||
connectedAccount: {
|
||||
accountOwnerId: workspaceMemberId,
|
||||
},
|
||||
syncStage: Not(
|
||||
MessageChannelSyncStage.FULL_MESSAGE_LIST_FETCH_PENDING,
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
|
||||
messageChannels.map((messageChannel) => messageChannel.id),
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Injectable, Scope } from '@nestjs/common';
|
||||
import { OnEvent } from '@nestjs/event-emitter';
|
||||
|
||||
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
|
||||
@ -7,28 +7,22 @@ import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/t
|
||||
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
|
||||
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
|
||||
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
|
||||
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
|
||||
import {
|
||||
BlocklistItemDeleteMessagesJob,
|
||||
BlocklistItemDeleteMessagesJobData,
|
||||
} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
|
||||
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
|
||||
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
|
||||
import {
|
||||
BlocklistReimportMessagesJob,
|
||||
BlocklistReimportMessagesJobData,
|
||||
} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job';
|
||||
|
||||
@Injectable()
|
||||
@Injectable({ scope: Scope.REQUEST })
|
||||
export class MessagingBlocklistListener {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.messagingQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity)
|
||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||
private readonly messagingChannelSyncStatusService: MessageChannelSyncStatusService,
|
||||
private readonly twentyORMManager: TwentyORMManager,
|
||||
) {}
|
||||
|
||||
@OnEvent('blocklist.created')
|
||||
@ -37,17 +31,9 @@ export class MessagingBlocklistListener {
|
||||
ObjectRecordCreateEvent<BlocklistWorkspaceEntity>
|
||||
>,
|
||||
) {
|
||||
await Promise.all(
|
||||
payload.events.map((eventPayload) =>
|
||||
// TODO: modify to pass an array of blocklist items
|
||||
this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
|
||||
BlocklistItemDeleteMessagesJob.name,
|
||||
{
|
||||
workspaceId: payload.workspaceId,
|
||||
blocklistItemId: eventPayload.recordId,
|
||||
},
|
||||
),
|
||||
),
|
||||
await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
|
||||
BlocklistItemDeleteMessagesJob.name,
|
||||
payload,
|
||||
);
|
||||
}
|
||||
|
||||
@ -57,38 +43,10 @@ export class MessagingBlocklistListener {
|
||||
ObjectRecordDeleteEvent<BlocklistWorkspaceEntity>
|
||||
>,
|
||||
) {
|
||||
const workspaceId = payload.workspaceId;
|
||||
|
||||
for (const eventPayload of payload.events) {
|
||||
const workspaceMemberId =
|
||||
eventPayload.properties.before.workspaceMember.id;
|
||||
|
||||
const connectedAccount =
|
||||
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
|
||||
workspaceMemberId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount || connectedAccount.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const messageChannelRepository =
|
||||
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
|
||||
'messageChannel',
|
||||
);
|
||||
|
||||
const messageChannel = await messageChannelRepository.findOneOrFail({
|
||||
where: {
|
||||
connectedAccountId: connectedAccount[0].id,
|
||||
},
|
||||
});
|
||||
|
||||
await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
|
||||
BlocklistReimportMessagesJob.name,
|
||||
payload,
|
||||
);
|
||||
}
|
||||
|
||||
@OnEvent('blocklist.updated')
|
||||
@ -97,45 +55,14 @@ export class MessagingBlocklistListener {
|
||||
ObjectRecordUpdateEvent<BlocklistWorkspaceEntity>
|
||||
>,
|
||||
) {
|
||||
const workspaceId = payload.workspaceId;
|
||||
await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
|
||||
BlocklistItemDeleteMessagesJob.name,
|
||||
payload,
|
||||
);
|
||||
|
||||
for (const eventPayload of payload.events) {
|
||||
const workspaceMemberId =
|
||||
eventPayload.properties.before.workspaceMember.id;
|
||||
|
||||
await this.messageQueueService.add<BlocklistItemDeleteMessagesJobData>(
|
||||
BlocklistItemDeleteMessagesJob.name,
|
||||
{
|
||||
workspaceId,
|
||||
blocklistItemId: eventPayload.recordId,
|
||||
},
|
||||
);
|
||||
|
||||
const connectedAccount =
|
||||
await this.connectedAccountRepository.getAllByWorkspaceMemberId(
|
||||
workspaceMemberId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
if (!connectedAccount || connectedAccount.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const messageChannelRepository =
|
||||
await this.twentyORMManager.getRepository<MessageChannelWorkspaceEntity>(
|
||||
'messageChannel',
|
||||
);
|
||||
|
||||
const messageChannel = await messageChannelRepository.findOneOrFail({
|
||||
where: {
|
||||
connectedAccountId: connectedAccount[0].id,
|
||||
},
|
||||
});
|
||||
|
||||
await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch(
|
||||
messageChannel.id,
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
await this.messageQueueService.add<BlocklistReimportMessagesJobData>(
|
||||
BlocklistReimportMessagesJob.name,
|
||||
payload,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job';
|
||||
import { BlocklistReimportMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job';
|
||||
import { MessagingBlocklistListener } from 'src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener';
|
||||
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
|
||||
import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module';
|
||||
@ -9,10 +10,8 @@ import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cle
|
||||
imports: [MessagingCommonModule, MessagingMessageCleanerModule],
|
||||
providers: [
|
||||
MessagingBlocklistListener,
|
||||
{
|
||||
provide: BlocklistItemDeleteMessagesJob.name,
|
||||
useClass: BlocklistItemDeleteMessagesJob,
|
||||
},
|
||||
BlocklistItemDeleteMessagesJob,
|
||||
BlocklistReimportMessagesJob,
|
||||
],
|
||||
exports: [],
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user