admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

5901 refactor email and calendar auto contact creation to create them by batch (#6038)

Browse Source 
Closes #5901
This commit is contained in:
bosiraphael
2024-06-27 16:37:34 +02:00
committed by GitHub
parent d1bb0fb822
commit 4f9527c860
10 changed files with 86 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module.ts
View File

@ -11,6 +11,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
import { AutoCompaniesAndContactsCreationMessageChannelListener } from 'src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener';
@Module({
imports: [
@ -25,7 +26,10 @@ import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-co
CalendarEventParticipantModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
],
providers: [CreateCompanyAndContactService],
providers: [
CreateCompanyAndContactService,
AutoCompaniesAndContactsCreationMessageChannelListener,
],
exports: [CreateCompanyAndContactService],
})
export class AutoCompaniesAndContactsCreationModule {}

1
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant.ts Normal file
View File

@ -0,0 +1 @@
export const CONTACTS_CREATION_BATCH_SIZE = 100;

4
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/messaging-message-channel.listener.ts → packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/listeners/auto-companies-and-contacts-creation-message-channel.listener.ts
View File

@ -13,9 +13,9 @@ import {
} from 'src/modules/messaging/message-participants-manager/jobs/messaging-create-company-and-contact-after-sync.job';
@Injectable()
export class MessagingMessageChannelListener {
export class AutoCompaniesAndContactsCreationMessageChannelListener {
constructor(
@InjectMessageQueue(MessageQueue.messagingQueue)
@InjectMessageQueue(MessageQueue.contactCreationQueue)
private readonly messageQueueService: MessageQueueService,
) {}

89
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/services/create-company-and-contact.service.ts
View File

@ -3,6 +3,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
import { EntityManager } from 'typeorm';
import compact from 'lodash.compact';
import chunk from 'lodash.chunk';
import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
import { CreateCompanyService } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company/create-company.service';
@ -14,7 +15,6 @@ import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repos
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { CalendarEventParticipantService } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.service';
import { filterOutContactsFromCompanyOrWorkspace } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util';
import { MessagingMessageParticipantService } from 'src/modules/messaging/common/services/messaging-message-participant.service';
@ -23,6 +23,8 @@ import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/st
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/connected-account/auto-companies-and-contacts-creation/constants/contacts-creation-batch-size.constant';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
@Injectable()
export class CreateCompanyAndContactService {
@ -42,7 +44,7 @@ export class CreateCompanyAndContactService {
async createCompaniesAndPeople(
connectedAccountHandle: string,
contactsToCreate: Contacts,
contactsToCreate: Contact[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<PersonWorkspaceEntity[]> {
@ -132,48 +134,55 @@ export class CreateCompanyAndContactService {
async createCompaniesAndContactsAndUpdateParticipants(
connectedAccount: ConnectedAccountWorkspaceEntity,
contactsToCreate: Contacts,
contactsToCreate: Contact[],
workspaceId: string,
) {
let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] =
[];
await this.workspaceDataSource?.transaction(
async (transactionManager: EntityManager) => {
const createdPeople = await this.createCompaniesAndPeople(
connectedAccount.handle,
contactsToCreate,
workspaceId,
transactionManager,
);
updatedMessageParticipants =
await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
updatedCalendarEventParticipants =
await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
},
const contactsBatches = chunk(
contactsToCreate,
CONTACTS_CREATION_BATCH_SIZE,
);
this.eventEmitter.emit(`messageParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
messageParticipants: updatedMessageParticipants,
});
for (const contactsBatch of contactsBatches) {
let updatedMessageParticipants: MessageParticipantWorkspaceEntity[] = [];
let updatedCalendarEventParticipants: CalendarEventParticipantWorkspaceEntity[] =
[];
this.eventEmitter.emit(`calendarEventParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
calendarEventParticipants: updatedCalendarEventParticipants,
});
await this.workspaceDataSource?.transaction(
async (transactionManager: EntityManager) => {
const createdPeople = await this.createCompaniesAndPeople(
connectedAccount.handle,
contactsBatch,
workspaceId,
transactionManager,
);
updatedMessageParticipants =
await this.messageParticipantService.updateMessageParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
updatedCalendarEventParticipants =
await this.calendarEventParticipantService.updateCalendarEventParticipantsAfterPeopleCreation(
createdPeople,
workspaceId,
transactionManager,
);
},
);
this.eventEmitter.emit(`messageParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
messageParticipants: updatedMessageParticipants,
});
this.eventEmitter.emit(`calendarEventParticipant.matched`, {
workspaceId,
workspaceMemberId: connectedAccount.accountOwnerId,
calendarEventParticipants: updatedCalendarEventParticipants,
});
}
}
}

2
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type.ts
View File

@ -2,5 +2,3 @@ export type Contact = {
handle: string;
displayName: string;
};
export type Contacts = Contact[];

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/__tests__/get-unique-contacts-and-handles.spec.ts
View File

@ -1,9 +1,9 @@
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { getUniqueContactsAndHandles } from 'src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util';
describe('getUniqueContactsAndHandles', () => {
it('should return empty arrays when contacts is empty', () => {
const contacts: Contacts = [];
const contacts: Contact[] = [];
const result = getUniqueContactsAndHandles(contacts);
expect(result.uniqueContacts).toEqual([]);
@ -11,7 +11,7 @@ describe('getUniqueContactsAndHandles', () => {
});
it('should return unique contacts and handles', () => {
const contacts: Contacts = [
const contacts: Contact[] = [
{ handle: 'john@twenty.com', displayName: 'John Doe' },
{ handle: 'john@twenty.com', displayName: 'John Doe' },
{ handle: 'jane@twenty.com', displayName: 'Jane Smith' },

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/filter-out-contacts-from-company-or-workspace.util.ts
View File

@ -1,12 +1,12 @@
import { getDomainNameFromHandle } from 'src/modules/calendar-messaging-participant/utils/get-domain-name-from-handle.util';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
export function filterOutContactsFromCompanyOrWorkspace(
contacts: Contacts,
contacts: Contact[],
selfHandle: string,
workspaceMembers: WorkspaceMemberWorkspaceEntity[],
): Contacts {
): Contact[] {
const selfDomainName = getDomainNameFromHandle(selfHandle);
const workspaceMembersMap = workspaceMembers.reduce(

6
packages/twenty-server/src/modules/connected-account/auto-companies-and-contacts-creation/utils/get-unique-contacts-and-handles.util.ts
View File

@ -1,10 +1,10 @@
import uniq from 'lodash.uniq';
import uniqBy from 'lodash.uniqby';
import { Contacts } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
import { Contact } from 'src/modules/connected-account/auto-companies-and-contacts-creation/types/contact.type';
export function getUniqueContactsAndHandles(contacts: Contacts): {
uniqueContacts: Contacts;
export function getUniqueContactsAndHandles(contacts: Contact[]): {
uniqueContacts: Contact[];
uniqueHandles: string[];
} {
if (contacts.length === 0) {