Refactor connected account module (#6225)

- Refactor connected account module
- Move blocklist into it's own module
- Move contact-creation-manager into it's own module

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
bosiraphael
2024-07-12 20:15:33 +02:00
committed by GitHub
parent c8a889995f
commit 11da718482
53 changed files with 212 additions and 192 deletions

View File

@ -0,0 +1 @@
export const CONTACTS_CREATION_BATCH_SIZE = 100;

View File

@ -0,0 +1,37 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CompanyWorkspaceEntity } from 'src/modules/company/standard-objects/company.workspace-entity';
import { AutoCompaniesAndContactsCreationCalendarChannelListener } from 'src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-calendar-channel.listener';
import { AutoCompaniesAndContactsCreationMessageChannelListener } from 'src/modules/contact-creation-manager/listeners/auto-companies-and-contacts-creation-message-channel.listener';
import { CreateCompanyAndContactService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
import { CreateCompanyService } from 'src/modules/contact-creation-manager/services/create-company.service';
import { CreateContactService } from 'src/modules/contact-creation-manager/services/create-contact.service';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@Module({
imports: [
ObjectMetadataRepositoryModule.forFeature([
PersonWorkspaceEntity,
WorkspaceMemberWorkspaceEntity,
CompanyWorkspaceEntity,
]),
WorkspaceDataSourceModule,
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
],
providers: [
CreateCompanyService,
CreateContactService,
CreateCompanyAndContactService,
AutoCompaniesAndContactsCreationMessageChannelListener,
AutoCompaniesAndContactsCreationCalendarChannelListener,
],
exports: [CreateCompanyAndContactService],
})
export class ContactCreationManagerModule {}

View File

@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { ContactCreationManagerModule } from 'src/modules/contact-creation-manager/contact-creation-manager.module';
import { CreateCompanyAndContactJob } from 'src/modules/contact-creation-manager/jobs/create-company-and-contact.job';
@Module({
imports: [ContactCreationManagerModule],
providers: [CreateCompanyAndContactJob],
})
export class AutoCompaniesAndContactsCreationJobModule {}

View File

@ -0,0 +1,35 @@
import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CreateCompanyAndContactService } from 'src/modules/contact-creation-manager/services/create-company-and-contact.service';
export type CreateCompanyAndContactJobData = {
workspaceId: string;
connectedAccount: ConnectedAccountWorkspaceEntity;
contactsToCreate: {
displayName: string;
handle: string;
}[];
};
@Processor(MessageQueue.contactCreationQueue)
export class CreateCompanyAndContactJob {
constructor(
private readonly createCompanyAndContactService: CreateCompanyAndContactService,
) {}
@Process(CreateCompanyAndContactJob.name)
async handle(data: CreateCompanyAndContactJobData): Promise<void> {
const { workspaceId, connectedAccount, contactsToCreate } = data;
await this.createCompanyAndContactService.createCompaniesAndContactsAndUpdateParticipants(
connectedAccount,
contactsToCreate.map((contact) => ({
handle: contact.handle,
displayName: contact.displayName,
})),
workspaceId,
);
}
}

View File

@ -0,0 +1,42 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event';
import { objectRecordChangedProperties } from 'src/engine/integrations/event-emitter/utils/object-record-changed-properties.util';
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import {
CalendarCreateCompanyAndContactAfterSyncJobData,
CalendarCreateCompanyAndContactAfterSyncJob,
} from 'src/modules/calendar/calendar-event-participant-manager/jobs/calendar-create-company-and-contact-after-sync.job';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
@Injectable()
export class AutoCompaniesAndContactsCreationCalendarChannelListener {
constructor(
@InjectMessageQueue(MessageQueue.calendarQueue)
private readonly messageQueueService: MessageQueueService,
) {}
@OnEvent('calendarChannel.updated')
async handleUpdatedEvent(
payload: ObjectRecordUpdateEvent<MessageChannelWorkspaceEntity>,
) {
if (
objectRecordChangedProperties(
payload.properties.before,
payload.properties.after,
).includes('isContactAutoCreationEnabled') &&
payload.properties.after.isContactAutoCreationEnabled
) {
await this.messageQueueService.add<CalendarCreateCompanyAndContactAfterSyncJobData>(
CalendarCreateCompanyAndContactAfterSyncJob.name,
{
workspaceId: payload.workspaceId,
calendarChannelId: payload.recordId,
},
);
}
}
}

View File

@ -0,0 +1,42 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event';
import { objectRecordChangedProperties } from 'src/engine/integrations/event-emitter/utils/object-record-changed-properties.util';
import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
import {
MessagingCreateCompanyAndContactAfterSyncJobData,
MessagingCreateCompanyAndContactAfterSyncJob,
} from 'src/modules/messaging/message-participant-manager/jobs/messaging-create-company-and-contact-after-sync.job';
@Injectable()
export class AutoCompaniesAndContactsCreationMessageChannelListener {
constructor(
@InjectMessageQueue(MessageQueue.contactCreationQueue)
private readonly messageQueueService: MessageQueueService,
) {}
@OnEvent('messageChannel.updated')
async handleUpdatedEvent(
payload: ObjectRecordUpdateEvent<MessageChannelWorkspaceEntity>,
) {
if (
objectRecordChangedProperties(
payload.properties.before,
payload.properties.after,
).includes('isContactAutoCreationEnabled') &&
payload.properties.after.isContactAutoCreationEnabled
) {
await this.messageQueueService.add<MessagingCreateCompanyAndContactAfterSyncJobData>(
MessagingCreateCompanyAndContactAfterSyncJob.name,
{
workspaceId: payload.workspaceId,
messageChannelId: payload.recordId,
},
);
}
}
}

View File

@ -0,0 +1,169 @@
import { Injectable } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { InjectRepository } from '@nestjs/typeorm';
import chunk from 'lodash.chunk';
import compact from 'lodash.compact';
import { EntityManager, Repository } from 'typeorm';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { CONTACTS_CREATION_BATCH_SIZE } from 'src/modules/contact-creation-manager/constants/contacts-creation-batch-size.constant';
import { CreateCompanyService } from 'src/modules/contact-creation-manager/services/create-company.service';
import { CreateContactService } from 'src/modules/contact-creation-manager/services/create-contact.service';
import { Contact } from 'src/modules/contact-creation-manager/types/contact.type';
import { filterOutSelfAndContactsFromCompanyOrWorkspace } from 'src/modules/contact-creation-manager/utils/filter-out-contacts-from-company-or-workspace.util';
import { getDomainNameFromHandle } from 'src/modules/contact-creation-manager/utils/get-domain-name-from-handle.util';
import { getUniqueContactsAndHandles } from 'src/modules/contact-creation-manager/utils/get-unique-contacts-and-handles.util';
import { PersonRepository } from 'src/modules/person/repositories/person.repository';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { isWorkEmail } from 'src/utils/is-work-email';
@Injectable()
export class CreateCompanyAndContactService {
constructor(
private readonly createContactService: CreateContactService,
private readonly createCompaniesService: CreateCompanyService,
@InjectObjectMetadataRepository(PersonWorkspaceEntity)
private readonly personRepository: PersonRepository,
@InjectObjectMetadataRepository(WorkspaceMemberWorkspaceEntity)
private readonly workspaceMemberRepository: WorkspaceMemberRepository,
private readonly eventEmitter: EventEmitter2,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
) {}
private async createCompaniesAndPeople(
connectedAccount: ConnectedAccountWorkspaceEntity,
contactsToCreate: Contact[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<PersonWorkspaceEntity[]> {
if (!contactsToCreate || contactsToCreate.length === 0) {
return [];
}
const workspaceMembers =
await this.workspaceMemberRepository.getAllByWorkspaceId(
workspaceId,
transactionManager,
);
const contactsToCreateFromOtherCompanies =
filterOutSelfAndContactsFromCompanyOrWorkspace(
contactsToCreate,
connectedAccount,
workspaceMembers,
);
const { uniqueContacts, uniqueHandles } = getUniqueContactsAndHandles(
contactsToCreateFromOtherCompanies,
);
if (uniqueHandles.length === 0) {
return [];
}
const alreadyCreatedContacts = await this.personRepository.getByEmails(
uniqueHandles,
workspaceId,
transactionManager,
);
const alreadyCreatedContactEmails: string[] = alreadyCreatedContacts?.map(
({ email }) => email,
);
const filteredContactsToCreate = uniqueContacts.filter(
(participant) =>
!alreadyCreatedContactEmails.includes(participant.handle) &&
participant.handle.includes('@'),
);
const filteredContactsToCreateWithCompanyDomainNames =
filteredContactsToCreate?.map((participant) => ({
handle: participant.handle,
displayName: participant.displayName,
companyDomainName: isWorkEmail(participant.handle)
? getDomainNameFromHandle(participant.handle)
: undefined,
}));
const domainNamesToCreate = compact(
filteredContactsToCreateWithCompanyDomainNames.map(
(participant) => participant.companyDomainName,
),
);
const companiesObject = await this.createCompaniesService.createCompanies(
domainNamesToCreate,
workspaceId,
transactionManager,
);
const formattedContactsToCreate =
filteredContactsToCreateWithCompanyDomainNames.map((contact) => ({
handle: contact.handle,
displayName: contact.displayName,
companyId:
contact.companyDomainName && contact.companyDomainName !== ''
? companiesObject[contact.companyDomainName]
: undefined,
}));
return await this.createContactService.createPeople(
formattedContactsToCreate,
workspaceId,
transactionManager,
);
}
async createCompaniesAndContactsAndUpdateParticipants(
connectedAccount: ConnectedAccountWorkspaceEntity,
contactsToCreate: Contact[],
workspaceId: string,
) {
const contactsBatches = chunk(
contactsToCreate,
CONTACTS_CREATION_BATCH_SIZE,
);
// TODO: Remove this when events are emitted directly inside TwentyORM
const objectMetadata = await this.objectMetadataRepository.findOne({
where: {
standardId: STANDARD_OBJECT_IDS.person,
workspaceId,
},
});
if (!objectMetadata) {
throw new Error('Object metadata not found');
}
for (const contactsBatch of contactsBatches) {
const createdPeople = await this.createCompaniesAndPeople(
connectedAccount,
contactsBatch,
workspaceId,
);
for (const createdPerson of createdPeople) {
this.eventEmitter.emit('person.created', {
name: 'person.created',
workspaceId,
recordId: createdPerson.id,
objectMetadata,
properties: {
after: createdPerson,
},
} satisfies ObjectRecordCreateEvent<any>);
}
}
}
}

View File

@ -0,0 +1,122 @@
import { Injectable } from '@nestjs/common';
import axios, { AxiosInstance } from 'axios';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { CompanyRepository } from 'src/modules/company/repositories/company.repository';
import { CompanyWorkspaceEntity } from 'src/modules/company/standard-objects/company.workspace-entity';
import { getCompanyNameFromDomainName } from 'src/modules/contact-creation-manager/utils/get-company-name-from-domain-name.util';
@Injectable()
export class CreateCompanyService {
private readonly httpService: AxiosInstance;
constructor(
@InjectObjectMetadataRepository(CompanyWorkspaceEntity)
private readonly companyRepository: CompanyRepository,
) {
this.httpService = axios.create({
baseURL: 'https://companies.twenty.com',
});
}
async createCompanies(
domainNames: string[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<{
[domainName: string]: string;
}> {
if (domainNames.length === 0) {
return {};
}
const uniqueDomainNames = [...new Set(domainNames)];
const existingCompanies =
await this.companyRepository.getExistingCompaniesByDomainNames(
uniqueDomainNames,
workspaceId,
transactionManager,
);
const companiesObject = existingCompanies.reduce(
(
acc: {
[domainName: string]: string;
},
company: {
domainName: string;
id: string;
},
) => ({
...acc,
[company.domainName]: company.id,
}),
{},
);
const filteredDomainNames = uniqueDomainNames.filter(
(domainName) =>
!existingCompanies.some(
(company: { domainName: string }) =>
company.domainName === domainName,
),
);
for (const domainName of filteredDomainNames) {
companiesObject[domainName] = await this.createCompany(
domainName,
workspaceId,
transactionManager,
);
}
return companiesObject;
}
private async createCompany(
domainName: string,
workspaceId: string,
transactionManager?: EntityManager,
): Promise<string> {
const companyId = v4();
const { name, city } = await this.getCompanyInfoFromDomainName(domainName);
await this.companyRepository.createCompany(
workspaceId,
{
id: companyId,
domainName,
name,
city,
},
transactionManager,
);
return companyId;
}
private async getCompanyInfoFromDomainName(domainName: string): Promise<{
name: string;
city: string;
}> {
try {
const response = await this.httpService.get(`/${domainName}`);
const data = response.data;
return {
name: data.name ?? getCompanyNameFromDomainName(domainName),
city: data.city,
};
} catch (e) {
return {
name: getCompanyNameFromDomainName(domainName),
city: '',
};
}
}
}

View File

@ -0,0 +1,68 @@
import { Injectable } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { v4 } from 'uuid';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { getFirstNameAndLastNameFromHandleAndDisplayName } from 'src/modules/contact-creation-manager/utils/get-first-name-and-last-name-from-handle-and-display-name.util';
import { PersonRepository } from 'src/modules/person/repositories/person.repository';
import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity';
type ContactToCreate = {
handle: string;
displayName: string;
companyId?: string;
};
type FormattedContactToCreate = {
id: string;
handle: string;
firstName: string;
lastName: string;
companyId?: string;
};
@Injectable()
export class CreateContactService {
constructor(
@InjectObjectMetadataRepository(PersonWorkspaceEntity)
private readonly personRepository: PersonRepository,
) {}
private formatContacts(
contactsToCreate: ContactToCreate[],
): FormattedContactToCreate[] {
return contactsToCreate.map((contact) => {
const id = v4();
const { handle, displayName, companyId } = contact;
const { firstName, lastName } =
getFirstNameAndLastNameFromHandleAndDisplayName(handle, displayName);
return {
id,
handle,
firstName,
lastName,
companyId,
};
});
}
public async createPeople(
contactsToCreate: ContactToCreate[],
workspaceId: string,
transactionManager?: EntityManager,
): Promise<PersonWorkspaceEntity[]> {
if (contactsToCreate.length === 0) return [];
const formattedContacts = this.formatContacts(contactsToCreate);
return await this.personRepository.createPeople(
formattedContacts,
workspaceId,
transactionManager,
);
}
}

View File

@ -0,0 +1,4 @@
export type Contact = {
handle: string;
displayName: string;
};

View File

@ -0,0 +1,32 @@
import { Contact } from 'src/modules/contact-creation-manager/types/contact.type';
import { getUniqueContactsAndHandles } from 'src/modules/contact-creation-manager/utils/get-unique-contacts-and-handles.util';
describe('getUniqueContactsAndHandles', () => {
it('should return empty arrays when contacts is empty', () => {
const contacts: Contact[] = [];
const result = getUniqueContactsAndHandles(contacts);
expect(result.uniqueContacts).toEqual([]);
expect(result.uniqueHandles).toEqual([]);
});
it('should return unique contacts and handles', () => {
const contacts: Contact[] = [
{ handle: 'john@twenty.com', displayName: 'John Doe' },
{ handle: 'john@twenty.com', displayName: 'John Doe' },
{ handle: 'jane@twenty.com', displayName: 'Jane Smith' },
{ handle: 'jane@twenty.com', displayName: 'Jane Smith' },
{ handle: 'jane@twenty.com', displayName: 'Jane Smith' },
];
const result = getUniqueContactsAndHandles(contacts);
expect(result.uniqueContacts).toEqual([
{ handle: 'john@twenty.com', displayName: 'John Doe' },
{ handle: 'jane@twenty.com', displayName: 'Jane Smith' },
]);
expect(result.uniqueHandles).toEqual([
'john@twenty.com',
'jane@twenty.com',
]);
});
});

View File

@ -0,0 +1,30 @@
import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
import { Contact } from 'src/modules/contact-creation-manager/types/contact.type';
import { getDomainNameFromHandle } from 'src/modules/contact-creation-manager/utils/get-domain-name-from-handle.util';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
export function filterOutSelfAndContactsFromCompanyOrWorkspace(
contacts: Contact[],
connectedAccount: ConnectedAccountWorkspaceEntity,
workspaceMembers: WorkspaceMemberWorkspaceEntity[],
): Contact[] {
const selfDomainName = getDomainNameFromHandle(connectedAccount.handle);
const handleAliases = connectedAccount.handleAliases?.split(',') || [];
const workspaceMembersMap = workspaceMembers.reduce(
(map, workspaceMember) => {
map[workspaceMember.userEmail] = true;
return map;
},
new Map<string, boolean>(),
);
return contacts.filter(
(contact) =>
getDomainNameFromHandle(contact.handle) !== selfDomainName &&
!workspaceMembersMap[contact.handle] &&
!handleAliases.includes(contact.handle),
);
}

View File

@ -0,0 +1,9 @@
import psl from 'psl';
import { capitalize } from 'src/utils/capitalize';
export const getCompanyNameFromDomainName = (domainName: string) => {
const { sld } = psl.parse(domainName);
return sld ? capitalize(sld) : '';
};

View File

@ -0,0 +1,9 @@
import psl from 'psl';
export const getDomainNameFromHandle = (handle: string): string => {
const wholeDomain = handle?.split('@')?.[1] || '';
const { domain } = psl.parse(wholeDomain);
return domain || '';
};

View File

@ -0,0 +1,18 @@
import { capitalize } from 'src/utils/capitalize';
export function getFirstNameAndLastNameFromHandleAndDisplayName(
handle: string,
displayName: string,
): { firstName: string; lastName: string } {
const firstName = displayName.split(' ')[0];
const lastName = displayName.split(' ')[1];
const contactFullNameFromHandle = handle.split('@')[0];
const firstNameFromHandle = contactFullNameFromHandle.split('.')[0];
const lastNameFromHandle = contactFullNameFromHandle.split('.')[1];
return {
firstName: capitalize(firstName || firstNameFromHandle || ''),
lastName: capitalize(lastName || lastNameFromHandle || ''),
};
}

View File

@ -0,0 +1,19 @@
import uniq from 'lodash.uniq';
import uniqBy from 'lodash.uniqby';
import { Contact } from 'src/modules/contact-creation-manager/types/contact.type';
export function getUniqueContactsAndHandles(contacts: Contact[]): {
uniqueContacts: Contact[];
uniqueHandles: string[];
} {
if (contacts.length === 0) {
return { uniqueContacts: [], uniqueHandles: [] };
}
const uniqueHandles = uniq(contacts.map((participant) => participant.handle));
const uniqueContacts = uniqBy(contacts, 'handle');
return { uniqueContacts, uniqueHandles };
}