Basic data enrichment (#3023)

* Add Enrich to frontend

* Naive backend implementation

* Add work email check

* Rename Enrich to Quick Action

* Refactor logic to a separate service

* Refacto to separate IntelligenceService

* Small fixes

* Missing Break statement

* Address PR comments

* Create company interface

* Improve edge case handling

* Use httpService instead of Axios

* Fix server tests
This commit is contained in:
Félix Malfait
2023-12-18 15:45:30 +01:00
committed by GitHub
parent 576492f3c0
commit fff51a2d91
38 changed files with 16928 additions and 27 deletions

View File

@ -1,9 +1,15 @@
import { Module } from '@nestjs/common';
import { HttpModule } from '@nestjs/axios';
import { AnalyticsService } from './analytics.service';
import { AnalyticsResolver } from './analytics.resolver';
@Module({
providers: [AnalyticsResolver, AnalyticsService],
imports: [
HttpModule.register({
baseURL: 'https://t.twenty.com/api/v1/s2s',
}),
],
})
export class AnalyticsModule {}

View File

@ -1,4 +1,5 @@
import { Test, TestingModule } from '@nestjs/testing';
import { HttpService } from '@nestjs/axios';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@ -17,6 +18,10 @@ describe('AnalyticsResolver', () => {
provide: EnvironmentService,
useValue: {},
},
{
provide: HttpService,
useValue: {},
},
],
}).compile();

View File

@ -1,4 +1,5 @@
import { Test, TestingModule } from '@nestjs/testing';
import { HttpService } from '@nestjs/axios';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@ -15,6 +16,10 @@ describe('AnalyticsService', () => {
provide: EnvironmentService,
useValue: {},
},
{
provide: HttpService,
useValue: {},
},
],
}).compile();

View File

@ -1,6 +1,5 @@
import { Injectable } from '@nestjs/common';
import axios, { AxiosInstance } from 'axios';
import { HttpService } from '@nestjs/axios';
import { anonymize } from 'src/utils/anonymize';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@ -11,13 +10,10 @@ import { CreateAnalyticsInput } from './dto/create-analytics.input';
@Injectable()
export class AnalyticsService {
private readonly httpService: AxiosInstance;
constructor(private readonly environmentService: EnvironmentService) {
this.httpService = axios.create({
baseURL: 'https://t.twenty.com/api/v1/s2s',
});
}
constructor(
private readonly environmentService: EnvironmentService,
private readonly httpService: HttpService,
) {}
async create(
createEventInput: CreateAnalyticsInput,

View File

@ -1,4 +1,5 @@
import { Module } from '@nestjs/common';
import { HttpModule } from '@nestjs/axios';
import { ApiRestController } from 'src/core/api-rest/api-rest.controller';
import { ApiRestService } from 'src/core/api-rest/api-rest.service';
@ -6,7 +7,7 @@ import { ApiRestQueryBuilderModule } from 'src/core/api-rest/api-rest-query-buil
import { AuthModule } from 'src/core/auth/auth.module';
@Module({
imports: [ApiRestQueryBuilderModule, AuthModule],
imports: [ApiRestQueryBuilderModule, AuthModule, HttpModule],
controllers: [ApiRestController],
providers: [ApiRestService],
})

View File

@ -1,4 +1,5 @@
import { Test, TestingModule } from '@nestjs/testing';
import { HttpService } from '@nestjs/axios';
import { ApiRestService } from 'src/core/api-rest/api-rest.service';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@ -24,6 +25,10 @@ describe('ApiRestService', () => {
provide: TokenService,
useValue: {},
},
{
provide: HttpService,
useValue: {},
},
],
}).compile();

View File

@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { HttpService } from '@nestjs/axios';
import axios from 'axios';
import { Request } from 'express';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@ -15,6 +15,7 @@ export class ApiRestService {
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
private readonly apiRestQueryBuilderFactory: ApiRestQueryBuilderFactory,
private readonly httpService: HttpService,
) {}
async callGraphql(
@ -26,7 +27,7 @@ export class ApiRestService {
`${request.protocol}://${request.get('host')}`;
try {
return await axios.post(`${baseUrl}/graphql`, data, {
return await this.httpService.axiosRef.post(`${baseUrl}/graphql`, data, {
headers: {
'Content-Type': 'application/json',
Authorization: request.headers.authorization,

View File

@ -2,6 +2,7 @@
import { Module } from '@nestjs/common';
import { JwtModule } from '@nestjs/jwt';
import { TypeOrmModule } from '@nestjs/typeorm';
import { HttpModule } from '@nestjs/axios';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { FileModule } from 'src/core/file/file.module';
@ -43,6 +44,7 @@ const jwtModule = JwtModule.registerAsync({
WorkspaceManagerModule,
TypeORMModule,
TypeOrmModule.forFeature([Workspace, User, RefreshToken], 'core'),
HttpModule,
],
controllers: [
GoogleAuthController,

View File

@ -1,5 +1,6 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { HttpService } from '@nestjs/axios';
import { UserService } from 'src/core/user/services/user.service';
import { WorkspaceManagerService } from 'src/workspace/workspace-manager/workspace-manager.service';
@ -33,6 +34,10 @@ describe('AuthService', () => {
provide: FileUploadService,
useValue: {},
},
{
provide: HttpService,
useValue: {},
},
{
provide: getRepositoryToken(Workspace, 'core'),
useValue: {},

View File

@ -5,6 +5,7 @@ import {
NotFoundException,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { HttpService } from '@nestjs/axios';
import FileType from 'file-type';
import { Repository } from 'typeorm';
@ -48,6 +49,7 @@ export class AuthService {
private readonly workspaceRepository: Repository<Workspace>,
@InjectRepository(User, 'core')
private readonly userRepository: Repository<User>,
private readonly httpService: HttpService,
) {}
async challenge(challengeInput: ChallengeInput) {
@ -135,7 +137,10 @@ export class AuthService {
let imagePath: string | undefined = undefined;
if (picture) {
const buffer = await getImageBufferFromUrl(picture);
const buffer = await getImageBufferFromUrl(
picture,
this.httpService.axiosRef,
);
const type = await FileType.fromBuffer(buffer);

View File

@ -0,0 +1,52 @@
import { HttpService } from '@nestjs/axios';
import { Injectable } from '@nestjs/common';
import { CompanyInteface } from 'src/core/quick-actions/interfaces/company.interface';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
@Injectable()
export class IntelligenceService {
constructor(
private readonly environmentService: EnvironmentService,
private readonly httpService: HttpService,
) {}
async enrichCompany(domainName: string): Promise<CompanyInteface> {
const enrichedCompany = await this.httpService.axiosRef.get(
`https://companies.twenty.com/${domainName}`,
{
validateStatus: function () {
// This ensures the promise is always resolved, preventing axios from throwing an error
return true;
},
},
);
if (enrichedCompany.status !== 200) {
return {};
}
return {
linkedinLinkUrl: `https://linkedin.com/` + enrichedCompany.data.handle,
};
}
async completeWithAi(content: string) {
return this.httpService.axiosRef.post(
'https://openrouter.ai/api/v1/chat/completions',
{
headers: {
Authorization: `Bearer ${this.environmentService.getOpenRouterApiKey()}`,
'HTTP-Referer': `https://twenty.com`,
'X-Title': `Twenty CRM`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: 'mistralai/mixtral-8x7b-instruct',
messages: [{ role: 'user', content: content }],
}),
},
);
}
}

View File

@ -0,0 +1,3 @@
export interface CompanyInteface {
linkedinLinkUrl?: string;
}

View File

@ -0,0 +1,14 @@
import { Module } from '@nestjs/common';
import { HttpModule } from '@nestjs/axios';
import { IntelligenceService } from 'src/core/quick-actions/intelligence.service';
import { QuickActionsService } from 'src/core/quick-actions/quick-actions.service';
import { WorkspaceQueryRunnerModule } from 'src/workspace/workspace-query-runner/workspace-query-runner.module';
@Module({
imports: [WorkspaceQueryRunnerModule, HttpModule],
controllers: [],
providers: [QuickActionsService, IntelligenceService],
exports: [QuickActionsService, IntelligenceService],
})
export class QuickActionsModule {}

View File

@ -0,0 +1,154 @@
import { Injectable } from '@nestjs/common';
import { v4 as uuidv4 } from 'uuid';
import { Record as IRecord } from 'src/workspace/workspace-query-builder/interfaces/record.interface';
import { isWorkEmail } from 'src/utils/is-work-email';
import { stringifyWithoutKeyQuote } from 'src/workspace/workspace-query-builder/utils/stringify-without-key-quote.util';
import { WorkspaceQueryRunnerService } from 'src/workspace/workspace-query-runner/workspace-query-runner.service';
import { IntelligenceService } from 'src/core/quick-actions/intelligence.service';
import { capitalize } from 'src/utils/capitalize';
@Injectable()
export class QuickActionsService {
constructor(
private readonly workspaceQueryRunnunerService: WorkspaceQueryRunnerService,
private readonly intelligenceService: IntelligenceService,
) {}
async createCompanyFromPerson(id: string, workspaceId: string) {
const personRequest =
await this.workspaceQueryRunnunerService.executeAndParse<IRecord>(
`query {
personCollection(filter: {id: {eq: "${id}"}}) {
edges {
node {
id
email
companyId
}
}
}
}
`,
'person',
'',
workspaceId,
);
const person = personRequest.edges?.[0]?.node;
if (!person) {
return;
}
if (!person.companyId && person.email && isWorkEmail(person.email)) {
const companyDomainName = person.email.split('@')?.[1].toLowerCase();
const companyName = capitalize(companyDomainName.split('.')[0]);
let relatedCompanyId = uuidv4();
const existingCompany =
await this.workspaceQueryRunnunerService.executeAndParse<IRecord>(
`query {companyCollection(filter: {domainName: {eq: "${companyDomainName}"}}) {
edges {
node {
id
}
}
}
}
`,
'company',
'',
workspaceId,
);
if (existingCompany.edges?.length) {
relatedCompanyId = existingCompany.edges[0].node.id;
}
await this.workspaceQueryRunnunerService.execute(
`mutation {
insertIntocompanyCollection(objects: ${stringifyWithoutKeyQuote([
{
id: relatedCompanyId,
name: companyName,
domainName: companyDomainName,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
},
])}) {
affectedCount
records {
id
}
}
}
`,
workspaceId,
);
await this.workspaceQueryRunnunerService.execute(
`mutation {
updatepersonCollection(set: ${stringifyWithoutKeyQuote({
companyId: relatedCompanyId,
})}, filter: { id: { eq: "${person.id}" } }) {
affectedCount
records {
id
}
}
}
`,
workspaceId,
);
}
}
async executeQuickActionOnCompany(id: string, workspaceId: string) {
const companyQuery = `query {
companyCollection(filter: {id: {eq: "${id}"}}) {
edges {
node {
id
domainName
createdAt
linkedinLinkUrl
}
}
}
}
`;
const companyRequest =
await this.workspaceQueryRunnunerService.executeAndParse<IRecord>(
companyQuery,
'company',
'',
workspaceId,
);
const company = companyRequest.edges?.[0]?.node;
if (!company) {
return;
}
const enrichedData = await this.intelligenceService.enrichCompany(
company.domainName,
);
await this.workspaceQueryRunnunerService.execute(
`mutation {
updatecompanyCollection(set: ${stringifyWithoutKeyQuote(
enrichedData,
)}, filter: { id: { eq: "${id}" } }) {
affectedCount
records {
id
}
}
}`,
workspaceId,
);
}
}

View File

@ -210,10 +210,14 @@ export class EnvironmentService {
}
getSentryDSN(): string | undefined {
return this.configService.get<string>('SENTRY_DSN');
return this.configService.get<string | undefined>('SENTRY_DSN');
}
getDemoWorkspaceIds(): string[] {
return this.configService.get<string[]>('DEMO_WORKSPACE_IDS') ?? [];
}
getOpenRouterApiKey(): string | undefined {
return this.configService.get<string | undefined>('OPENROUTER_API_KEY');
}
}

File diff suppressed because it is too large Load Diff

View File

@ -1,4 +1,4 @@
import axios from 'axios';
import { Axios } from 'axios';
const cropRegex = /([w|h])([0-9]+)/;
@ -22,8 +22,11 @@ export const getCropSize = (value: ShortCropSize): CropSize | null => {
};
};
export const getImageBufferFromUrl = async (url: string): Promise<Buffer> => {
const response = await axios.get(url, {
export const getImageBufferFromUrl = async (
url: string,
axiosInstance: Axios,
): Promise<Buffer> => {
const response = await axiosInstance.get(url, {
responseType: 'arraybuffer',
});

View File

@ -0,0 +1,21 @@
import { emailProvidersSet } from 'src/utils/email-providers';
export const isWorkEmail = (email: string) => {
if (!email) {
return false;
}
const fields = email.split('@');
if (fields.length !== 2) {
return false;
}
const domain = fields[1];
if (!domain) {
return false;
}
return !emailProvidersSet.has(domain);
};

View File

@ -15,6 +15,7 @@ describe('getResolverName', () => {
['createOne', 'createEntity'],
['updateOne', 'updateEntity'],
['deleteOne', 'deleteEntity'],
['executeQuickActionOnOne', 'executeQuickActionOnEntity'],
])('should return correct name for %s resolver', (type, expectedResult) => {
expect(
getResolverName(metadata, type as WorkspaceResolverBuilderMethodNames),

View File

@ -21,6 +21,8 @@ export const getResolverName = (
return `update${pascalCase(objectMetadata.nameSingular)}`;
case 'deleteOne':
return `delete${pascalCase(objectMetadata.nameSingular)}`;
case 'executeQuickActionOnOne':
return `executeQuickActionOn${pascalCase(objectMetadata.nameSingular)}`;
case 'updateMany':
return `update${pascalCase(objectMetadata.namePlural)}`;
case 'deleteMany':

View File

@ -1,6 +1,7 @@
import { Module } from '@nestjs/common';
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { FieldsStringFactory } from 'src/workspace/workspace-query-builder/factories/fields-string.factory';
import { WorkspaceQueryBuilderFactory } from './workspace-query-builder.factory';
@ -9,6 +10,6 @@ import { workspaceQueryBuilderFactories } from './factories/factories';
@Module({
imports: [ObjectMetadataModule],
providers: [...workspaceQueryBuilderFactories, WorkspaceQueryBuilderFactory],
exports: [WorkspaceQueryBuilderFactory],
exports: [WorkspaceQueryBuilderFactory, FieldsStringFactory],
})
export class WorkspaceQueryBuilderModule {}

View File

@ -181,7 +181,7 @@ export class WorkspaceQueryRunnerService {
)?.records;
}
private async execute(
async execute(
query: string,
workspaceId: string,
): Promise<PGGraphQLResult | undefined> {
@ -215,7 +215,7 @@ export class WorkspaceQueryRunnerService {
const errors = graphqlResult?.[0]?.resolve?.errors;
if (Array.isArray(errors) && errors.length > 0) {
console.error('GraphQL errors', errors);
console.error(`GraphQL errors on ${command}${targetTableName}`, errors);
}
if (!result) {
@ -224,4 +224,15 @@ export class WorkspaceQueryRunnerService {
return parseResult(result);
}
async executeAndParse<Result>(
query: string,
targetTableName: string,
command: string,
workspaceId: string,
): Promise<Result> {
const result = await this.execute(query, workspaceId);
return this.parseResult(result, targetTableName, command);
}
}

View File

@ -0,0 +1,72 @@
import { Injectable } from '@nestjs/common';
import {
Resolver,
FindOneResolverArgs,
ExecuteQuickActionOnOneResolverArgs,
DeleteOneResolverArgs,
} from 'src/workspace/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { Record as IRecord } from 'src/workspace/workspace-query-builder/interfaces/record.interface';
import { WorkspaceSchemaBuilderContext } from 'src/workspace/workspace-schema-builder/interfaces/workspace-schema-builder-context.interface';
import { WorkspaceResolverBuilderFactoryInterface } from 'src/workspace/workspace-resolver-builder/interfaces/workspace-resolver-builder-factory.interface';
import { WorkspaceQueryRunnerOptions } from 'src/workspace/workspace-query-runner/interfaces/query-runner-optionts.interface';
import { WorkspaceQueryRunnerService } from 'src/workspace/workspace-query-runner/workspace-query-runner.service';
import { QuickActionsService } from 'src/core/quick-actions/quick-actions.service';
@Injectable()
export class ExecuteQuickActionOnOneResolverFactory
implements WorkspaceResolverBuilderFactoryInterface
{
public static methodName = 'executeQuickActionOnOne' as const;
constructor(
private readonly workspaceQueryRunnerService: WorkspaceQueryRunnerService,
private readonly quickActionsService: QuickActionsService,
) {}
create(
context: WorkspaceSchemaBuilderContext,
): Resolver<ExecuteQuickActionOnOneResolverArgs> {
const internalContext = context;
return (_source, args, context, info) => {
return this.executeQuickActionOnOne(args, {
targetTableName: internalContext.targetTableName,
workspaceId: internalContext.workspaceId,
info,
fieldMetadataCollection: internalContext.fieldMetadataCollection,
});
};
}
private async executeQuickActionOnOne<Record extends IRecord = IRecord>(
args: DeleteOneResolverArgs,
options: WorkspaceQueryRunnerOptions,
): Promise<Record | undefined> {
switch (options.targetTableName) {
case 'company': {
await this.quickActionsService.executeQuickActionOnCompany(
args.id,
options.workspaceId,
);
break;
}
case 'person': {
await this.quickActionsService.createCompanyFromPerson(
args.id,
options.workspaceId,
);
break;
}
default:
// TODO: different quick actions per object on frontend
break;
}
return this.workspaceQueryRunnerService.findOne(
{ filter: { id: { eq: args.id } } } as FindOneResolverArgs,
options,
);
}
}

View File

@ -7,6 +7,7 @@ import { CreateOneResolverFactory } from './create-one-resolver.factory';
import { UpdateOneResolverFactory } from './update-one-resolver.factory';
import { DeleteOneResolverFactory } from './delete-one-resolver.factory';
import { DeleteManyResolverFactory } from './delete-many-resolver.factory';
import { ExecuteQuickActionOnOneResolverFactory } from './execute-quick-action-on-one-resolver.factory';
export const workspaceResolverBuilderFactories = [
FindManyResolverFactory,
@ -15,6 +16,7 @@ export const workspaceResolverBuilderFactories = [
CreateOneResolverFactory,
UpdateOneResolverFactory,
DeleteOneResolverFactory,
ExecuteQuickActionOnOneResolverFactory,
UpdateManyResolverFactory,
DeleteManyResolverFactory,
];
@ -29,6 +31,7 @@ export const workspaceResolverBuilderMethodNames = {
CreateOneResolverFactory.methodName,
UpdateOneResolverFactory.methodName,
DeleteOneResolverFactory.methodName,
ExecuteQuickActionOnOneResolverFactory.methodName,
UpdateManyResolverFactory.methodName,
DeleteManyResolverFactory.methodName,
],

View File

@ -51,6 +51,10 @@ export interface DeleteOneResolverArgs {
id: string;
}
export interface ExecuteQuickActionOnOneResolverArgs {
id: string;
}
export interface DeleteManyResolverArgs<Filter = any> {
filter: Filter;
}

View File

@ -1,13 +1,14 @@
import { Module } from '@nestjs/common';
import { WorkspaceQueryRunnerModule } from 'src/workspace/workspace-query-runner/workspace-query-runner.module';
import { QuickActionsModule } from 'src/core/quick-actions/quick-actions.module';
import { WorkspaceResolverFactory } from './workspace-resolver.factory';
import { workspaceResolverBuilderFactories } from './factories/factories';
@Module({
imports: [WorkspaceQueryRunnerModule],
imports: [WorkspaceQueryRunnerModule, QuickActionsModule],
providers: [...workspaceResolverBuilderFactories, WorkspaceResolverFactory],
exports: [WorkspaceResolverFactory],
})

View File

@ -7,6 +7,7 @@ import { ObjectMetadataInterface } from 'src/metadata/field-metadata/interfaces/
import { getResolverName } from 'src/workspace/utils/get-resolver-name.util';
import { UpdateManyResolverFactory } from 'src/workspace/workspace-resolver-builder/factories/update-many-resolver.factory';
import { DeleteManyResolverFactory } from 'src/workspace/workspace-resolver-builder/factories/delete-many-resolver.factory';
import { ExecuteQuickActionOnOneResolverFactory } from 'src/workspace/workspace-resolver-builder/factories/execute-quick-action-on-one-resolver.factory';
import { FindManyResolverFactory } from './factories/find-many-resolver.factory';
import { FindOneResolverFactory } from './factories/find-one-resolver.factory';
@ -31,6 +32,7 @@ export class WorkspaceResolverFactory {
private readonly createOneResolverFactory: CreateOneResolverFactory,
private readonly updateOneResolverFactory: UpdateOneResolverFactory,
private readonly deleteOneResolverFactory: DeleteOneResolverFactory,
private readonly executeQuickActionOnOneResolverFactory: ExecuteQuickActionOnOneResolverFactory,
private readonly updateManyResolverFactory: UpdateManyResolverFactory,
private readonly deleteManyResolverFactory: DeleteManyResolverFactory,
) {}
@ -50,6 +52,7 @@ export class WorkspaceResolverFactory {
['createOne', this.createOneResolverFactory],
['updateOne', this.updateOneResolverFactory],
['deleteOne', this.deleteOneResolverFactory],
['executeQuickActionOnOne', this.executeQuickActionOnOneResolverFactory],
['updateMany', this.updateManyResolverFactory],
['deleteMany', this.deleteManyResolverFactory],
]);

View File

@ -34,6 +34,9 @@ describe('getResolverArgs', () => {
deleteOne: {
id: { type: FieldMetadataType.UUID, isNullable: false },
},
executeQuickActionOnOne: {
id: { type: FieldMetadataType.UUID, isNullable: false },
},
};
// Test each resolver type

View File

@ -76,6 +76,13 @@ export const getResolverArgs = (
isNullable: false,
},
};
case 'executeQuickActionOnOne':
return {
id: {
type: FieldMetadataType.UUID,
isNullable: false,
},
};
case 'updateMany':
return {
data: {