mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Upsert endpoint and CSV import upsert (#5970)

Browse Source 
This PR introduces an `upsert` parameter (along the existing `data`
param) for `createOne` and `createMany` mutations.

When upsert is set to `true`, the function will look for records with
the same id if an id was passed. If not id was passed, it will leverage
the existing duplicate check mechanism to find a duplicate. If a record
is found, then the function will perform an update instead of a create.

Unfortunately I had to remove some nice tests that existing on the args
factory. Those tests where mostly testing the duplication rule
generation logic but through a GraphQL angle. Since I moved the
duplication rule logic to a dedicated service, if I kept the tests but
mocked the service we wouldn't really be testing anything useful. The
right path would be to create new tests for this service that compare
the JSON output and not the GraphQL output but I chose not to work on
this as it's equivalent to rewriting the tests from scratch and I have
other competing priorities.
This commit is contained in:
Félix Malfait
2024-06-26 11:39:16 +02:00
committed by GitHub
parent 1736aee7ff
commit cf67ed09d0
43 changed files with 609 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/__tests__/find-duplicates-query.factory.spec.ts
View File

@ -1,206 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { FindDuplicatesResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { ArgsAliasFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/args-alias.factory';
import { FieldsStringFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/fields-string.factory';
import { FindDuplicatesQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/find-duplicates-query.factory';
import { workspaceQueryBuilderOptionsMock } from 'src/engine/api/graphql/workspace-query-builder/__mocks__/workspace-query-builder-options.mock';
describe('FindDuplicatesQueryFactory', () => {
let service: FindDuplicatesQueryFactory;
const argAliasCreate = jest.fn();
beforeEach(async () => {
jest.resetAllMocks();
const module: TestingModule = await Test.createTestingModule({
providers: [
FindDuplicatesQueryFactory,
{
provide: FieldsStringFactory,
useValue: {
create: jest.fn().mockResolvedValue('fieldsString'),
// Mock implementation of FieldsStringFactory methods if needed
},
},
{
provide: ArgsAliasFactory,
useValue: {
create: argAliasCreate,
// Mock implementation of ArgsAliasFactory methods if needed
},
},
],
}).compile();
service = module.get<FindDuplicatesQueryFactory>(
FindDuplicatesQueryFactory,
);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('create', () => {
it('should return (first: 0) as a filter when args are missing', async () => {
const args: FindDuplicatesResolverArgs<RecordFilter> = {};
const query = await service.create(
args,
workspaceQueryBuilderOptionsMock,
);
expect(query.trim()).toEqual(`query {
objectNameCollection(first: 0) {
fieldsString
}
}`);
});
it('should use firstName and lastName as a filter when both args are present', async () => {
argAliasCreate.mockReturnValue({
nameFirstName: 'John',
nameLastName: 'Doe',
});
const args: FindDuplicatesResolverArgs<RecordFilter> = {
data: {
name: {
firstName: 'John',
lastName: 'Doe',
},
} as unknown as RecordFilter,
};
const query = await service.create(args, {
...workspaceQueryBuilderOptionsMock,
objectMetadataItem: {
...workspaceQueryBuilderOptionsMock.objectMetadataItem,
nameSingular: 'person',
},
});
expect(query.trim()).toEqual(`query {
personCollection(filter: {or:[{nameFirstName:{eq:"John"},nameLastName:{eq:"Doe"}}]}) {
fieldsString
}
}`);
});
it('should ignore an argument if the string length is less than 3', async () => {
argAliasCreate.mockReturnValue({
linkedinLinkUrl: 'ab',
email: 'test@test.com',
});
const args: FindDuplicatesResolverArgs<RecordFilter> = {
data: {
linkedinLinkUrl: 'ab',
email: 'test@test.com',
} as unknown as RecordFilter,
};
const query = await service.create(args, {
...workspaceQueryBuilderOptionsMock,
objectMetadataItem: {
...workspaceQueryBuilderOptionsMock.objectMetadataItem,
nameSingular: 'person',
},
});
expect(query.trim()).toEqual(`query {
personCollection(filter: {or:[{email:{eq:"test@test.com"}}]}) {
fieldsString
}
}`);
});
it('should return (first: 0) as a filter when only firstName is present', async () => {
argAliasCreate.mockReturnValue({
nameFirstName: 'John',
});
const args: FindDuplicatesResolverArgs<RecordFilter> = {
data: {
name: {
firstName: 'John',
},
} as unknown as RecordFilter,
};
const query = await service.create(args, {
...workspaceQueryBuilderOptionsMock,
objectMetadataItem: {
...workspaceQueryBuilderOptionsMock.objectMetadataItem,
nameSingular: 'person',
},
});
expect(query.trim()).toEqual(`query {
personCollection(first: 0) {
fieldsString
}
}`);
});
it('should use "currentRecord" as query args when its present', async () => {
argAliasCreate.mockReturnValue({
nameFirstName: 'John',
});
const args: FindDuplicatesResolverArgs<RecordFilter> = {
id: 'uuid',
};
const query = await service.create(
args,
{
...workspaceQueryBuilderOptionsMock,
objectMetadataItem: {
...workspaceQueryBuilderOptionsMock.objectMetadataItem,
nameSingular: 'person',
},
},
{
nameFirstName: 'Peter',
nameLastName: 'Parker',
},
);
expect(query.trim()).toEqual(`query {
personCollection(filter: {id:{neq:"uuid"},or:[{nameFirstName:{eq:"Peter"},nameLastName:{eq:"Parker"}}]}) {
fieldsString
}
}`);
});
});
describe('buildQueryForExistingRecord', () => {
it(`should include all the fields that exist for person inside "duplicateCriteriaCollection" constant`, async () => {
const query = service.buildQueryForExistingRecord('uuid', {
...workspaceQueryBuilderOptionsMock,
objectMetadataItem: {
...workspaceQueryBuilderOptionsMock.objectMetadataItem,
nameSingular: 'person',
},
});
expect(query.trim()).toEqual(`query {
personCollection(filter: { id: { eq: "uuid" }}){
edges {
node {
__typename
nameFirstName
nameLastName
linkedinLinkUrl
email
}
}
}
}`);
});
});
});

2
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/create-many-query.factory.ts
View File

@ -22,7 +22,7 @@ export class CreateManyQueryFactory {
) {}
async create<Record extends IRecord = IRecord>(
args: CreateManyResolverArgs<Record>,
args: CreateManyResolverArgs<Partial<Record>>,
options: WorkspaceQueryBuilderOptions,
) {
const fieldsString = await this.fieldsStringFactory.create(

5
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/fields-string.factory.ts
View File

@ -6,6 +6,7 @@ import isEmpty from 'lodash.isempty';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { Record } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
@ -26,7 +27,7 @@ export class FieldsStringFactory {
fieldMetadataCollection: FieldMetadataInterface[],
objectMetadataCollection: ObjectMetadataInterface[],
): Promise<string> {
const selectedFields: Record<string, any> = graphqlFields(info);
const selectedFields: Partial<Record> = graphqlFields(info);
return this.createFieldsStringRecursive(
info,
@ -38,7 +39,7 @@ export class FieldsStringFactory {
async createFieldsStringRecursive(
info: GraphQLResolveInfo,
selectedFields: Record<string, any>,
selectedFields: Partial<Record>,
fieldMetadataCollection: FieldMetadataInterface[],
objectMetadataCollection: ObjectMetadataInterface[],
accumulator = '',

180
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/find-duplicates-query.factory.ts
View File

@ -3,15 +3,13 @@ import { Injectable, Logger } from '@nestjs/common';
import isEmpty from 'lodash.isempty';
import { WorkspaceQueryBuilderOptions } from 'src/engine/api/graphql/workspace-query-builder/interfaces/workspace-query-builder-options.interface';
import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { Record } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { FindDuplicatesResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { stringifyWithoutKeyQuote } from 'src/engine/api/graphql/workspace-query-builder/utils/stringify-without-key-quote.util';
import { ArgsAliasFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/args-alias.factory';
import { DUPLICATE_CRITERIA_COLLECTION } from 'src/engine/api/graphql/workspace-resolver-builder/constants/duplicate-criteria.constants';
import { settings } from 'src/engine/constants/settings';
import { DuplicateService } from 'src/engine/core-modules/duplicate/duplicate.service';
import { FieldsStringFactory } from './fields-string.factory';
@ -22,12 +20,13 @@ export class FindDuplicatesQueryFactory {
constructor(
private readonly fieldsStringFactory: FieldsStringFactory,
private readonly argsAliasFactory: ArgsAliasFactory,
private readonly duplicateService: DuplicateService,
) {}
async create<Filter extends RecordFilter = RecordFilter>(
args: FindDuplicatesResolverArgs<Filter>,
async create(
args: FindDuplicatesResolverArgs,
options: WorkspaceQueryBuilderOptions,
currentRecord?: Record<string, unknown>,
existingRecords?: Record[],
) {
const fieldsString = await this.fieldsStringFactory.create(
options.info,
@ -35,121 +34,66 @@ export class FindDuplicatesQueryFactory {
options.objectMetadataCollection,
);
const argsData = this.getFindDuplicateBy<Filter>(
args,
options,
currentRecord,
);
if (existingRecords) {
const query = existingRecords.reduce((acc, record, index) => {
return (
acc + this.buildQuery(fieldsString, options, undefined, record, index)
);
}, '');
const duplicateCondition = this.buildDuplicateCondition(
options.objectMetadataItem,
argsData,
args.id,
);
return `query {
${query}
}`;
}
const query = args.data?.reduce((acc, dataItem, index) => {
const argsData = this.argsAliasFactory.create(
dataItem ?? {},
options.fieldMetadataCollection,
);
return (
acc +
this.buildQuery(
fieldsString,
options,
argsData as Record,
undefined,
index,
)
);
}, '');
return `query {
${query}
}`;
}
buildQuery(
fieldsString: string,
options: WorkspaceQueryBuilderOptions,
data?: Record,
existingRecord?: Record,
index?: number,
) {
const duplicateCondition =
this.duplicateService.buildDuplicateConditionForGraphQL(
options.objectMetadataItem,
data ?? existingRecord,
existingRecord?.id,
);
const filters = stringifyWithoutKeyQuote(duplicateCondition);
return `
query {
${computeObjectTargetTable(options.objectMetadataItem)}Collection${
isEmpty(duplicateCondition?.or)
? '(first: 0)'
: `(filter: ${filters})`
} {
${fieldsString}
}
}
`;
}
getFindDuplicateBy<Filter extends RecordFilter = RecordFilter>(
args: FindDuplicatesResolverArgs<Filter>,
options: WorkspaceQueryBuilderOptions,
currentRecord?: Record<string, unknown>,
) {
if (currentRecord) {
return currentRecord;
return `${computeObjectTargetTable(
options.objectMetadataItem,
)}Collection${index}: ${computeObjectTargetTable(
options.objectMetadataItem,
)}Collection${
isEmpty(duplicateCondition?.or) ? '(first: 0)' : `(filter: ${filters})`
} {
${fieldsString}
}
return this.argsAliasFactory.create(
args.data ?? {},
options.fieldMetadataCollection,
);
}
buildQueryForExistingRecord(
id: string | number,
options: WorkspaceQueryBuilderOptions,
) {
const idQueryField = typeof id === 'string' ? `"${id}"` : id;
return `
query {
${computeObjectTargetTable(
options.objectMetadataItem,
)}Collection(filter: { id: { eq: ${idQueryField} }}){
edges {
node {
__typename
${this.getApplicableDuplicateCriteriaCollection(
options.objectMetadataItem,
)
.flatMap((dc) => dc.columnNames)
.join('\n')}
}
}
}
}
`;
}
private buildDuplicateCondition(
objectMetadataItem: ObjectMetadataInterface,
argsData?: Record<string, unknown>,
filteringByExistingRecordId?: string,
) {
if (!argsData) {
return;
}
const criteriaCollection =
this.getApplicableDuplicateCriteriaCollection(objectMetadataItem);
const criteriaWithMatchingArgs = criteriaCollection.filter((criteria) =>
criteria.columnNames.every((columnName) => {
const value = argsData[columnName] as string | undefined;
return (
!!value && value.length >= settings.minLengthOfStringForDuplicateCheck
);
}),
);
const filterCriteria = criteriaWithMatchingArgs.map((criteria) =>
Object.fromEntries(
criteria.columnNames.map((columnName) => [
columnName,
{ eq: argsData[columnName] },
]),
),
);
return {
// when filtering by an existing record, we need to filter that explicit record out
...(filteringByExistingRecordId && {
id: { neq: filteringByExistingRecordId },
}),
// keep condition as "or" to get results by more duplicate criteria
or: filterCriteria,
};
}
private getApplicableDuplicateCriteriaCollection(
objectMetadataItem: ObjectMetadataInterface,
) {
return DUPLICATE_CRITERIA_COLLECTION.filter(
(duplicateCriteria) =>
duplicateCriteria.objectName === objectMetadataItem.nameSingular,
);
`;
}
}

2
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/update-many-query.factory.ts
View File

@ -28,7 +28,7 @@ export class UpdateManyQueryFactory {
Record extends IRecord = IRecord,
Filter extends RecordFilter = RecordFilter,
>(
args: UpdateManyResolverArgs<Record, Filter>,
args: UpdateManyResolverArgs<Partial<Record>, Filter>,
options: UpdateManyQueryFactoryOptions,
) {
const fieldsString = await this.fieldsStringFactory.create(

3
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/update-one-query.factory.ts
View File

@ -20,7 +20,7 @@ export class UpdateOneQueryFactory {
) {}
async create<Record extends IRecord = IRecord>(
args: UpdateOneResolverArgs<Record>,
args: UpdateOneResolverArgs<Partial<Record>>,
options: WorkspaceQueryBuilderOptions,
) {
const fieldsString = await this.fieldsStringFactory.create(
@ -35,6 +35,7 @@ export class UpdateOneQueryFactory {
const argsData = {
...computedArgs.data,
id: undefined, // do not allow updating an existing object's id
updatedAt: new Date().toISOString(),
};

26
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/workspace-query-builder.factory.ts
View File

@ -64,37 +64,27 @@ export class WorkspaceQueryBuilderFactory {
return this.findOneQueryFactory.create<Filter>(args, options);
}
findDuplicates<Filter extends RecordFilter = RecordFilter>(
args: FindDuplicatesResolverArgs<Filter>,
findDuplicates(
args: FindDuplicatesResolverArgs,
options: WorkspaceQueryBuilderOptions,
existingRecord?: Record<string, unknown>,
existingRecords?: IRecord[],
): Promise<string> {
return this.findDuplicatesQueryFactory.create<Filter>(
return this.findDuplicatesQueryFactory.create(
args,
options,
existingRecord,
);
}
findDuplicatesExistingRecord(
id: string | number,
options: WorkspaceQueryBuilderOptions,
): string {
return this.findDuplicatesQueryFactory.buildQueryForExistingRecord(
id,
options,
existingRecords,
);
}
createMany<Record extends IRecord = IRecord>(
args: CreateManyResolverArgs<Record>,
args: CreateManyResolverArgs<Partial<Record>>,
options: WorkspaceQueryBuilderOptions,
): Promise<string> {
return this.createManyQueryFactory.create<Record>(args, options);
}
updateOne<Record extends IRecord = IRecord>(
initialArgs: UpdateOneResolverArgs<Record>,
initialArgs: UpdateOneResolverArgs<Partial<Record>>,
options: WorkspaceQueryBuilderOptions,
): Promise<string> {
return this.updateOneQueryFactory.create<Record>(initialArgs, options);
@ -111,7 +101,7 @@ export class WorkspaceQueryBuilderFactory {
Record extends IRecord = IRecord,
Filter extends RecordFilter = RecordFilter,
>(
args: UpdateManyResolverArgs<Record, Filter>,
args: UpdateManyResolverArgs<Partial<Record>, Filter>,
options: UpdateManyQueryFactoryOptions,
): Promise<string> {
return this.updateManyQueryFactory.create(args, options);

3
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/workspace-query-builder.module.ts
View File

@ -3,13 +3,14 @@ import { Module } from '@nestjs/common';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { FieldsStringFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/fields-string.factory';
import { RecordPositionQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/record-position-query.factory';
import { DuplicateModule } from 'src/engine/core-modules/duplicate/duplicate.module';
import { WorkspaceQueryBuilderFactory } from './workspace-query-builder.factory';
import { workspaceQueryBuilderFactories } from './factories/factories';
@Module({
imports: [ObjectMetadataModule],
imports: [ObjectMetadataModule, DuplicateModule],
providers: [...workspaceQueryBuilderFactories, WorkspaceQueryBuilderFactory],
exports: [
WorkspaceQueryBuilderFactory,