feat: find duplicate objects init (#4038)

* feat: find duplicate objects backend init

* refactor: move duplicate criteria to constants

* fix: correct constant usage after type change

* feat: skip query generation in case its not necessary

* feat: filter out existing duplicate

* feat: FE queries and hooks

* feat: show duplicates on FE

* refactor: should-skip-query moved to workspace utils

* refactor: naming improvements

* refactor: current record typings/parsing improvements

* refactor: throw error if existing record not found

* fix: domain -> domainName duplicate criteria

* refactor: fieldNames -> columnNames

* docs: add explanation to duplicate criteria collection

* feat: add person linkedinLinkUrl as duplicate criteria

* feat: throw early when bot id and data are empty

* refactor: trying to improve readability of filter criteria query

* refactor: naming improvements

* refactor: remove shouldSkipQuery

* feat: resolve empty array in case of empty filter

* feat: hide whole section in case of no duplicates

* feat: FE display list the same way as relations

* test: basic unit test coverage

* Refactor Record detail section front

* Use Create as input argument of findDuplicates

* Improve coverage

* Fix

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
rostaklein
2024-02-24 19:12:21 +01:00
committed by GitHub
parent 05c206073d
commit 1b04dfe3c6
30 changed files with 875 additions and 100 deletions

View File

@ -13,6 +13,8 @@ export const getResolverName = (
return `${camelCase(objectMetadata.namePlural)}`;
case 'findOne':
return `${camelCase(objectMetadata.nameSingular)}`;
case 'findDuplicates':
return `${camelCase(objectMetadata.nameSingular)}Duplicates`;
case 'createMany':
return `create${pascalCase(objectMetadata.namePlural)}`;
case 'createOne':

View File

@ -0,0 +1,175 @@
import { Test, TestingModule } from '@nestjs/testing';
import { RecordFilter } from 'src/workspace/workspace-query-builder/interfaces/record.interface';
import { FindDuplicatesResolverArgs } from 'src/workspace/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { ArgsAliasFactory } from 'src/workspace/workspace-query-builder/factories/args-alias.factory';
import { FieldsStringFactory } from 'src/workspace/workspace-query-builder/factories/fields-string.factory';
import { FindDuplicatesQueryFactory } from 'src/workspace/workspace-query-builder/factories/find-duplicates-query.factory';
import { workspaceQueryBuilderOptions } from 'src/workspace/workspace-query-builder/utils-test/workspace-query-builder-options';
describe('FindDuplicatesQueryFactory', () => {
let service: FindDuplicatesQueryFactory;
const argAliasCreate = jest.fn();
beforeEach(async () => {
jest.resetAllMocks();
const module: TestingModule = await Test.createTestingModule({
providers: [
FindDuplicatesQueryFactory,
{
provide: FieldsStringFactory,
useValue: {
create: jest.fn().mockResolvedValue('fieldsString'),
// Mock implementation of FieldsStringFactory methods if needed
},
},
{
provide: ArgsAliasFactory,
useValue: {
create: argAliasCreate,
// Mock implementation of ArgsAliasFactory methods if needed
},
},
],
}).compile();
service = module.get<FindDuplicatesQueryFactory>(
FindDuplicatesQueryFactory,
);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('create', () => {
it('should return (first: 0) as a filter when args are missing', async () => {
const args: FindDuplicatesResolverArgs<RecordFilter> = {};
const query = await service.create(args, workspaceQueryBuilderOptions);
expect(query.trim()).toEqual(`query {
objectNameCollection(first: 0) {
fieldsString
}
}`);
});
it('should use firstName and lastName as a filter when both args are present', async () => {
argAliasCreate.mockReturnValue({
nameFirstName: 'John',
nameLastName: 'Doe',
});
const args: FindDuplicatesResolverArgs<RecordFilter> = {
data: {
name: {
firstName: 'John',
lastName: 'Doe',
},
} as unknown as RecordFilter,
};
const query = await service.create(args, {
...workspaceQueryBuilderOptions,
objectMetadataItem: {
...workspaceQueryBuilderOptions.objectMetadataItem,
nameSingular: 'person',
},
});
expect(query.trim()).toEqual(`query {
personCollection(filter: {or:[{nameFirstName:{ilike:\"%John%\"},nameLastName:{ilike:\"%Doe%\"}}]}) {
fieldsString
}
}`);
});
it('should return (first: 0) as a filter when only firstName is present', async () => {
argAliasCreate.mockReturnValue({
nameFirstName: 'John',
});
const args: FindDuplicatesResolverArgs<RecordFilter> = {
data: {
name: {
firstName: 'John',
},
} as unknown as RecordFilter,
};
const query = await service.create(args, {
...workspaceQueryBuilderOptions,
objectMetadataItem: {
...workspaceQueryBuilderOptions.objectMetadataItem,
nameSingular: 'person',
},
});
expect(query.trim()).toEqual(`query {
personCollection(first: 0) {
fieldsString
}
}`);
});
it('should use "currentRecord" as query args when its present', async () => {
argAliasCreate.mockReturnValue({
nameFirstName: 'John',
});
const args: FindDuplicatesResolverArgs<RecordFilter> = {
id: 'uuid',
};
const query = await service.create(
args,
{
...workspaceQueryBuilderOptions,
objectMetadataItem: {
...workspaceQueryBuilderOptions.objectMetadataItem,
nameSingular: 'person',
},
},
{
nameFirstName: 'Peter',
nameLastName: 'Parker',
},
);
expect(query.trim()).toEqual(`query {
personCollection(filter: {id:{neq:\"uuid\"},or:[{nameFirstName:{ilike:\"%Peter%\"},nameLastName:{ilike:\"%Parker%\"}}]}) {
fieldsString
}
}`);
});
});
describe('buildQueryForExistingRecord', () => {
it(`should include all the fields that exist for person inside "duplicateCriteriaCollection" constant`, async () => {
const query = service.buildQueryForExistingRecord('uuid', {
...workspaceQueryBuilderOptions,
objectMetadataItem: {
...workspaceQueryBuilderOptions.objectMetadataItem,
nameSingular: 'person',
},
});
expect(query.trim()).toEqual(`query {
personCollection(filter: { id: { eq: \"uuid\" }}){
edges {
node {
__typename
nameFirstName
nameLastName
linkedinLinkUrl
email
}
}
}
}`);
});
});
});

View File

@ -10,6 +10,7 @@ import { FindOneQueryFactory } from './find-one-query.factory';
import { UpdateOneQueryFactory } from './update-one-query.factory';
import { UpdateManyQueryFactory } from './update-many-query.factory';
import { DeleteManyQueryFactory } from './delete-many-query.factory';
import { FindDuplicatesQueryFactory } from './find-duplicates-query.factory';
export const workspaceQueryBuilderFactories = [
ArgsAliasFactory,
@ -21,6 +22,7 @@ export const workspaceQueryBuilderFactories = [
FieldsStringFactory,
FindManyQueryFactory,
FindOneQueryFactory,
FindDuplicatesQueryFactory,
UpdateOneQueryFactory,
UpdateManyQueryFactory,
DeleteManyQueryFactory,

View File

@ -0,0 +1,146 @@
import { Injectable, Logger } from '@nestjs/common';
import isEmpty from 'lodash.isempty';
import { WorkspaceQueryBuilderOptions } from 'src/workspace/workspace-query-builder/interfaces/workspace-query-builder-options.interface';
import { RecordFilter } from 'src/workspace/workspace-query-builder/interfaces/record.interface';
import { FindDuplicatesResolverArgs } from 'src/workspace/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { ObjectMetadataInterface } from 'src/metadata/field-metadata/interfaces/object-metadata.interface';
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
import { stringifyWithoutKeyQuote } from 'src/workspace/workspace-query-builder/utils/stringify-without-key-quote.util';
import { ArgsAliasFactory } from 'src/workspace/workspace-query-builder/factories/args-alias.factory';
import { duplicateCriteriaCollection } from 'src/workspace/workspace-resolver-builder/constants/duplicate-criteria.constants';
import { FieldsStringFactory } from './fields-string.factory';
@Injectable()
export class FindDuplicatesQueryFactory {
private readonly logger = new Logger(FindDuplicatesQueryFactory.name);
constructor(
private readonly fieldsStringFactory: FieldsStringFactory,
private readonly argsAliasFactory: ArgsAliasFactory,
) {}
async create<Filter extends RecordFilter = RecordFilter>(
args: FindDuplicatesResolverArgs<Filter>,
options: WorkspaceQueryBuilderOptions,
currentRecord?: Record<string, unknown>,
) {
const fieldsString = await this.fieldsStringFactory.create(
options.info,
options.fieldMetadataCollection,
options.objectMetadataCollection,
);
const argsData = this.getFindDuplicateBy<Filter>(
args,
options,
currentRecord,
);
const duplicateCondition = this.buildDuplicateCondition(
options.objectMetadataItem,
argsData,
args.id,
);
const filters = stringifyWithoutKeyQuote(duplicateCondition);
return `
query {
${computeObjectTargetTable(options.objectMetadataItem)}Collection${
isEmpty(duplicateCondition?.or)
? '(first: 0)'
: `(filter: ${filters})`
} {
${fieldsString}
}
}
`;
}
getFindDuplicateBy<Filter extends RecordFilter = RecordFilter>(
args: FindDuplicatesResolverArgs<Filter>,
options: WorkspaceQueryBuilderOptions,
currentRecord?: Record<string, unknown>,
) {
if (currentRecord) {
return currentRecord;
}
return this.argsAliasFactory.create(
args.data ?? {},
options.fieldMetadataCollection,
);
}
buildQueryForExistingRecord(
id: string,
options: WorkspaceQueryBuilderOptions,
) {
return `
query {
${computeObjectTargetTable(
options.objectMetadataItem,
)}Collection(filter: { id: { eq: "${id}" }}){
edges {
node {
__typename
${this.getApplicableDuplicateCriteriaCollection(
options.objectMetadataItem,
)
.flatMap((dc) => dc.columnNames)
.join('\n')}
}
}
}
}
`;
}
private buildDuplicateCondition(
objectMetadataItem: ObjectMetadataInterface,
argsData?: Record<string, unknown>,
filteringByExistingRecordId?: string,
) {
if (!argsData) {
return;
}
const criteriaCollection =
this.getApplicableDuplicateCriteriaCollection(objectMetadataItem);
const criteriaWithMatchingArgs = criteriaCollection.filter((criteria) =>
criteria.columnNames.every((columnName) => !!argsData[columnName]),
);
const filterCriteria = criteriaWithMatchingArgs.map((criteria) =>
Object.fromEntries(
criteria.columnNames.map((columnName) => [
columnName,
{ ilike: `%${argsData[columnName]}%` },
]),
),
);
return {
// when filtering by an existing record, we need to filter that explicit record out
...(filteringByExistingRecordId && {
id: { neq: filteringByExistingRecordId },
}),
// keep condition as "or" to get results by more duplicate criteria
or: filterCriteria,
};
}
private getApplicableDuplicateCriteriaCollection(
objectMetadataItem: ObjectMetadataInterface,
) {
return duplicateCriteriaCollection.filter(
(duplicateCriteria) =>
duplicateCriteria.objectName === objectMetadataItem.nameSingular,
);
}
}

View File

@ -19,3 +19,8 @@ export enum OrderByDirection {
export type RecordOrderBy = {
[Property in keyof Record]?: OrderByDirection;
};
export interface RecordDuplicateCriteria {
objectName: string;
columnNames: string[];
}

View File

@ -0,0 +1,13 @@
import { GraphQLResolveInfo } from 'graphql';
import { ObjectMetadataInterface } from 'src/metadata/field-metadata/interfaces/object-metadata.interface';
import { WorkspaceQueryBuilderOptions } from 'src/workspace/workspace-query-builder/interfaces/workspace-query-builder-options.interface';
import { objectMetadataItem } from 'src/utils/utils-test/object-metadata-item';
export const workspaceQueryBuilderOptions: WorkspaceQueryBuilderOptions = {
fieldMetadataCollection: [],
info: {} as GraphQLResolveInfo,
objectMetadataCollection: [],
objectMetadataItem: objectMetadataItem as ObjectMetadataInterface,
};

View File

@ -14,6 +14,7 @@ import {
DeleteOneResolverArgs,
UpdateManyResolverArgs,
DeleteManyResolverArgs,
FindDuplicatesResolverArgs,
} from 'src/workspace/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { FindManyQueryFactory } from './factories/find-many-query.factory';
@ -29,6 +30,7 @@ import {
DeleteManyQueryFactory,
DeleteManyQueryFactoryOptions,
} from './factories/delete-many-query.factory';
import { FindDuplicatesQueryFactory } from './factories/find-duplicates-query.factory';
@Injectable()
export class WorkspaceQueryBuilderFactory {
@ -37,6 +39,7 @@ export class WorkspaceQueryBuilderFactory {
constructor(
private readonly findManyQueryFactory: FindManyQueryFactory,
private readonly findOneQueryFactory: FindOneQueryFactory,
private readonly findDuplicatesQueryFactory: FindDuplicatesQueryFactory,
private readonly createManyQueryFactory: CreateManyQueryFactory,
private readonly updateOneQueryFactory: UpdateOneQueryFactory,
private readonly deleteOneQueryFactory: DeleteOneQueryFactory,
@ -61,6 +64,28 @@ export class WorkspaceQueryBuilderFactory {
return this.findOneQueryFactory.create<Filter>(args, options);
}
findDuplicates<Filter extends RecordFilter = RecordFilter>(
args: FindDuplicatesResolverArgs<Filter>,
options: WorkspaceQueryBuilderOptions,
existingRecord?: Record<string, unknown>,
): Promise<string> {
return this.findDuplicatesQueryFactory.create<Filter>(
args,
options,
existingRecord,
);
}
findDuplicatesExistingRecord(
id: string,
options: WorkspaceQueryBuilderOptions,
): string {
return this.findDuplicatesQueryFactory.buildQueryForExistingRecord(
id,
options,
);
}
createMany<Record extends IRecord = IRecord>(
args: CreateManyResolverArgs<Record>,
options: WorkspaceQueryBuilderOptions,

View File

@ -3,6 +3,7 @@ import {
CreateOneResolverArgs,
DeleteManyResolverArgs,
DeleteOneResolverArgs,
FindDuplicatesResolverArgs,
FindManyResolverArgs,
FindOneResolverArgs,
UpdateManyResolverArgs,
@ -16,6 +17,7 @@ export type ExecutePreHookMethod =
| 'deleteOne'
| 'findMany'
| 'findOne'
| 'findDuplicates'
| 'updateMany'
| 'updateOne';
@ -45,4 +47,6 @@ export type WorkspacePreQueryHookPayload<T> = T extends 'createMany'
? UpdateManyResolverArgs
: T extends 'updateOne'
? UpdateOneResolverArgs
: never;
: T extends 'findDuplicates'
? FindDuplicatesResolverArgs
: never;

View File

@ -6,6 +6,8 @@ import {
} from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import isEmpty from 'lodash.isempty';
import { IConnection } from 'src/utils/pagination/interfaces/connection.interface';
import {
Record as IRecord,
@ -17,6 +19,7 @@ import {
CreateOneResolverArgs,
DeleteManyResolverArgs,
DeleteOneResolverArgs,
FindDuplicatesResolverArgs,
FindManyResolverArgs,
FindOneResolverArgs,
UpdateManyResolverArgs,
@ -40,6 +43,7 @@ import { ObjectRecordCreateEvent } from 'src/integrations/event-emitter/types/ob
import { ObjectRecordUpdateEvent } from 'src/integrations/event-emitter/types/object-record-update.event';
import { WorkspacePreQueryHookService } from 'src/workspace/workspace-query-runner/workspace-pre-query-hook/workspace-pre-query-hook.service';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { NotFoundError } from 'src/filters/utils/graphql-errors.util';
import { WorkspaceQueryRunnerOptions } from './interfaces/query-runner-option.interface';
import {
@ -136,6 +140,74 @@ export class WorkspaceQueryRunnerService {
return parsedResult?.edges?.[0]?.node;
}
async findDuplicates<TRecord extends IRecord = IRecord>(
args: FindDuplicatesResolverArgs<TRecord>,
options: WorkspaceQueryRunnerOptions,
): Promise<IConnection<TRecord> | undefined> {
if (!args.data && !args.id) {
throw new BadRequestException(
'You have to provide either "data" or "id" argument',
);
}
if (!args.id && isEmpty(args.data)) {
throw new BadRequestException(
'The "data" condition can not be empty when ID input not provided',
);
}
const { workspaceId, userId, objectMetadataItem } = options;
let existingRecord: Record<string, unknown> | undefined;
if (args.id) {
const existingRecordQuery =
this.workspaceQueryBuilderFactory.findDuplicatesExistingRecord(
args.id,
options,
);
const existingRecordResult = await this.execute(
existingRecordQuery,
workspaceId,
);
const parsedResult = this.parseResult<Record<string, unknown>>(
existingRecordResult,
objectMetadataItem,
'',
);
existingRecord = parsedResult?.edges?.[0]?.node;
if (!existingRecord) {
throw new NotFoundError(`Object with id ${args.id} not found`);
}
}
const query = await this.workspaceQueryBuilderFactory.findDuplicates(
args,
options,
existingRecord,
);
await this.workspacePreQueryHookService.executePreHooks(
userId,
workspaceId,
objectMetadataItem.nameSingular,
'findDuplicates',
args,
);
const result = await this.execute(query, workspaceId);
return this.parseResult<IConnection<TRecord>>(
result,
objectMetadataItem,
'',
);
}
async createMany<Record extends IRecord = IRecord>(
args: CreateManyResolverArgs<Record>,
options: WorkspaceQueryRunnerOptions,

View File

@ -0,0 +1,30 @@
import { RecordDuplicateCriteria } from 'src/workspace/workspace-query-builder/interfaces/record.interface';
/**
* objectName: directly reference the name of the object from the metadata tables.
* columnNames: reference the column names not the field names.
* So if we need to reference a custom field, we should directly add the column name like `_customColumn`.
* If we need to terence a composite field, we should add all children of the composite like `nameFirstName` and `nameLastName`
*/
export const duplicateCriteriaCollection: RecordDuplicateCriteria[] = [
{
objectName: 'company',
columnNames: ['domainName'],
},
{
objectName: 'company',
columnNames: ['name'],
},
{
objectName: 'person',
columnNames: ['nameFirstName', 'nameLastName'],
},
{
objectName: 'person',
columnNames: ['linkedinLinkUrl'],
},
{
objectName: 'person',
columnNames: ['email'],
},
];

View File

@ -1,5 +1,6 @@
import { UpdateManyResolverFactory } from 'src/workspace/workspace-resolver-builder/factories/update-many-resolver.factory';
import { FindDuplicatesResolverFactory } from './find-duplicates-resolver.factory';
import { FindManyResolverFactory } from './find-many-resolver.factory';
import { FindOneResolverFactory } from './find-one-resolver.factory';
import { CreateManyResolverFactory } from './create-many-resolver.factory';
@ -12,6 +13,7 @@ import { ExecuteQuickActionOnOneResolverFactory } from './execute-quick-action-o
export const workspaceResolverBuilderFactories = [
FindManyResolverFactory,
FindOneResolverFactory,
FindDuplicatesResolverFactory,
CreateManyResolverFactory,
CreateOneResolverFactory,
UpdateOneResolverFactory,
@ -25,6 +27,7 @@ export const workspaceResolverBuilderMethodNames = {
queries: [
FindManyResolverFactory.methodName,
FindOneResolverFactory.methodName,
FindDuplicatesResolverFactory.methodName,
],
mutations: [
CreateManyResolverFactory.methodName,

View File

@ -0,0 +1,38 @@
import { Injectable } from '@nestjs/common';
import {
FindDuplicatesResolverArgs,
Resolver,
} from 'src/workspace/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { WorkspaceSchemaBuilderContext } from 'src/workspace/workspace-schema-builder/interfaces/workspace-schema-builder-context.interface';
import { WorkspaceResolverBuilderFactoryInterface } from 'src/workspace/workspace-resolver-builder/interfaces/workspace-resolver-builder-factory.interface';
import { WorkspaceQueryRunnerService } from 'src/workspace/workspace-query-runner/workspace-query-runner.service';
@Injectable()
export class FindDuplicatesResolverFactory
implements WorkspaceResolverBuilderFactoryInterface
{
public static methodName = 'findDuplicates' as const;
constructor(
private readonly workspaceQueryRunnerService: WorkspaceQueryRunnerService,
) {}
create(
context: WorkspaceSchemaBuilderContext,
): Resolver<FindDuplicatesResolverArgs> {
const internalContext = context;
return (_source, args, context, info) => {
return this.workspaceQueryRunnerService.findDuplicates(args, {
objectMetadataItem: internalContext.objectMetadataItem,
workspaceId: internalContext.workspaceId,
userId: internalContext.userId,
info,
fieldMetadataCollection: internalContext.fieldMetadataCollection,
objectMetadataCollection: internalContext.objectMetadataCollection,
});
};
}
}

View File

@ -26,6 +26,11 @@ export interface FindOneResolverArgs<Filter = any> {
filter?: Filter;
}
export interface FindDuplicatesResolverArgs<Data extends Record = Record> {
id?: string;
data?: Data;
}
export interface CreateOneResolverArgs<Data extends Record = Record> {
data: Data;
}
@ -81,5 +86,6 @@ export type ResolverArgs =
| DeleteOneResolverArgs
| FindManyResolverArgs
| FindOneResolverArgs
| FindDuplicatesResolverArgs
| UpdateManyResolverArgs
| UpdateOneResolverArgs;

View File

@ -9,6 +9,7 @@ import { UpdateManyResolverFactory } from 'src/workspace/workspace-resolver-buil
import { DeleteManyResolverFactory } from 'src/workspace/workspace-resolver-builder/factories/delete-many-resolver.factory';
import { ExecuteQuickActionOnOneResolverFactory } from 'src/workspace/workspace-resolver-builder/factories/execute-quick-action-on-one-resolver.factory';
import { FindDuplicatesResolverFactory } from './factories/find-duplicates-resolver.factory';
import { FindManyResolverFactory } from './factories/find-many-resolver.factory';
import { FindOneResolverFactory } from './factories/find-one-resolver.factory';
import { CreateManyResolverFactory } from './factories/create-many-resolver.factory';
@ -28,6 +29,7 @@ export class WorkspaceResolverFactory {
constructor(
private readonly findManyResolverFactory: FindManyResolverFactory,
private readonly findOneResolverFactory: FindOneResolverFactory,
private readonly findDuplicatesResolverFactory: FindDuplicatesResolverFactory,
private readonly createManyResolverFactory: CreateManyResolverFactory,
private readonly createOneResolverFactory: CreateOneResolverFactory,
private readonly updateOneResolverFactory: UpdateOneResolverFactory,
@ -49,6 +51,7 @@ export class WorkspaceResolverFactory {
>([
['findMany', this.findManyResolverFactory],
['findOne', this.findOneResolverFactory],
['findDuplicates', this.findDuplicatesResolverFactory],
['createMany', this.createManyResolverFactory],
['createOne', this.createOneResolverFactory],
['updateOne', this.updateOneResolverFactory],

View File

@ -74,7 +74,7 @@ export class RootTypeFactory {
const args = getResolverArgs(methodName);
const objectType = this.typeDefinitionsStorage.getObjectTypeByKey(
objectMetadata.id,
methodName === 'findMany'
['findMany', 'findDuplicates'].includes(methodName)
? ObjectTypeDefinitionKind.Connection
: ObjectTypeDefinitionKind.Plain,
);

View File

@ -69,6 +69,17 @@ export const getResolverArgs = (
isNullable: false,
},
};
case 'findDuplicates':
return {
id: {
type: FieldMetadataType.UUID,
isNullable: true,
},
data: {
kind: InputTypeDefinitionKind.Create,
isNullable: true,
},
};
case 'deleteOne':
return {
id: {