mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rest Api] Fix find duplicates endpoint (#12044)

Browse Source 
- fix endpoint
- migrate to new rest api v2 service
- add integration test
This commit is contained in:
martmull
2025-05-14 22:03:59 +02:00
committed by GitHub
parent fdc7d6c93c
commit 81cc5da982
19 changed files with 722 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/__tests__/graphql-query-find-duplicates-resolver.service.spec.ts
View File

@ -1,129 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { mockPersonObjectMetadata } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonObjectMetadata';
import { mockPersonRecords } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonRecords';
import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper';
import { GraphqlQueryFindDuplicatesResolverService } from 'src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service';
import { ApiEventEmitterService } from 'src/engine/api/graphql/graphql-query-runner/services/api-event-emitter.service';
import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory';
import { QueryRunnerArgsFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory';
import { WorkspaceQueryHookService } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/workspace-query-hook.service';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
describe('GraphqlQueryFindDuplicatesResolverService', () => {
let service: GraphqlQueryFindDuplicatesResolverService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
GraphqlQueryFindDuplicatesResolverService,
WorkspaceQueryHookService,
QueryRunnerArgsFactory,
QueryResultGettersFactory,
ApiEventEmitterService,
TwentyORMGlobalManager,
ProcessNestedRelationsHelper,
FeatureFlagService,
PermissionsService,
UserRoleService,
],
})
.overrideProvider(WorkspaceQueryHookService)
.useValue({})
.overrideProvider(QueryRunnerArgsFactory)
.useValue({})
.overrideProvider(QueryResultGettersFactory)
.useValue({})
.overrideProvider(ApiEventEmitterService)
.useValue({})
.overrideProvider(TwentyORMGlobalManager)
.useValue({})
.overrideProvider(ProcessNestedRelationsHelper)
.useValue({})
.overrideProvider(FeatureFlagService)
.useValue({})
.overrideProvider(PermissionsService)
.useValue({})
.overrideProvider(UserRoleService)
.useValue({})
.compile();
service = module.get<GraphqlQueryFindDuplicatesResolverService>(
GraphqlQueryFindDuplicatesResolverService,
);
});
describe('buildDuplicateConditions', () => {
it('should build conditions based on duplicate criteria from composite field', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['emailsPrimaryEmail']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({
or: [
{
emailsPrimaryEmail: {
eq: 'test@test.fr',
},
},
],
id: {
neq: 'recordId',
},
});
});
it('should build conditions based on duplicate criteria from basic field', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({
or: [
{
jobTitle: {
eq: 'Test job',
},
},
],
id: {
neq: 'recordId',
},
});
});
it('should not build conditions based on duplicate criteria if record value is null or too small', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['linkedinLinkPrimaryLinkUrl']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({});
});
it('should build conditions based on duplicate criteria and without recordId filter', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonRecords,
);
expect(duplicateConditons).toEqual({
or: [
{
jobTitle: {
eq: 'Test job',
},
},
],
});
});
});
});

77
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service.ts
View File

@ -9,7 +9,6 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/interfaces/base-resolver-service';
import {
ObjectRecord,
ObjectRecordFilter,
OrderByDirection,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { IConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/connection.interface';
@ -22,14 +21,10 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { settings } from 'src/engine/constants/settings';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util';
import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
import {
formatResult,
getCompositeFieldMetadataMap,
} from 'src/engine/twenty-orm/utils/format-result.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
@Injectable()
export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseResolverService<
@ -90,7 +85,7 @@ export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseR
const duplicateConnections: IConnection<ObjectRecord>[] = await Promise.all(
objectRecords.map(async (record) => {
const duplicateConditions = this.buildDuplicateConditions(
const duplicateConditions = buildDuplicateConditions(
objectMetadataItemWithFieldMaps,
[record],
record.id,
@ -143,72 +138,6 @@ export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseR
return duplicateConnections;
}
buildDuplicateConditions(
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
records?: Partial<ObjectRecord>[] | undefined,
filteringByExistingRecordId?: string,
): Partial<ObjectRecordFilter> {
if (!records || records.length === 0) {
return {};
}
const criteriaCollection =
objectMetadataItemWithFieldMaps.duplicateCriteria || [];
const formattedRecords = formatData(
records,
objectMetadataItemWithFieldMaps,
);
const compositeFieldMetadataMap = getCompositeFieldMetadataMap(
objectMetadataItemWithFieldMaps,
);
const conditions = formattedRecords.flatMap((record) => {
const criteriaWithMatchingArgs = criteriaCollection.filter((criteria) =>
criteria.every((columnName) => {
const value = record[columnName] as string | undefined;
return (
value && value.length >= settings.minLengthOfStringForDuplicateCheck
);
}),
);
return criteriaWithMatchingArgs.map((criteria) => {
const condition = {};
criteria.forEach((columnName) => {
const compositeFieldMetadata =
compositeFieldMetadataMap.get(columnName);
if (compositeFieldMetadata) {
condition[compositeFieldMetadata.parentField] = {
...condition[compositeFieldMetadata.parentField],
[compositeFieldMetadata.name]: { eq: record[columnName] },
};
} else {
condition[columnName] = { eq: record[columnName] };
}
});
return condition;
});
});
const filter: Partial<ObjectRecordFilter> = {};
if (conditions && !isEmpty(conditions)) {
filter.or = conditions;
if (filteringByExistingRecordId) {
filter.id = { neq: filteringByExistingRecordId };
}
}
return filter;
}
async validate(
args: FindDuplicatesResolverArgs,
_options: WorkspaceQueryRunnerOptions,

2
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts
View File

@ -23,7 +23,7 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper';
import { computeCursorArgFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-cursor-arg-filter';
import { computeCursorArgFilter } from 'src/engine/api/utils/compute-cursor-arg-filter.utils';
import {
getCursor,
getPaginationInfo,

25
packages/twenty-server/src/engine/api/rest/core/controllers/rest-api-core.controller.ts
View File

@ -13,10 +13,8 @@ import {
import { Request, Response } from 'express';
import { RestApiCoreServiceV2 } from 'src/engine/api/rest/core/rest-api-core-v2.service';
import { RestApiCoreService } from 'src/engine/api/rest/core/rest-api-core.service';
import { RestApiCoreService } from 'src/engine/api/rest/core/services/rest-api-core.service';
import { RestApiExceptionFilter } from 'src/engine/api/rest/rest-api-exception.filter';
import { cleanGraphQLResponse } from 'src/engine/api/rest/utils/clean-graphql-response.utils';
import { JwtAuthGuard } from 'src/engine/guards/jwt-auth.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
@ -24,49 +22,46 @@ import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
@UseGuards(JwtAuthGuard, WorkspaceAuthGuard)
@UseFilters(RestApiExceptionFilter)
export class RestApiCoreController {
constructor(
private readonly restApiCoreService: RestApiCoreService,
private readonly restApiCoreServiceV2: RestApiCoreServiceV2,
) {}
constructor(private readonly restApiCoreService: RestApiCoreService) {}
@Post('batch/*')
async handleApiPostBatch(@Req() request: Request, @Res() res: Response) {
const result = await this.restApiCoreServiceV2.createMany(request);
const result = await this.restApiCoreService.createMany(request);
res.status(201).send(result);
}
@Post('duplicates')
@Post('*/duplicates')
async handleApiFindDuplicates(@Req() request: Request, @Res() res: Response) {
const result = await this.restApiCoreService.findDuplicates(request);
res.status(200).send(cleanGraphQLResponse(result.data.data));
res.status(200).send(result);
}
@Post('*')
async handleApiPost(@Req() request: Request, @Res() res: Response) {
const result = await this.restApiCoreServiceV2.createOne(request);
const result = await this.restApiCoreService.createOne(request);
res.status(201).send(result);
}
@Get('*')
async handleApiGet(@Req() request: Request, @Res() res: Response) {
const result = await this.restApiCoreServiceV2.get(request);
const result = await this.restApiCoreService.get(request);
res.status(200).send(result);
}
@Delete('*')
async handleApiDelete(@Req() request: Request, @Res() res: Response) {
const result = await this.restApiCoreServiceV2.delete(request);
const result = await this.restApiCoreService.delete(request);
res.status(200).send(result);
}
@Patch('*')
async handleApiPatch(@Req() request: Request, @Res() res: Response) {
const result = await this.restApiCoreServiceV2.update(request);
const result = await this.restApiCoreService.update(request);
res.status(200).send(result);
}
@ -76,7 +71,7 @@ export class RestApiCoreController {
// of PATCH, and because the PUT verb is often used as a PATCH.
@Put('*')
async handleApiPut(@Req() request: Request, @Res() res: Response) {
const result = await this.restApiCoreServiceV2.update(request);
const result = await this.restApiCoreService.update(request);
res.status(200).send(result);
}

147
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-find-duplicates.handler.ts Normal file
View File

@ -0,0 +1,147 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import { Request } from 'express';
import isEmpty from 'lodash.isempty';
import { In } from 'typeorm';
import {
FormatResult,
RestApiBaseHandler,
} from 'src/engine/api/rest/core/interfaces/rest-api-base.handler';
import { ObjectRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
@Injectable()
export class RestApiFindDuplicatesHandler extends RestApiBaseHandler {
async handle(request: Request) {
this.validate(request);
const {
objectMetadataNameSingular,
repository,
objectMetadata,
objectMetadataItemWithFieldsMaps,
} = await this.getRepositoryAndMetadataOrFail(request);
const existingRecordsQueryBuilder = repository.createQueryBuilder(
objectMetadataItemWithFieldsMaps.nameSingular,
);
let objectRecords: Partial<ObjectRecord>[] = [];
if (request.body.ids) {
const nonFormattedObjectRecords = (await existingRecordsQueryBuilder
.where({ id: In(request.body.ids) })
.getMany()) as ObjectRecord[];
objectRecords = formatResult(
nonFormattedObjectRecords,
objectMetadataItemWithFieldsMaps,
objectMetadata.objectMetadataMaps,
);
} else if (request.body.data && !isEmpty(request.body.data)) {
objectRecords = request.body.data;
}
const duplicateConditions = objectRecords.map((record) =>
buildDuplicateConditions(
objectMetadataItemWithFieldsMaps,
[record],
record.id,
),
);
const result: { data: FormatResult[] } = {
data: [],
};
for (const duplicateCondition of duplicateConditions) {
const {
records,
isForwardPagination,
hasMoreRecords,
totalCount,
startCursor,
endCursor,
} = await this.findRecords({
request,
repository,
objectMetadata,
objectMetadataNameSingular,
objectMetadataItemWithFieldsMaps,
extraFilters: duplicateCondition,
});
const paginatedResult = this.formatPaginatedDuplicatesResult({
finalRecords: records,
objectMetadataNameSingular,
isForwardPagination,
hasMoreRecords,
totalCount,
startCursor,
endCursor,
});
result.data.push(paginatedResult);
}
return result;
}
private validate(request: Request) {
const { data, ids } = request.body;
if (!data && !ids) {
throw new BadRequestException(
'You have to provide either "data" or "ids" argument',
);
}
if (data && ids) {
throw new BadRequestException(
'You cannot provide both "data" and "ids" arguments',
);
}
if (!ids && isEmpty(data)) {
throw new BadRequestException(
'The "data" condition can not be empty when "ids" input not provided',
);
}
}
formatPaginatedDuplicatesResult({
finalRecords,
objectMetadataNameSingular,
isForwardPagination,
hasMoreRecords,
totalCount,
startCursor,
endCursor,
}: {
finalRecords: any[];
objectMetadataNameSingular: string;
isForwardPagination: boolean;
hasMoreRecords: boolean;
totalCount: number;
startCursor: string | null;
endCursor: string | null;
}) {
const hasPreviousPage = !isForwardPagination && hasMoreRecords;
return this.formatResult({
operation: 'findDuplicates',
objectNameSingular: objectMetadataNameSingular,
data: isForwardPagination ? finalRecords : finalRecords.reverse(),
pageInfo: {
hasNextPage: isForwardPagination && hasMoreRecords,
...(hasPreviousPage ? { hasPreviousPage } : {}),
startCursor,
endCursor,
},
totalCount,
});
}
}

35
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-get-many.handler.ts → packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-find-many.handler.ts
View File

@ -5,7 +5,7 @@ import { Request } from 'express';
import { RestApiBaseHandler } from 'src/engine/api/rest/core/interfaces/rest-api-base.handler';
@Injectable()
export class RestApiGetManyHandler extends RestApiBaseHandler {
export class RestApiFindManyHandler extends RestApiBaseHandler {
async handle(request: Request) {
const {
objectMetadataNameSingular,
@ -40,4 +40,37 @@ export class RestApiGetManyHandler extends RestApiBaseHandler {
endCursor,
});
}
formatPaginatedResult({
finalRecords,
objectMetadataNamePlural,
isForwardPagination,
hasMoreRecords,
totalCount,
startCursor,
endCursor,
}: {
finalRecords: any[];
objectMetadataNamePlural: string;
isForwardPagination: boolean;
hasMoreRecords: boolean;
totalCount: number;
startCursor: string | null;
endCursor: string | null;
}) {
const hasPreviousPage = !isForwardPagination && hasMoreRecords;
return this.formatResult({
operation: 'findMany',
objectNamePlural: objectMetadataNamePlural,
data: isForwardPagination ? finalRecords : finalRecords.reverse(),
pageInfo: {
hasNextPage: isForwardPagination && hasMoreRecords,
...(hasPreviousPage ? { hasPreviousPage } : {}),
startCursor,
endCursor,
},
totalCount,
});
}
}

2
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-get-one.handler.ts → packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-find-one.handler.ts
View File

@ -8,7 +8,7 @@ import { RestApiBaseHandler } from 'src/engine/api/rest/core/interfaces/rest-api
import { parseCorePath } from 'src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils';
@Injectable()
export class RestApiGetOneHandler extends RestApiBaseHandler {
export class RestApiFindOneHandler extends RestApiBaseHandler {
async handle(request: Request) {
const { id: recordId } = parseCorePath(request);

84
packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts
View File

@ -31,7 +31,7 @@ import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { formatResult as formatGetManyData } from 'src/engine/twenty-orm/utils/format-result.util';
import { encodeCursor } from 'src/engine/api/graphql/graphql-query-runner/utils/cursors.util';
import { computeCursorArgFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-cursor-arg-filter';
import { computeCursorArgFilter } from 'src/engine/api/utils/compute-cursor-arg-filter.utils';
export interface PageInfo {
hasNextPage?: boolean;
@ -41,7 +41,13 @@ export interface PageInfo {
}
interface FormatResultParams<T> {
operation: 'delete' | 'create' | 'update' | 'findOne' | 'findMany';
operation:
| 'delete'
| 'create'
| 'update'
| 'findOne'
| 'findMany'
| 'findDuplicates';
objectNameSingular?: string;
objectNamePlural?: string;
data: T;
@ -50,7 +56,7 @@ interface FormatResultParams<T> {
}
export interface FormatResult {
data: {
data?: {
[operation: string]: object;
};
pageInfo?: PageInfo;
@ -73,7 +79,9 @@ export abstract class RestApiBaseHandler {
@Inject()
protected readonly apiEventEmitterService: ApiEventEmitterService;
protected abstract handle(request: Request): Promise<FormatResult>;
protected abstract handle(
request: Request,
): Promise<FormatResult | { data: FormatResult[] }>;
public async getRepositoryAndMetadataOrFail(request: Request) {
const { workspace, apiKey, userWorkspaceId } = request;
@ -107,6 +115,12 @@ export abstract class RestApiBaseHandler {
objectMetadataNameSingular,
);
if (!isDefined(objectMetadataItemWithFieldsMaps)) {
throw new BadRequestException(
`Object metadata item with name singular ${objectMetadataNameSingular} not found`,
);
}
const shouldBypassPermissionChecks = !!apiKey;
const roleId =
@ -238,53 +252,28 @@ export abstract class RestApiBaseHandler {
prefix = objectNameSingular || '';
} else if (operation === 'findMany') {
prefix = objectNamePlural || '';
} else if (operation === 'findDuplicates') {
prefix = `${objectNameSingular}Duplicates`;
} else {
prefix =
operation + capitalize(objectNameSingular || objectNamePlural || '');
}
return {
data: {
[prefix]: data,
},
...(operation === 'findDuplicates'
? {
[prefix]: data,
}
: {
data: {
[prefix]: data,
},
}),
...(isDefined(pageInfo) ? { pageInfo } : {}),
...(isDefined(totalCount) ? { totalCount } : {}),
};
}
formatPaginatedResult({
finalRecords,
objectMetadataNamePlural,
isForwardPagination,
hasMoreRecords,
totalCount,
startCursor,
endCursor,
}: {
finalRecords: any[];
objectMetadataNamePlural: string;
isForwardPagination: boolean;
hasMoreRecords: boolean;
totalCount: number;
startCursor: string | null;
endCursor: string | null;
}) {
const hasPreviousPage = !isForwardPagination && hasMoreRecords;
return this.formatResult({
operation: 'findMany',
objectNamePlural: objectMetadataNamePlural,
data: isForwardPagination ? finalRecords : finalRecords.reverse(),
pageInfo: {
hasNextPage: isForwardPagination && hasMoreRecords,
...(hasPreviousPage ? { hasPreviousPage } : {}),
startCursor,
endCursor,
},
totalCount,
});
}
async findRecords({
request,
recordId,
@ -292,6 +281,7 @@ export abstract class RestApiBaseHandler {
objectMetadata,
objectMetadataNameSingular,
objectMetadataItemWithFieldsMaps,
extraFilters,
}: {
request: Request;
recordId?: string;
@ -304,6 +294,7 @@ export abstract class RestApiBaseHandler {
objectMetadataItemWithFieldsMaps:
| ObjectMetadataItemWithFieldMaps
| undefined;
extraFilters?: Partial<ObjectRecordFilter>;
}) {
const qb = repository.createQueryBuilder(objectMetadataNameSingular);
@ -331,6 +322,7 @@ export abstract class RestApiBaseHandler {
inputs,
objectMetadata,
isForwardPagination,
extraFilters,
});
let selectQueryBuilder = isDefined(filters)
@ -409,6 +401,7 @@ export abstract class RestApiBaseHandler {
inputs,
objectMetadata,
isForwardPagination,
extraFilters,
}: {
inputs: QueryVariables;
objectMetadata: {
@ -416,9 +409,16 @@ export abstract class RestApiBaseHandler {
objectMetadataMapItem: ObjectMetadataItemWithFieldMaps;
};
isForwardPagination: boolean;
extraFilters?: Partial<ObjectRecordFilter>;
}) {
let appliedFilters = inputs.filter;
if (extraFilters) {
appliedFilters = (appliedFilters
? { and: [appliedFilters, extraFilters] }
: extraFilters) as unknown as ObjectRecordFilter;
}
const cursor = inputs.startingAfter || inputs.endingBefore;
if (cursor) {
@ -429,9 +429,9 @@ export abstract class RestApiBaseHandler {
isForwardPagination,
);
appliedFilters = (inputs.filter
appliedFilters = (appliedFilters
? {
and: [inputs.filter, { or: cursorArgFilter }],
and: [appliedFilters, { or: cursorArgFilter }],
}
: { or: cursorArgFilter }) as unknown as ObjectRecordFilter;
}

17
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/__tests__/parse-core-path.utils.spec.ts
View File

@ -27,6 +27,14 @@ describe('parseCorePath', () => {
);
});
it('should throw for wrong request', () => {
const request: any = { path: '/rest' };
expect(() => parseCorePath(request)).toThrow(
"Query path '/rest' invalid. Valid examples: /rest/companies/id or /rest/companies or /rest/batch/companies",
);
});
it('should parse object from batch request', () => {
const request: any = { path: '/rest/batch/companies' };
@ -43,4 +51,13 @@ describe('parseCorePath', () => {
"Query path '/rest/batch/companies/uuid' invalid. Valid examples: /rest/companies/id or /rest/companies or /rest/batch/companies",
);
});
it('should parse object from duplicates request', () => {
const request: any = { path: '/rest/companies/duplicates' };
expect(parseCorePath(request)).toEqual({
object: 'companies',
id: undefined,
});
});
});

20
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts
View File

@ -5,7 +5,11 @@ import { Request } from 'express';
export const parseCorePath = (
request: Request,
): { object: string; id?: string } => {
const queryAction = request.path.replace('/rest/', '').split('/');
const queryAction = request.path
.replace('/rest/', '')
.replace('/rest', '')
.split('/')
.filter(Boolean);
if (queryAction.length > 2) {
throw new BadRequestException(
@ -13,13 +17,23 @@ export const parseCorePath = (
);
}
if (queryAction.length === 2 && queryAction[0] === 'batch') {
return { object: queryAction[1] };
if (queryAction.length === 0) {
throw new BadRequestException(
`Query path '${request.path}' invalid. Valid examples: /rest/companies/id or /rest/companies or /rest/batch/companies`,
);
}
if (queryAction.length === 1) {
return { object: queryAction[0] };
}
if (queryAction[0] === 'batch') {
return { object: queryAction[1] };
}
if (queryAction[1] === 'duplicates') {
return { object: queryAction[0] };
}
return { object: queryAction[0], id: queryAction[1] };
};

14
packages/twenty-server/src/engine/api/rest/core/rest-api-core.module.ts
View File

@ -4,8 +4,8 @@ import { HttpModule } from '@nestjs/axios';
import { RestApiDeleteOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-delete-one.handler';
import { RestApiCreateOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-create-one.handler';
import { RestApiUpdateOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-update-one.handler';
import { RestApiGetOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-get-one.handler';
import { RestApiGetManyHandler } from 'src/engine/api/rest/core/handlers/rest-api-get-many.handler';
import { RestApiFindOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-find-one.handler';
import { RestApiFindManyHandler } from 'src/engine/api/rest/core/handlers/rest-api-find-many.handler';
import { CoreQueryBuilderModule } from 'src/engine/api/rest/core/query-builder/core-query-builder.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
@ -13,20 +13,21 @@ import { RecordTransformerModule } from 'src/engine/core-modules/record-transfor
import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module';
import { RestApiCoreController } from 'src/engine/api/rest/core/controllers/rest-api-core.controller';
import { coreQueryBuilderFactories } from 'src/engine/api/rest/core/query-builder/factories/factories';
import { RestApiCoreServiceV2 } from 'src/engine/api/rest/core/rest-api-core-v2.service';
import { RestApiCoreService } from 'src/engine/api/rest/core/rest-api-core.service';
import { RestApiCoreService } from 'src/engine/api/rest/core/services/rest-api-core.service';
import { RestApiService } from 'src/engine/api/rest/rest-api.service';
import { ApiEventEmitterService } from 'src/engine/api/graphql/graphql-query-runner/services/api-event-emitter.service';
import { AuthModule } from 'src/engine/core-modules/auth/auth.module';
import { RestApiCreateManyHandler } from 'src/engine/api/rest/core/handlers/rest-api-create-many.handler';
import { RestApiFindDuplicatesHandler } from 'src/engine/api/rest/core/handlers/rest-api-find-duplicates.handler';
const restApiCoreResolvers = [
RestApiDeleteOneHandler,
RestApiCreateOneHandler,
RestApiCreateManyHandler,
RestApiUpdateOneHandler,
RestApiGetOneHandler,
RestApiGetManyHandler,
RestApiFindOneHandler,
RestApiFindManyHandler,
RestApiFindDuplicatesHandler,
];
@Module({
@ -43,7 +44,6 @@ const restApiCoreResolvers = [
providers: [
RestApiService,
RestApiCoreService,
RestApiCoreServiceV2,
ApiEventEmitterService,
...coreQueryBuilderFactories,
...restApiCoreResolvers,

29
packages/twenty-server/src/engine/api/rest/core/rest-api-core.service.ts
View File

@ -1,29 +0,0 @@
import { Injectable } from '@nestjs/common';
import { Request } from 'express';
import { CoreQueryBuilderFactory } from 'src/engine/api/rest/core/query-builder/core-query-builder.factory';
import {
GraphqlApiType,
RestApiService,
} from 'src/engine/api/rest/rest-api.service';
@Injectable()
export class RestApiCoreService {
constructor(
private readonly coreQueryBuilderFactory: CoreQueryBuilderFactory,
private readonly restApiService: RestApiService,
) {}
async createMany(request: Request) {
const data = await this.coreQueryBuilderFactory.createMany(request);
return await this.restApiService.call(GraphqlApiType.CORE, request, data);
}
async findDuplicates(request: Request) {
const data = await this.coreQueryBuilderFactory.findDuplicates(request);
return await this.restApiService.call(GraphqlApiType.CORE, request, data);
}
}

20
packages/twenty-server/src/engine/api/rest/core/rest-api-core-v2.service.ts → packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts
View File

@ -7,19 +7,21 @@ import { parseCorePath } from 'src/engine/api/rest/core/query-builder/utils/path
import { RestApiDeleteOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-delete-one.handler';
import { RestApiCreateOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-create-one.handler';
import { RestApiUpdateOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-update-one.handler';
import { RestApiGetOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-get-one.handler';
import { RestApiGetManyHandler } from 'src/engine/api/rest/core/handlers/rest-api-get-many.handler';
import { RestApiFindOneHandler } from 'src/engine/api/rest/core/handlers/rest-api-find-one.handler';
import { RestApiFindManyHandler } from 'src/engine/api/rest/core/handlers/rest-api-find-many.handler';
import { RestApiCreateManyHandler } from 'src/engine/api/rest/core/handlers/rest-api-create-many.handler';
import { RestApiFindDuplicatesHandler } from 'src/engine/api/rest/core/handlers/rest-api-find-duplicates.handler';
@Injectable()
export class RestApiCoreServiceV2 {
export class RestApiCoreService {
constructor(
private readonly restApiDeleteOneHandler: RestApiDeleteOneHandler,
private readonly restApiCreateOneHandler: RestApiCreateOneHandler,
private readonly restApiCreateManyHandler: RestApiCreateManyHandler,
private readonly restApiUpdateOneHandler: RestApiUpdateOneHandler,
private readonly restApiGetOneHandler: RestApiGetOneHandler,
private readonly restApiGetManyHandler: RestApiGetManyHandler,
private readonly restApiFindOneHandler: RestApiFindOneHandler,
private readonly restApiFindManyHandler: RestApiFindManyHandler,
private readonly restApiFindDuplicatesHandler: RestApiFindDuplicatesHandler,
) {}
async delete(request: Request) {
@ -34,6 +36,10 @@ export class RestApiCoreServiceV2 {
return await this.restApiCreateManyHandler.handle(request);
}
async findDuplicates(request: Request) {
return await this.restApiFindDuplicatesHandler.handle(request);
}
async update(request: Request) {
return await this.restApiUpdateOneHandler.handle(request);
}
@ -42,9 +48,9 @@ export class RestApiCoreServiceV2 {
const { id: recordId } = parseCorePath(request);
if (isDefined(recordId)) {
return await this.restApiGetOneHandler.handle(request);
return await this.restApiFindOneHandler.handle(request);
} else {
return await this.restApiGetManyHandler.handle(request);
return await this.restApiFindManyHandler.handle(request);
}
}
}

74
packages/twenty-server/src/engine/api/utils/__tests__/build-duplicate-conditions.utils.spec.ts Normal file
View File

@ -0,0 +1,74 @@
import { mockPersonObjectMetadata } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonObjectMetadata';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
import { mockPersonRecords } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonRecords';
describe('buildDuplicateConditions', () => {
it('should build conditions based on duplicate criteria from composite field', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['emailsPrimaryEmail']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({
or: [
{
emailsPrimaryEmail: {
eq: 'test@test.fr',
},
},
],
id: {
neq: 'recordId',
},
});
});
it('should build conditions based on duplicate criteria from basic field', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({
or: [
{
jobTitle: {
eq: 'Test job',
},
},
],
id: {
neq: 'recordId',
},
});
});
it('should not build conditions based on duplicate criteria if record value is null or too small', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['linkedinLinkPrimaryLinkUrl']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({});
});
it('should build conditions based on duplicate criteria and without recordId filter', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonRecords,
);
expect(duplicateConditons).toEqual({
or: [
{
jobTitle: {
eq: 'Test job',
},
},
],
});
});
});

2
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/compute-cursor-arg-filter.spec.ts → packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts
View File

@ -3,7 +3,7 @@ import { FieldMetadataType } from 'twenty-shared/types';
import { OrderByDirection } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { GraphqlQueryRunnerException } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { computeCursorArgFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-cursor-arg-filter';
import { computeCursorArgFilter } from 'src/engine/api/utils/compute-cursor-arg-filter.utils';
describe('computeCursorArgFilter', () => {
const mockFieldMetadataMap = {

74
packages/twenty-server/src/engine/api/utils/build-duplicate-conditions.utils.ts Normal file
View File

@ -0,0 +1,74 @@
import isEmpty from 'lodash.isempty';
import {
ObjectRecord,
ObjectRecordFilter,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { settings } from 'src/engine/constants/settings';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { getCompositeFieldMetadataMap } from 'src/engine/twenty-orm/utils/format-result.util';
import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
export const buildDuplicateConditions = (
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
records?: Partial<ObjectRecord>[] | undefined,
filteringByExistingRecordId?: string,
): Partial<ObjectRecordFilter> => {
if (!records || records.length === 0) {
return {};
}
const criteriaCollection =
objectMetadataItemWithFieldMaps.duplicateCriteria || [];
const formattedRecords = formatData(records, objectMetadataItemWithFieldMaps);
const compositeFieldMetadataMap = getCompositeFieldMetadataMap(
objectMetadataItemWithFieldMaps,
);
const conditions = formattedRecords.flatMap((record) => {
const criteriaWithMatchingArgs = criteriaCollection.filter((criteria) =>
criteria.every((columnName) => {
const value = record[columnName] as string | undefined;
return (
value && value.length >= settings.minLengthOfStringForDuplicateCheck
);
}),
);
return criteriaWithMatchingArgs.map((criteria) => {
const condition = {};
criteria.forEach((columnName) => {
const compositeFieldMetadata =
compositeFieldMetadataMap.get(columnName);
if (compositeFieldMetadata) {
condition[compositeFieldMetadata.parentField] = {
...condition[compositeFieldMetadata.parentField],
[compositeFieldMetadata.name]: { eq: record[columnName] },
};
} else {
condition[columnName] = { eq: record[columnName] };
}
});
return condition;
});
});
const filter: Partial<ObjectRecordFilter> = {};
if (conditions && !isEmpty(conditions)) {
filter.or = conditions;
if (filteringByExistingRecordId) {
filter.id = { neq: filteringByExistingRecordId };
}
}
return filter;
};

0
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-cursor-arg-filter.ts → packages/twenty-server/src/engine/api/utils/compute-cursor-arg-filter.utils.ts
View File

2
packages/twenty-server/src/engine/core-modules/open-api/utils/responses.utils.ts
View File

@ -302,7 +302,7 @@ export const getFindDuplicatesResponse200 = (
},
},
},
companyDuplicates: {
[`${item.nameSingular}Duplicates`]: {
type: 'array',
items: {
$ref: schemaRef,

280
packages/twenty-server/test/integration/rest/suites/rest-api-core-find-duplicates.integration-spec.ts Normal file
View File

@ -0,0 +1,280 @@
import {
TEST_PERSON_1_ID,
TEST_PERSON_2_ID,
TEST_PERSON_3_ID,
} from 'test/integration/constants/test-person-ids.constants';
import { makeRestAPIRequest } from 'test/integration/rest/utils/make-rest-api-request.util';
import { deleteAllRecords } from 'test/integration/utils/delete-all-records';
import { TEST_PRIMARY_LINK_URL } from 'test/integration/constants/test-primary-link-url.constant';
import { TEST_COMPANY_1_ID } from 'test/integration/constants/test-company-ids.constants';
describe('Core REST API Find Duplicates endpoint', () => {
beforeAll(async () => {
await deleteAllRecords('person');
await deleteAllRecords('company');
await makeRestAPIRequest({
method: 'post',
path: '/companies',
body: {
id: TEST_COMPANY_1_ID,
domainName: {
primaryLinkUrl: TEST_PRIMARY_LINK_URL,
},
},
}).expect(201);
await makeRestAPIRequest({
method: 'post',
path: '/batch/people',
body: [
{
id: TEST_PERSON_1_ID,
companyId: TEST_COMPANY_1_ID,
name: {
firstName: 'John',
lastName: 'Doe',
},
},
{
id: TEST_PERSON_2_ID,
companyId: TEST_COMPANY_1_ID,
name: {
firstName: 'John',
lastName: 'Doe',
},
},
{
id: TEST_PERSON_3_ID,
companyId: TEST_COMPANY_1_ID,
name: {
firstName: 'Phil',
lastName: 'Collins',
},
},
],
}).expect(201);
});
it('should retrieve duplicates by object data', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates`,
body: {
data: [
{
name: {
firstName: 'John',
lastName: 'Doe',
},
},
],
},
}).expect(200);
const data = response.body.data;
expect(data.length).toBe(1);
const duplicatesInfo = data[0];
expect(duplicatesInfo.totalCount).toBe(2);
expect(duplicatesInfo.personDuplicates.length).toBe(2);
const [personDuplicated1, personDuplicated2] =
duplicatesInfo.personDuplicates;
expect(personDuplicated1.id).toBe(TEST_PERSON_1_ID);
expect(personDuplicated2.id).toBe(TEST_PERSON_2_ID);
});
it('should retrieve duplicates by ids', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates`,
body: {
ids: [TEST_PERSON_1_ID],
},
}).expect(200);
const data = response.body.data;
expect(data.length).toBe(1);
const duplicatesInfo = data[0];
expect(duplicatesInfo.totalCount).toBe(1);
expect(duplicatesInfo.personDuplicates.length).toBe(1);
const [personDuplicated] = duplicatesInfo.personDuplicates;
expect(personDuplicated.id).toBe(TEST_PERSON_2_ID);
});
it('should not provide wrong duplicates', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates`,
body: {
data: [
{
name: {
firstName: 'Not',
lastName: 'Existing',
},
},
],
},
}).expect(200);
const data = response.body.data;
expect(data.length).toBe(1);
const duplicatesInfo = data[0];
expect(duplicatesInfo.totalCount).toBe(0);
expect(duplicatesInfo.personDuplicates.length).toBe(0);
});
it('should return 400 error when empty object data provided', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates`,
body: {
data: [],
},
}).expect(400);
expect(response.body.messages[0]).toContain(
'The "data" condition can not be empty when "ids" input not provided',
);
expect(response.body.error).toBe('BadRequestException');
});
it('should return empty result when empty ids provided', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates`,
body: {
ids: [],
},
}).expect(200);
expect(response.body.data.length).toBe(0);
});
it('should return 400 error when ids and data are provided', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates`,
body: {
data: [],
ids: [],
},
}).expect(400);
expect(response.body.messages[0]).toContain(
'You cannot provide both "data" and "ids" arguments',
);
expect(response.body.error).toBe('BadRequestException');
});
it('should support depth 0 parameter', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates?depth=0`,
body: {
data: [
{
name: {
firstName: 'John',
lastName: 'Doe',
},
},
],
},
}).expect(200);
const data = response.body.data;
expect(data.length).toBe(1);
const duplicatesInfo = data[0];
const [personDuplicated1, personDuplicated2] =
duplicatesInfo.personDuplicates;
expect(personDuplicated1.companyId).toBe(TEST_COMPANY_1_ID);
expect(personDuplicated1.company).not.toBeDefined();
expect(personDuplicated2.companyId).toBe(TEST_COMPANY_1_ID);
expect(personDuplicated2.company).not.toBeDefined();
});
it('should support depth 1 parameter', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates?depth=1`,
body: {
data: [
{
name: {
firstName: 'John',
lastName: 'Doe',
},
},
],
},
}).expect(200);
const data = response.body.data;
expect(data.length).toBe(1);
const duplicatesInfo = data[0];
const [personDuplicated1, personDuplicated2] =
duplicatesInfo.personDuplicates;
expect(personDuplicated1.company).toBeDefined();
expect(personDuplicated1.company.id).toBe(TEST_COMPANY_1_ID);
expect(personDuplicated1.company.people).not.toBeDefined();
expect(personDuplicated2.company).toBeDefined();
expect(personDuplicated2.company.id).toBe(TEST_COMPANY_1_ID);
expect(personDuplicated2.company.people).not.toBeDefined();
});
it('should support depth 2 parameter', async () => {
const response = await makeRestAPIRequest({
method: 'post',
path: `/people/duplicates?depth=2`,
body: {
data: [
{
name: {
firstName: 'John',
lastName: 'Doe',
},
},
],
},
}).expect(200);
const data = response.body.data;
expect(data.length).toBe(1);
const duplicatesInfo = data[0];
const [personDuplicated1, personDuplicated2] =
duplicatesInfo.personDuplicates;
expect(personDuplicated1.company.people).toBeDefined();
expect(personDuplicated2.company.people).toBeDefined();
const depth2Person1 = personDuplicated1.company.people.find(
(p) => p.id === personDuplicated1.id,
);
const depth2Person2 = personDuplicated2.company.people.find(
(p) => p.id === personDuplicated2.id,
);
expect(depth2Person1).toBeDefined();
expect(depth2Person2).toBeDefined();
});
});