admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rest Api] Fix find duplicates endpoint (#12044)

Browse Source 
- fix endpoint
- migrate to new rest api v2 service
- add integration test
This commit is contained in:
martmull
2025-05-14 22:03:59 +02:00
committed by GitHub
parent fdc7d6c93c
commit 81cc5da982
19 changed files with 722 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/__tests__/graphql-query-find-duplicates-resolver.service.spec.ts
View File

@ -1,129 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { mockPersonObjectMetadata } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonObjectMetadata';
import { mockPersonRecords } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonRecords';
import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper';
import { GraphqlQueryFindDuplicatesResolverService } from 'src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service';
import { ApiEventEmitterService } from 'src/engine/api/graphql/graphql-query-runner/services/api-event-emitter.service';
import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory';
import { QueryRunnerArgsFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory';
import { WorkspaceQueryHookService } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/workspace-query-hook.service';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
describe('GraphqlQueryFindDuplicatesResolverService', () => {
let service: GraphqlQueryFindDuplicatesResolverService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
GraphqlQueryFindDuplicatesResolverService,
WorkspaceQueryHookService,
QueryRunnerArgsFactory,
QueryResultGettersFactory,
ApiEventEmitterService,
TwentyORMGlobalManager,
ProcessNestedRelationsHelper,
FeatureFlagService,
PermissionsService,
UserRoleService,
],
})
.overrideProvider(WorkspaceQueryHookService)
.useValue({})
.overrideProvider(QueryRunnerArgsFactory)
.useValue({})
.overrideProvider(QueryResultGettersFactory)
.useValue({})
.overrideProvider(ApiEventEmitterService)
.useValue({})
.overrideProvider(TwentyORMGlobalManager)
.useValue({})
.overrideProvider(ProcessNestedRelationsHelper)
.useValue({})
.overrideProvider(FeatureFlagService)
.useValue({})
.overrideProvider(PermissionsService)
.useValue({})
.overrideProvider(UserRoleService)
.useValue({})
.compile();
service = module.get<GraphqlQueryFindDuplicatesResolverService>(
GraphqlQueryFindDuplicatesResolverService,
);
});
describe('buildDuplicateConditions', () => {
it('should build conditions based on duplicate criteria from composite field', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['emailsPrimaryEmail']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({
or: [
{
emailsPrimaryEmail: {
eq: 'test@test.fr',
},
},
],
id: {
neq: 'recordId',
},
});
});
it('should build conditions based on duplicate criteria from basic field', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({
or: [
{
jobTitle: {
eq: 'Test job',
},
},
],
id: {
neq: 'recordId',
},
});
});
it('should not build conditions based on duplicate criteria if record value is null or too small', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['linkedinLinkPrimaryLinkUrl']]),
mockPersonRecords,
'recordId',
);
expect(duplicateConditons).toEqual({});
});
it('should build conditions based on duplicate criteria and without recordId filter', () => {
const duplicateConditons = service.buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonRecords,
);
expect(duplicateConditons).toEqual({
or: [
{
jobTitle: {
eq: 'Test job',
},
},
],
});
});
});
});

77
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service.ts
View File

@ -9,7 +9,6 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/interfaces/base-resolver-service';
import {
ObjectRecord,
ObjectRecordFilter,
OrderByDirection,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { IConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/connection.interface';
@ -22,14 +21,10 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { settings } from 'src/engine/constants/settings';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util';
import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
import {
formatResult,
getCompositeFieldMetadataMap,
} from 'src/engine/twenty-orm/utils/format-result.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
@Injectable()
export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseResolverService<
@ -90,7 +85,7 @@ export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseR
const duplicateConnections: IConnection<ObjectRecord>[] = await Promise.all(
objectRecords.map(async (record) => {
const duplicateConditions = this.buildDuplicateConditions(
const duplicateConditions = buildDuplicateConditions(
objectMetadataItemWithFieldMaps,
[record],
record.id,
@ -143,72 +138,6 @@ export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseR
return duplicateConnections;
}
buildDuplicateConditions(
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
records?: Partial<ObjectRecord>[] | undefined,
filteringByExistingRecordId?: string,
): Partial<ObjectRecordFilter> {
if (!records || records.length === 0) {
return {};
}
const criteriaCollection =
objectMetadataItemWithFieldMaps.duplicateCriteria || [];
const formattedRecords = formatData(
records,
objectMetadataItemWithFieldMaps,
);
const compositeFieldMetadataMap = getCompositeFieldMetadataMap(
objectMetadataItemWithFieldMaps,
);
const conditions = formattedRecords.flatMap((record) => {
const criteriaWithMatchingArgs = criteriaCollection.filter((criteria) =>
criteria.every((columnName) => {
const value = record[columnName] as string | undefined;
return (
value && value.length >= settings.minLengthOfStringForDuplicateCheck
);
}),
);
return criteriaWithMatchingArgs.map((criteria) => {
const condition = {};
criteria.forEach((columnName) => {
const compositeFieldMetadata =
compositeFieldMetadataMap.get(columnName);
if (compositeFieldMetadata) {
condition[compositeFieldMetadata.parentField] = {
...condition[compositeFieldMetadata.parentField],
[compositeFieldMetadata.name]: { eq: record[columnName] },
};
} else {
condition[columnName] = { eq: record[columnName] };
}
});
return condition;
});
});
const filter: Partial<ObjectRecordFilter> = {};
if (conditions && !isEmpty(conditions)) {
filter.or = conditions;
if (filteringByExistingRecordId) {
filter.id = { neq: filteringByExistingRecordId };
}
}
return filter;
}
async validate(
args: FindDuplicatesResolverArgs,
_options: WorkspaceQueryRunnerOptions,

2
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts
View File

@ -23,7 +23,7 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper';
import { computeCursorArgFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-cursor-arg-filter';
import { computeCursorArgFilter } from 'src/engine/api/utils/compute-cursor-arg-filter.utils';
import {
getCursor,
getPaginationInfo,

142
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/__tests__/compute-cursor-arg-filter.spec.ts
View File

@ -1,142 +0,0 @@
import { FieldMetadataType } from 'twenty-shared/types';
import { OrderByDirection } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { GraphqlQueryRunnerException } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { computeCursorArgFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/compute-cursor-arg-filter';
describe('computeCursorArgFilter', () => {
const mockFieldMetadataMap = {
name: {
type: FieldMetadataType.TEXT,
id: 'name-id',
name: 'name',
label: 'Name',
objectMetadataId: 'object-id',
},
age: {
type: FieldMetadataType.NUMBER,
id: 'age-id',
name: 'age',
label: 'Age',
objectMetadataId: 'object-id',
},
fullName: {
type: FieldMetadataType.FULL_NAME,
id: 'fullname-id',
name: 'fullName',
label: 'Full Name',
objectMetadataId: 'object-id',
},
};
describe('basic cursor filtering', () => {
it('should return empty array when cursor is empty', () => {
const result = computeCursorArgFilter({}, [], mockFieldMetadataMap, true);
expect(result).toEqual([]);
});
it('should compute forward pagination filter for single field', () => {
const cursor = { name: 'John' };
const orderBy = [{ name: OrderByDirection.AscNullsLast }];
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
true,
);
expect(result).toEqual([{ name: { gt: 'John' } }]);
});
it('should compute backward pagination filter for single field', () => {
const cursor = { name: 'John' };
const orderBy = [{ name: OrderByDirection.AscNullsLast }];
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
false,
);
expect(result).toEqual([{ name: { lt: 'John' } }]);
});
});
describe('multiple fields cursor filtering', () => {
it('should handle multiple cursor fields with forward pagination', () => {
const cursor = { name: 'John', age: 30 };
const orderBy = [
{ name: OrderByDirection.AscNullsLast },
{ age: OrderByDirection.DescNullsLast },
];
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
true,
);
expect(result).toEqual([
{ name: { gt: 'John' } },
{ name: { eq: 'John' }, age: { lt: 30 } },
]);
});
});
describe('composite field handling', () => {
it('should handle fullName composite field', () => {
const cursor = {
fullName: { firstName: 'John', lastName: 'Doe' },
};
const orderBy = [
{
fullName: {
firstName: OrderByDirection.AscNullsLast,
lastName: OrderByDirection.AscNullsLast,
},
},
];
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
true,
);
expect(result).toEqual([
{
fullName: {
firstName: { gt: 'John' },
lastName: { gt: 'Doe' },
},
},
]);
});
});
describe('error handling', () => {
it('should throw error for invalid field metadata', () => {
const cursor = { invalidField: 'value' };
const orderBy = [{ invalidField: OrderByDirection.AscNullsLast }];
expect(() =>
computeCursorArgFilter(cursor, orderBy, mockFieldMetadataMap, true),
).toThrow(GraphqlQueryRunnerException);
});
it('should throw error for missing orderBy entry', () => {
const cursor = { name: 'John' };
const orderBy = [{ age: OrderByDirection.AscNullsLast }];
expect(() =>
computeCursorArgFilter(cursor, orderBy, mockFieldMetadataMap, true),
).toThrow(GraphqlQueryRunnerException);
});
});
});

183
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-cursor-arg-filter.ts
View File

@ -1,183 +0,0 @@
import { FieldMetadataType } from 'twenty-shared/types';
import {
ObjectRecordFilter,
ObjectRecordOrderBy,
OrderByDirection,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
const computeOperator = (
isAscending: boolean,
isForwardPagination: boolean,
defaultOperator?: string,
): string => {
if (defaultOperator) return defaultOperator;
return isAscending
? isForwardPagination
? 'gt'
: 'lt'
: isForwardPagination
? 'lt'
: 'gt';
};
const validateAndGetOrderBy = (
key: string,
orderBy: ObjectRecordOrderBy,
): Record<string, any> => {
const keyOrderBy = orderBy.find((order) => key in order);
if (!keyOrderBy) {
throw new GraphqlQueryRunnerException(
'Invalid cursor',
GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,
);
}
return keyOrderBy;
};
const isAscendingOrder = (direction: OrderByDirection): boolean =>
direction === OrderByDirection.AscNullsFirst ||
direction === OrderByDirection.AscNullsLast;
export const computeCursorArgFilter = (
cursor: Record<string, any>,
orderBy: ObjectRecordOrderBy,
fieldMetadataMapByName: FieldMetadataMap,
isForwardPagination = true,
): ObjectRecordFilter[] => {
const cursorKeys = Object.keys(cursor ?? {});
const cursorValues = Object.values(cursor ?? {});
if (cursorKeys.length === 0) {
return [];
}
return Object.entries(cursor ?? {}).map(([key, value], index) => {
let whereCondition = {};
for (
let subConditionIndex = 0;
subConditionIndex < index;
subConditionIndex++
) {
whereCondition = {
...whereCondition,
...buildWhereCondition(
cursorKeys[subConditionIndex],
cursorValues[subConditionIndex],
fieldMetadataMapByName,
orderBy,
isForwardPagination,
'eq',
),
};
}
return {
...whereCondition,
...buildWhereCondition(
key,
value,
fieldMetadataMapByName,
orderBy,
isForwardPagination,
),
} as ObjectRecordFilter;
});
};
const buildWhereCondition = (
key: string,
value: any,
fieldMetadataMapByName: FieldMetadataMap,
orderBy: ObjectRecordOrderBy,
isForwardPagination: boolean,
operator?: string,
): Record<string, any> => {
const fieldMetadata = fieldMetadataMapByName[key];
if (!fieldMetadata) {
throw new GraphqlQueryRunnerException(
`Field metadata not found for key: ${key}`,
GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,
);
}
if (isCompositeFieldMetadataType(fieldMetadata.type)) {
return buildCompositeWhereCondition(
key,
value,
fieldMetadata.type,
orderBy,
isForwardPagination,
operator,
);
}
const keyOrderBy = validateAndGetOrderBy(key, orderBy);
const isAscending = isAscendingOrder(keyOrderBy[key]);
const computedOperator = computeOperator(
isAscending,
isForwardPagination,
operator,
);
return { [key]: { [computedOperator]: value } };
};
const buildCompositeWhereCondition = (
key: string,
value: any,
fieldType: FieldMetadataType,
orderBy: ObjectRecordOrderBy,
isForwardPagination: boolean,
operator?: string,
): Record<string, any> => {
const compositeType = compositeTypeDefinitions.get(fieldType);
if (!compositeType) {
throw new GraphqlQueryRunnerException(
`Composite type definition not found for type: ${fieldType}`,
GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,
);
}
const keyOrderBy = validateAndGetOrderBy(key, orderBy);
const result: Record<string, any> = {};
compositeType.properties.forEach((property) => {
if (
property.type === FieldMetadataType.RAW_JSON ||
value[property.name] === undefined
) {
return;
}
const isAscending = isAscendingOrder(keyOrderBy[key][property.name]);
const computedOperator = computeOperator(
isAscending,
isForwardPagination,
operator,
);
result[key] = {
...result[key],
[property.name]: {
[computedOperator]: value[property.name],
},
};
});
return result;
};