admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Import - add duplicate check on import (#12810)

Browse Source 
<img width="1217" alt="Screenshot 2025-06-24 at 11 43 03"
src="https://github.com/user-attachments/assets/de2bc12e-5e25-484f-a034-f52b0f237a3e"
/>

Test : 
- Add duplicate on id or on primaryEmail for people or primaryUrlLink on
companies

closes https://github.com/twentyhq/core-team-issues/issues/909
This commit is contained in:
Etienne
2025-06-24 16:12:20 +02:00
committed by GitHub
parent cc489f971d
commit 08f8302148
4 changed files with 294 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts
View File

@ -3,6 +3,7 @@ import { useBatchCreateManyRecords } from '@/object-record/hooks/useBatchCreateM
import { useBuildAvailableFieldsForImport } from '@/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport';
import { buildRecordFromImportedStructuredRow } from '@/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow';
import { spreadsheetImportFilterAvailableFieldMetadataItems } from '@/object-record/spreadsheet-import/utils/spreadsheetImportFilterAvailableFieldMetadataItems.ts';
import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
import { SpreadsheetImportCreateRecordsBatchSize } from '@/spreadsheet-import/constants/SpreadsheetImportCreateRecordsBatchSize';
import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog';
import { spreadsheetImportCreatedRecordsProgressState } from '@/spreadsheet-import/states/spreadsheetImportCreatedRecordsProgressState';
@ -88,6 +89,7 @@ export const useOpenObjectRecordsSpreadsheetImportDialog = (
onAbortSubmit: () => {
abortController.abort();
},
rowHook: spreadsheetImportGetUnicityRowHook(objectMetadataItem),
});
};

175
packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/__tests__/spreadsheetImportGetUnicityRowHook.test.ts Normal file
View File

@ -0,0 +1,175 @@
import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
import { ImportedStructuredRow } from '@/spreadsheet-import/types';
import { isDefined } from 'twenty-shared/utils';
import { IndexType } from '~/generated-metadata/graphql';
import { getMockCompanyObjectMetadataItem } from '~/testing/mock-data/companies';
describe('spreadsheetImportGetUnicityRowHook', () => {
const baseMockCompany = getMockCompanyObjectMetadataItem();
const nameField = baseMockCompany.fields.find(
(field) => field.name === 'name',
);
const domainNameField = baseMockCompany.fields.find(
(field) => field.name === 'domainName',
);
const employeesField = baseMockCompany.fields.find(
(field) => field.name === 'employees',
);
if (
!isDefined(nameField) ||
!isDefined(domainNameField) ||
!isDefined(employeesField)
) {
throw new Error(
'Name, domainName or employees field not found in company metadata',
);
}
const mockObjectMetadataItem: ObjectMetadataItem = {
...baseMockCompany,
indexMetadatas: [
{
id: 'unique-name-index',
name: 'unique_name_idx',
indexType: IndexType.BTREE,
isUnique: true,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
indexFieldMetadatas: [
{
id: 'index-field-2',
fieldMetadataId: domainNameField.id,
order: 0,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
],
},
{
id: 'unique-domain-name-index',
name: 'unique_domain_name_idx',
indexType: IndexType.BTREE,
isUnique: true,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
indexFieldMetadatas: [
{
id: 'index-field-1',
fieldMetadataId: nameField.id,
order: 0,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
{
id: 'index-field-3',
fieldMetadataId: employeesField.id,
order: 1,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
],
},
],
};
it('should return row with error if row is not unique - index on composite field', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{ 'Link URL (domainName)': 'duplicaTe.com', id: '1' },
{ 'Link URL (domainName)': 'duplicate.com ', id: '2' },
{ 'Link URL (domainName)': 'other.com', id: '3' },
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).toHaveBeenCalledWith('Link URL (domainName)', {
message:
'This Link URL (domainName) value already exists in your import data',
level: 'error',
});
expect(result).toBe(testData[1]);
});
it('should return row with error if row is not unique - index on id', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{ 'Link URL (domainName)': 'test.com', id: '1' },
{ 'Link URL (domainName)': 'test2.com', id: '1' },
{ 'Link URL (domainName)': 'test3.com', id: '3' },
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).toHaveBeenCalledWith('id', {
message: 'This id value already exists in your import data',
level: 'error',
});
expect(result).toBe(testData[1]);
});
it('should return row with error if row is not unique - multi fields index', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{ name: 'test', employees: '100', id: '1' },
{ name: 'test', employees: '100', id: '2' },
{ name: 'test', employees: '101', id: '3' },
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).toHaveBeenCalledWith('name', {
message: 'This name value already exists in your import data',
level: 'error',
});
expect(addErrorMock).toHaveBeenCalledWith('employees', {
message: 'This employees value already exists in your import data',
level: 'error',
});
expect(result).toBe(testData[1]);
});
it('should not add error if row values are unique', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{
name: 'test',
'Link URL (domainName)': 'test.com',
employees: '100',
id: '1',
},
{
name: 'test',
'Link URL (domainName)': 'test2.com',
employees: '101',
id: '2',
},
{
name: 'test',
'Link URL (domainName)': 'test3.com',
employees: '102',
id: '3',
},
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).not.toHaveBeenCalled();
expect(result).toBe(testData[1]);
});
});

87
packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook.ts Normal file
View File

@ -0,0 +1,87 @@
import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
import { isCompositeFieldType } from '@/object-record/object-filter-dropdown/utils/isCompositeFieldType';
import { getSubFieldOptionKey } from '@/object-record/spreadsheet-import/utils/getSubFieldOptionKey';
import { SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS } from '@/settings/data-model/constants/SettingsCompositeFieldTypeConfigs';
import {
ImportedStructuredRow,
SpreadsheetImportRowHook,
} from '@/spreadsheet-import/types';
import { isDefined } from 'twenty-shared/utils';
export const spreadsheetImportGetUnicityRowHook = (
objectMetadataItem: ObjectMetadataItem,
) => {
const uniqueConstraints = objectMetadataItem.indexMetadatas.filter(
(indexMetadata) => indexMetadata.isUnique,
);
const uniqueConstraintFields = [
['id'],
...uniqueConstraints.map((indexMetadata) =>
indexMetadata.indexFieldMetadatas.flatMap((indexField) => {
const field = objectMetadataItem.fields.find(
(objectField) => objectField.id === indexField.fieldMetadataId,
);
if (!field) {
return [];
}
if (isCompositeFieldType(field.type)) {
const compositeTypeFieldConfig =
SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS[field.type];
const uniqueSubFields = compositeTypeFieldConfig.subFields.filter(
(subField) => subField.isIncludedInUniqueConstraint,
);
return uniqueSubFields.map((subField) =>
getSubFieldOptionKey(field, subField.subFieldName),
);
}
return [field.name];
}),
),
];
const rowHook: SpreadsheetImportRowHook<string> = (row, addError, table) => {
if (uniqueConstraints.length === 0) {
return row;
}
uniqueConstraintFields.forEach((uniqueConstraint) => {
const rowUniqueValues = getUniqueValues(row, uniqueConstraint);
const duplicateRows = table.filter(
(r) => getUniqueValues(r, uniqueConstraint) === rowUniqueValues,
);
if (duplicateRows.length <= 1) {
return row;
}
uniqueConstraint.forEach((field) => {
if (isDefined(row[field])) {
addError(field, {
message: `This ${field} value already exists in your import data`,
level: 'error',
});
}
});
});
return row;
};
return rowHook;
};
const getUniqueValues = (
row: ImportedStructuredRow<string>,
uniqueConstraint: string[],
) => {
return uniqueConstraint
.map((field) => row?.[field]?.toString().trim().toLowerCase())
.join('');
};

30
packages/twenty-front/src/modules/settings/data-model/constants/SettingsCompositeFieldTypeConfigs.ts
View File

@ -25,11 +25,14 @@ import {
} from 'twenty-ui/display';
import { FieldMetadataType } from '~/generated-metadata/graphql';
//TODO : isIncludedInUniqueConstraint refactor - https://github.com/twentyhq/core-team-issues/issues/1097
type CompositeSubFieldConfig<T> = {
subFieldName: keyof T;
subFieldLabel: string;
isImportable: boolean;
isFilterable: boolean;
isIncludedInUniqueConstraint: boolean;
};
export type SettingsCompositeFieldTypeConfig<T> = SettingsFieldTypeConfig<T> & {
@ -54,6 +57,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.amountMicros,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'currencyCode',
@ -62,6 +66,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.currencyCode,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@ -91,6 +96,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryEmail,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'additionalEmails',
@ -99,6 +105,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.additionalEmails,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@ -132,6 +139,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryLinkUrl,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'primaryLinkLabel',
@ -140,6 +148,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryLinkLabel,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'secondaryLinks',
@ -148,6 +157,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.secondaryLinks,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@ -180,6 +190,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryPhoneCallingCode,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'primaryPhoneCountryCode',
@ -188,6 +199,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryPhoneCountryCode,
isImportable: true,
isFilterable: false,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'primaryPhoneNumber',
@ -196,6 +208,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryPhoneNumber,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'additionalPhones',
@ -204,6 +217,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.additionalPhones,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@ -244,6 +258,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.firstName,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'lastName',
@ -252,6 +267,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.lastName,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: true,
},
],
exampleValues: [
@ -272,6 +288,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressStreet1,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressStreet2',
@ -280,6 +297,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressStreet2,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressCity',
@ -288,6 +306,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressCity,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressState',
@ -296,6 +315,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressState,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressCountry',
@ -304,6 +324,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressCountry,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressPostcode',
@ -312,6 +333,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressPostcode,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressLat',
@ -320,6 +342,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressLat,
isImportable: false,
isFilterable: false,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressLng',
@ -328,6 +351,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressLng,
isImportable: false,
isFilterable: false,
isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@ -375,6 +399,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].source,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'name',
@ -382,6 +407,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].name,
isImportable: true,
isFilterable: true,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'workspaceMemberId',
@ -390,6 +416,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.workspaceMemberId,
isImportable: true,
isFilterable: false,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'context',
@ -397,6 +424,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].context,
isImportable: true,
isFilterable: false,
isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@ -432,6 +460,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.blocknote,
isImportable: false,
isFilterable: false,
isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'markdown',
@ -440,6 +469,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.markdown,
isImportable: false,
isFilterable: false,
isIncludedInUniqueConstraint: false,
},
],
exampleValues: [