admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Import - add duplicate check on import (#12810)

Browse Source 
<img width="1217" alt="Screenshot 2025-06-24 at 11 43 03"
src="https://github.com/user-attachments/assets/de2bc12e-5e25-484f-a034-f52b0f237a3e"
/>

Test : 
- Add duplicate on id or on primaryEmail for people or primaryUrlLink on
companies

closes https://github.com/twentyhq/core-team-issues/issues/909
This commit is contained in:
Etienne
2025-06-24 16:12:20 +02:00
committed by GitHub
parent cc489f971d
commit 08f8302148
4 changed files with 294 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts
View File

@ -3,6 +3,7 @@ import { useBatchCreateManyRecords } from '@/object-record/hooks/useBatchCreateM
import { useBuildAvailableFieldsForImport } from '@/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport';
import { buildRecordFromImportedStructuredRow } from '@/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow';
import { spreadsheetImportFilterAvailableFieldMetadataItems } from '@/object-record/spreadsheet-import/utils/spreadsheetImportFilterAvailableFieldMetadataItems.ts';
import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
import { SpreadsheetImportCreateRecordsBatchSize } from '@/spreadsheet-import/constants/SpreadsheetImportCreateRecordsBatchSize';
import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog';
import { spreadsheetImportCreatedRecordsProgressState } from '@/spreadsheet-import/states/spreadsheetImportCreatedRecordsProgressState';
@ -88,6 +89,7 @@ export const useOpenObjectRecordsSpreadsheetImportDialog = (
onAbortSubmit: () => {
abortController.abort();
},
rowHook: spreadsheetImportGetUnicityRowHook(objectMetadataItem),
});
};

175
packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/__tests__/spreadsheetImportGetUnicityRowHook.test.ts Normal file
View File

@ -0,0 +1,175 @@
import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
import { ImportedStructuredRow } from '@/spreadsheet-import/types';
import { isDefined } from 'twenty-shared/utils';
import { IndexType } from '~/generated-metadata/graphql';
import { getMockCompanyObjectMetadataItem } from '~/testing/mock-data/companies';
describe('spreadsheetImportGetUnicityRowHook', () => {
const baseMockCompany = getMockCompanyObjectMetadataItem();
const nameField = baseMockCompany.fields.find(
(field) => field.name === 'name',
);
const domainNameField = baseMockCompany.fields.find(
(field) => field.name === 'domainName',
);
const employeesField = baseMockCompany.fields.find(
(field) => field.name === 'employees',
);
if (
!isDefined(nameField) ||
!isDefined(domainNameField) ||
!isDefined(employeesField)
) {
throw new Error(
'Name, domainName or employees field not found in company metadata',
);
}
const mockObjectMetadataItem: ObjectMetadataItem = {
...baseMockCompany,
indexMetadatas: [
{
id: 'unique-name-index',
name: 'unique_name_idx',
indexType: IndexType.BTREE,
isUnique: true,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
indexFieldMetadatas: [
{
id: 'index-field-2',
fieldMetadataId: domainNameField.id,
order: 0,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
],
},
{
id: 'unique-domain-name-index',
name: 'unique_domain_name_idx',
indexType: IndexType.BTREE,
isUnique: true,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
indexFieldMetadatas: [
{
id: 'index-field-1',
fieldMetadataId: nameField.id,
order: 0,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
{
id: 'index-field-3',
fieldMetadataId: employeesField.id,
order: 1,
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
],
},
],
};
it('should return row with error if row is not unique - index on composite field', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{ 'Link URL (domainName)': 'duplicaTe.com', id: '1' },
{ 'Link URL (domainName)': 'duplicate.com ', id: '2' },
{ 'Link URL (domainName)': 'other.com', id: '3' },
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).toHaveBeenCalledWith('Link URL (domainName)', {
message:
'This Link URL (domainName) value already exists in your import data',
level: 'error',
});
expect(result).toBe(testData[1]);
});
it('should return row with error if row is not unique - index on id', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{ 'Link URL (domainName)': 'test.com', id: '1' },
{ 'Link URL (domainName)': 'test2.com', id: '1' },
{ 'Link URL (domainName)': 'test3.com', id: '3' },
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).toHaveBeenCalledWith('id', {
message: 'This id value already exists in your import data',
level: 'error',
});
expect(result).toBe(testData[1]);
});
it('should return row with error if row is not unique - multi fields index', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{ name: 'test', employees: '100', id: '1' },
{ name: 'test', employees: '100', id: '2' },
{ name: 'test', employees: '101', id: '3' },
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).toHaveBeenCalledWith('name', {
message: 'This name value already exists in your import data',
level: 'error',
});
expect(addErrorMock).toHaveBeenCalledWith('employees', {
message: 'This employees value already exists in your import data',
level: 'error',
});
expect(result).toBe(testData[1]);
});
it('should not add error if row values are unique', () => {
const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
const testData: ImportedStructuredRow<string>[] = [
{
name: 'test',
'Link URL (domainName)': 'test.com',
employees: '100',
id: '1',
},
{
name: 'test',
'Link URL (domainName)': 'test2.com',
employees: '101',
id: '2',
},
{
name: 'test',
'Link URL (domainName)': 'test3.com',
employees: '102',
id: '3',
},
];
const addErrorMock = jest.fn();
const result = hook(testData[1], addErrorMock, testData);
expect(addErrorMock).not.toHaveBeenCalled();
expect(result).toBe(testData[1]);
});
});

87
packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook.ts Normal file
View File

@ -0,0 +1,87 @@
import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
import { isCompositeFieldType } from '@/object-record/object-filter-dropdown/utils/isCompositeFieldType';
import { getSubFieldOptionKey } from '@/object-record/spreadsheet-import/utils/getSubFieldOptionKey';
import { SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS } from '@/settings/data-model/constants/SettingsCompositeFieldTypeConfigs';
import {
ImportedStructuredRow,
SpreadsheetImportRowHook,
} from '@/spreadsheet-import/types';
import { isDefined } from 'twenty-shared/utils';
export const spreadsheetImportGetUnicityRowHook = (
objectMetadataItem: ObjectMetadataItem,
) => {
const uniqueConstraints = objectMetadataItem.indexMetadatas.filter(
(indexMetadata) => indexMetadata.isUnique,
);
const uniqueConstraintFields = [
['id'],
...uniqueConstraints.map((indexMetadata) =>
indexMetadata.indexFieldMetadatas.flatMap((indexField) => {
const field = objectMetadataItem.fields.find(
(objectField) => objectField.id === indexField.fieldMetadataId,
);
if (!field) {
return [];
}
if (isCompositeFieldType(field.type)) {
const compositeTypeFieldConfig =
SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS[field.type];
const uniqueSubFields = compositeTypeFieldConfig.subFields.filter(
(subField) => subField.isIncludedInUniqueConstraint,
);
return uniqueSubFields.map((subField) =>
getSubFieldOptionKey(field, subField.subFieldName),
);
}
return [field.name];
}),
),
];
const rowHook: SpreadsheetImportRowHook<string> = (row, addError, table) => {
if (uniqueConstraints.length === 0) {
return row;
}
uniqueConstraintFields.forEach((uniqueConstraint) => {
const rowUniqueValues = getUniqueValues(row, uniqueConstraint);
const duplicateRows = table.filter(
(r) => getUniqueValues(r, uniqueConstraint) === rowUniqueValues,
);
if (duplicateRows.length <= 1) {
return row;
}
uniqueConstraint.forEach((field) => {
if (isDefined(row[field])) {
addError(field, {
message: `This ${field} value already exists in your import data`,
level: 'error',
});
}
});
});
return row;
};
return rowHook;
};
const getUniqueValues = (
row: ImportedStructuredRow<string>,
uniqueConstraint: string[],
) => {
return uniqueConstraint
.map((field) => row?.[field]?.toString().trim().toLowerCase())
.join('');
};