From 08f8302148ac0b03834afcb0347189f220cd1450 Mon Sep 17 00:00:00 2001
From: Etienne <45695613+etiennejouan@users.noreply.github.com>
Date: Tue, 24 Jun 2025 16:12:20 +0200
Subject: [PATCH] Import - add duplicate check on import (#12810)
Test :
- Add duplicate on id or on primaryEmail for people or primaryUrlLink on
companies
closes https://github.com/twentyhq/core-team-issues/issues/909
---
...penObjectRecordsSpreadsheetImportDialog.ts | 2 +
...spreadsheetImportGetUnicityRowHook.test.ts | 175 ++++++++++++++++++
.../spreadsheetImportGetUnicityRowHook.ts | 87 +++++++++
.../SettingsCompositeFieldTypeConfigs.ts | 30 +++
4 files changed, 294 insertions(+)
create mode 100644 packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/__tests__/spreadsheetImportGetUnicityRowHook.test.ts
create mode 100644 packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook.ts
diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts
index a62cde8ac..8270a1cb9 100644
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts
@@ -3,6 +3,7 @@ import { useBatchCreateManyRecords } from '@/object-record/hooks/useBatchCreateM
import { useBuildAvailableFieldsForImport } from '@/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport';
import { buildRecordFromImportedStructuredRow } from '@/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow';
import { spreadsheetImportFilterAvailableFieldMetadataItems } from '@/object-record/spreadsheet-import/utils/spreadsheetImportFilterAvailableFieldMetadataItems.ts';
+import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
import { SpreadsheetImportCreateRecordsBatchSize } from '@/spreadsheet-import/constants/SpreadsheetImportCreateRecordsBatchSize';
import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog';
import { spreadsheetImportCreatedRecordsProgressState } from '@/spreadsheet-import/states/spreadsheetImportCreatedRecordsProgressState';
@@ -88,6 +89,7 @@ export const useOpenObjectRecordsSpreadsheetImportDialog = (
onAbortSubmit: () => {
abortController.abort();
},
+ rowHook: spreadsheetImportGetUnicityRowHook(objectMetadataItem),
});
};
diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/__tests__/spreadsheetImportGetUnicityRowHook.test.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/__tests__/spreadsheetImportGetUnicityRowHook.test.ts
new file mode 100644
index 000000000..c7ae7a175
--- /dev/null
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/__tests__/spreadsheetImportGetUnicityRowHook.test.ts
@@ -0,0 +1,175 @@
+import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
+import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
+import { ImportedStructuredRow } from '@/spreadsheet-import/types';
+import { isDefined } from 'twenty-shared/utils';
+import { IndexType } from '~/generated-metadata/graphql';
+import { getMockCompanyObjectMetadataItem } from '~/testing/mock-data/companies';
+
+describe('spreadsheetImportGetUnicityRowHook', () => {
+ const baseMockCompany = getMockCompanyObjectMetadataItem();
+
+ const nameField = baseMockCompany.fields.find(
+ (field) => field.name === 'name',
+ );
+
+ const domainNameField = baseMockCompany.fields.find(
+ (field) => field.name === 'domainName',
+ );
+
+ const employeesField = baseMockCompany.fields.find(
+ (field) => field.name === 'employees',
+ );
+
+ if (
+ !isDefined(nameField) ||
+ !isDefined(domainNameField) ||
+ !isDefined(employeesField)
+ ) {
+ throw new Error(
+ 'Name, domainName or employees field not found in company metadata',
+ );
+ }
+
+ const mockObjectMetadataItem: ObjectMetadataItem = {
+ ...baseMockCompany,
+ indexMetadatas: [
+ {
+ id: 'unique-name-index',
+ name: 'unique_name_idx',
+ indexType: IndexType.BTREE,
+ isUnique: true,
+ createdAt: '2024-01-01T00:00:00.000Z',
+ updatedAt: '2024-01-01T00:00:00.000Z',
+ indexFieldMetadatas: [
+ {
+ id: 'index-field-2',
+ fieldMetadataId: domainNameField.id,
+ order: 0,
+ createdAt: '2024-01-01T00:00:00.000Z',
+ updatedAt: '2024-01-01T00:00:00.000Z',
+ },
+ ],
+ },
+ {
+ id: 'unique-domain-name-index',
+ name: 'unique_domain_name_idx',
+ indexType: IndexType.BTREE,
+ isUnique: true,
+ createdAt: '2024-01-01T00:00:00.000Z',
+ updatedAt: '2024-01-01T00:00:00.000Z',
+ indexFieldMetadatas: [
+ {
+ id: 'index-field-1',
+ fieldMetadataId: nameField.id,
+ order: 0,
+ createdAt: '2024-01-01T00:00:00.000Z',
+ updatedAt: '2024-01-01T00:00:00.000Z',
+ },
+ {
+ id: 'index-field-3',
+ fieldMetadataId: employeesField.id,
+ order: 1,
+ createdAt: '2024-01-01T00:00:00.000Z',
+ updatedAt: '2024-01-01T00:00:00.000Z',
+ },
+ ],
+ },
+ ],
+ };
+
+ it('should return row with error if row is not unique - index on composite field', () => {
+ const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+ const testData: ImportedStructuredRow[] = [
+ { 'Link URL (domainName)': 'duplicaTe.com', id: '1' },
+ { 'Link URL (domainName)': 'duplicate.com ', id: '2' },
+ { 'Link URL (domainName)': 'other.com', id: '3' },
+ ];
+
+ const addErrorMock = jest.fn();
+
+ const result = hook(testData[1], addErrorMock, testData);
+
+ expect(addErrorMock).toHaveBeenCalledWith('Link URL (domainName)', {
+ message:
+ 'This Link URL (domainName) value already exists in your import data',
+ level: 'error',
+ });
+ expect(result).toBe(testData[1]);
+ });
+
+ it('should return row with error if row is not unique - index on id', () => {
+ const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+ const testData: ImportedStructuredRow[] = [
+ { 'Link URL (domainName)': 'test.com', id: '1' },
+ { 'Link URL (domainName)': 'test2.com', id: '1' },
+ { 'Link URL (domainName)': 'test3.com', id: '3' },
+ ];
+
+ const addErrorMock = jest.fn();
+
+ const result = hook(testData[1], addErrorMock, testData);
+
+ expect(addErrorMock).toHaveBeenCalledWith('id', {
+ message: 'This id value already exists in your import data',
+ level: 'error',
+ });
+ expect(result).toBe(testData[1]);
+ });
+
+ it('should return row with error if row is not unique - multi fields index', () => {
+ const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+ const testData: ImportedStructuredRow[] = [
+ { name: 'test', employees: '100', id: '1' },
+ { name: 'test', employees: '100', id: '2' },
+ { name: 'test', employees: '101', id: '3' },
+ ];
+
+ const addErrorMock = jest.fn();
+
+ const result = hook(testData[1], addErrorMock, testData);
+
+ expect(addErrorMock).toHaveBeenCalledWith('name', {
+ message: 'This name value already exists in your import data',
+ level: 'error',
+ });
+ expect(addErrorMock).toHaveBeenCalledWith('employees', {
+ message: 'This employees value already exists in your import data',
+ level: 'error',
+ });
+ expect(result).toBe(testData[1]);
+ });
+ it('should not add error if row values are unique', () => {
+ const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+ const testData: ImportedStructuredRow[] = [
+ {
+ name: 'test',
+ 'Link URL (domainName)': 'test.com',
+ employees: '100',
+ id: '1',
+ },
+ {
+ name: 'test',
+ 'Link URL (domainName)': 'test2.com',
+ employees: '101',
+ id: '2',
+ },
+ {
+ name: 'test',
+ 'Link URL (domainName)': 'test3.com',
+ employees: '102',
+ id: '3',
+ },
+ ];
+
+ const addErrorMock = jest.fn();
+
+ const result = hook(testData[1], addErrorMock, testData);
+
+ expect(addErrorMock).not.toHaveBeenCalled();
+ expect(result).toBe(testData[1]);
+ });
+});
diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook.ts
new file mode 100644
index 000000000..efb9a9ef8
--- /dev/null
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook.ts
@@ -0,0 +1,87 @@
+import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
+import { isCompositeFieldType } from '@/object-record/object-filter-dropdown/utils/isCompositeFieldType';
+import { getSubFieldOptionKey } from '@/object-record/spreadsheet-import/utils/getSubFieldOptionKey';
+import { SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS } from '@/settings/data-model/constants/SettingsCompositeFieldTypeConfigs';
+import {
+ ImportedStructuredRow,
+ SpreadsheetImportRowHook,
+} from '@/spreadsheet-import/types';
+import { isDefined } from 'twenty-shared/utils';
+
+export const spreadsheetImportGetUnicityRowHook = (
+ objectMetadataItem: ObjectMetadataItem,
+) => {
+ const uniqueConstraints = objectMetadataItem.indexMetadatas.filter(
+ (indexMetadata) => indexMetadata.isUnique,
+ );
+
+ const uniqueConstraintFields = [
+ ['id'],
+ ...uniqueConstraints.map((indexMetadata) =>
+ indexMetadata.indexFieldMetadatas.flatMap((indexField) => {
+ const field = objectMetadataItem.fields.find(
+ (objectField) => objectField.id === indexField.fieldMetadataId,
+ );
+
+ if (!field) {
+ return [];
+ }
+
+ if (isCompositeFieldType(field.type)) {
+ const compositeTypeFieldConfig =
+ SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS[field.type];
+
+ const uniqueSubFields = compositeTypeFieldConfig.subFields.filter(
+ (subField) => subField.isIncludedInUniqueConstraint,
+ );
+
+ return uniqueSubFields.map((subField) =>
+ getSubFieldOptionKey(field, subField.subFieldName),
+ );
+ }
+
+ return [field.name];
+ }),
+ ),
+ ];
+
+ const rowHook: SpreadsheetImportRowHook = (row, addError, table) => {
+ if (uniqueConstraints.length === 0) {
+ return row;
+ }
+
+ uniqueConstraintFields.forEach((uniqueConstraint) => {
+ const rowUniqueValues = getUniqueValues(row, uniqueConstraint);
+
+ const duplicateRows = table.filter(
+ (r) => getUniqueValues(r, uniqueConstraint) === rowUniqueValues,
+ );
+
+ if (duplicateRows.length <= 1) {
+ return row;
+ }
+
+ uniqueConstraint.forEach((field) => {
+ if (isDefined(row[field])) {
+ addError(field, {
+ message: `This ${field} value already exists in your import data`,
+ level: 'error',
+ });
+ }
+ });
+ });
+
+ return row;
+ };
+
+ return rowHook;
+};
+
+const getUniqueValues = (
+ row: ImportedStructuredRow,
+ uniqueConstraint: string[],
+) => {
+ return uniqueConstraint
+ .map((field) => row?.[field]?.toString().trim().toLowerCase())
+ .join('');
+};
diff --git a/packages/twenty-front/src/modules/settings/data-model/constants/SettingsCompositeFieldTypeConfigs.ts b/packages/twenty-front/src/modules/settings/data-model/constants/SettingsCompositeFieldTypeConfigs.ts
index f525810c4..cffb55eac 100644
--- a/packages/twenty-front/src/modules/settings/data-model/constants/SettingsCompositeFieldTypeConfigs.ts
+++ b/packages/twenty-front/src/modules/settings/data-model/constants/SettingsCompositeFieldTypeConfigs.ts
@@ -25,11 +25,14 @@ import {
} from 'twenty-ui/display';
import { FieldMetadataType } from '~/generated-metadata/graphql';
+//TODO : isIncludedInUniqueConstraint refactor - https://github.com/twentyhq/core-team-issues/issues/1097
+
type CompositeSubFieldConfig = {
subFieldName: keyof T;
subFieldLabel: string;
isImportable: boolean;
isFilterable: boolean;
+ isIncludedInUniqueConstraint: boolean;
};
export type SettingsCompositeFieldTypeConfig = SettingsFieldTypeConfig & {
@@ -54,6 +57,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.amountMicros,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'currencyCode',
@@ -62,6 +66,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.currencyCode,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@@ -91,6 +96,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryEmail,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'additionalEmails',
@@ -99,6 +105,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.additionalEmails,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@@ -132,6 +139,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryLinkUrl,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'primaryLinkLabel',
@@ -140,6 +148,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryLinkLabel,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'secondaryLinks',
@@ -148,6 +157,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.secondaryLinks,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@@ -180,6 +190,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryPhoneCallingCode,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'primaryPhoneCountryCode',
@@ -188,6 +199,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryPhoneCountryCode,
isImportable: true,
isFilterable: false,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'primaryPhoneNumber',
@@ -196,6 +208,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.primaryPhoneNumber,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'additionalPhones',
@@ -204,6 +217,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.additionalPhones,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@@ -244,6 +258,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.firstName,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: true,
},
{
subFieldName: 'lastName',
@@ -252,6 +267,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.lastName,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: true,
},
],
exampleValues: [
@@ -272,6 +288,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressStreet1,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressStreet2',
@@ -280,6 +297,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressStreet2,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressCity',
@@ -288,6 +306,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressCity,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressState',
@@ -296,6 +315,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressState,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressCountry',
@@ -304,6 +324,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressCountry,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressPostcode',
@@ -312,6 +333,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressPostcode,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressLat',
@@ -320,6 +342,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressLat,
isImportable: false,
isFilterable: false,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'addressLng',
@@ -328,6 +351,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.addressLng,
isImportable: false,
isFilterable: false,
+ isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@@ -375,6 +399,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].source,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'name',
@@ -382,6 +407,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].name,
isImportable: true,
isFilterable: true,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'workspaceMemberId',
@@ -390,6 +416,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.workspaceMemberId,
isImportable: true,
isFilterable: false,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'context',
@@ -397,6 +424,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].context,
isImportable: true,
isFilterable: false,
+ isIncludedInUniqueConstraint: false,
},
],
exampleValues: [
@@ -432,6 +460,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.blocknote,
isImportable: false,
isFilterable: false,
+ isIncludedInUniqueConstraint: false,
},
{
subFieldName: 'markdown',
@@ -440,6 +469,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
.markdown,
isImportable: false,
isFilterable: false,
+ isIncludedInUniqueConstraint: false,
},
],
exampleValues: [