Import - add duplicate check on import (#12810)

<img width="1217" alt="Screenshot 2025-06-24 at 11 43 03"
src="https://github.com/user-attachments/assets/de2bc12e-5e25-484f-a034-f52b0f237a3e"
/>

Test : 
- Add duplicate on id or on primaryEmail for people or primaryUrlLink on
companies

closes https://github.com/twentyhq/core-team-issues/issues/909

packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts

@@ -3,6 +3,7 @@ import { useBatchCreateManyRecords } from '@/object-record/hooks/useBatchCreateM
 import { useBuildAvailableFieldsForImport } from '@/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport';
 import { buildRecordFromImportedStructuredRow } from '@/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow';
 import { spreadsheetImportFilterAvailableFieldMetadataItems } from '@/object-record/spreadsheet-import/utils/spreadsheetImportFilterAvailableFieldMetadataItems.ts';
+import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
 import { SpreadsheetImportCreateRecordsBatchSize } from '@/spreadsheet-import/constants/SpreadsheetImportCreateRecordsBatchSize';
 import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog';
 import { spreadsheetImportCreatedRecordsProgressState } from '@/spreadsheet-import/states/spreadsheetImportCreatedRecordsProgressState';
@@ -88,6 +89,7 @@ export const useOpenObjectRecordsSpreadsheetImportDialog = (
       onAbortSubmit: () => {
         abortController.abort();
       },
+      rowHook: spreadsheetImportGetUnicityRowHook(objectMetadataItem),
     });
   };
 

packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/__tests__/spreadsheetImportGetUnicityRowHook.test.ts

@@ -0,0 +1,175 @@
+import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
+import { spreadsheetImportGetUnicityRowHook } from '@/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook';
+import { ImportedStructuredRow } from '@/spreadsheet-import/types';
+import { isDefined } from 'twenty-shared/utils';
+import { IndexType } from '~/generated-metadata/graphql';
+import { getMockCompanyObjectMetadataItem } from '~/testing/mock-data/companies';
+
+describe('spreadsheetImportGetUnicityRowHook', () => {
+  const baseMockCompany = getMockCompanyObjectMetadataItem();
+
+  const nameField = baseMockCompany.fields.find(
+    (field) => field.name === 'name',
+  );
+
+  const domainNameField = baseMockCompany.fields.find(
+    (field) => field.name === 'domainName',
+  );
+
+  const employeesField = baseMockCompany.fields.find(
+    (field) => field.name === 'employees',
+  );
+
+  if (
+    !isDefined(nameField) ||
+    !isDefined(domainNameField) ||
+    !isDefined(employeesField)
+  ) {
+    throw new Error(
+      'Name, domainName or employees field not found in company metadata',
+    );
+  }
+
+  const mockObjectMetadataItem: ObjectMetadataItem = {
+    ...baseMockCompany,
+    indexMetadatas: [
+      {
+        id: 'unique-name-index',
+        name: 'unique_name_idx',
+        indexType: IndexType.BTREE,
+        isUnique: true,
+        createdAt: '2024-01-01T00:00:00.000Z',
+        updatedAt: '2024-01-01T00:00:00.000Z',
+        indexFieldMetadatas: [
+          {
+            id: 'index-field-2',
+            fieldMetadataId: domainNameField.id,
+            order: 0,
+            createdAt: '2024-01-01T00:00:00.000Z',
+            updatedAt: '2024-01-01T00:00:00.000Z',
+          },
+        ],
+      },
+      {
+        id: 'unique-domain-name-index',
+        name: 'unique_domain_name_idx',
+        indexType: IndexType.BTREE,
+        isUnique: true,
+        createdAt: '2024-01-01T00:00:00.000Z',
+        updatedAt: '2024-01-01T00:00:00.000Z',
+        indexFieldMetadatas: [
+          {
+            id: 'index-field-1',
+            fieldMetadataId: nameField.id,
+            order: 0,
+            createdAt: '2024-01-01T00:00:00.000Z',
+            updatedAt: '2024-01-01T00:00:00.000Z',
+          },
+          {
+            id: 'index-field-3',
+            fieldMetadataId: employeesField.id,
+            order: 1,
+            createdAt: '2024-01-01T00:00:00.000Z',
+            updatedAt: '2024-01-01T00:00:00.000Z',
+          },
+        ],
+      },
+    ],
+  };
+
+  it('should return row with error if row is not unique - index on composite field', () => {
+    const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+    const testData: ImportedStructuredRow<string>[] = [
+      { 'Link URL (domainName)': 'duplicaTe.com', id: '1' },
+      { 'Link URL (domainName)': 'duplicate.com ', id: '2' },
+      { 'Link URL (domainName)': 'other.com', id: '3' },
+    ];
+
+    const addErrorMock = jest.fn();
+
+    const result = hook(testData[1], addErrorMock, testData);
+
+    expect(addErrorMock).toHaveBeenCalledWith('Link URL (domainName)', {
+      message:
+        'This Link URL (domainName) value already exists in your import data',
+      level: 'error',
+    });
+    expect(result).toBe(testData[1]);
+  });
+
+  it('should return row with error if row is not unique - index on id', () => {
+    const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+    const testData: ImportedStructuredRow<string>[] = [
+      { 'Link URL (domainName)': 'test.com', id: '1' },
+      { 'Link URL (domainName)': 'test2.com', id: '1' },
+      { 'Link URL (domainName)': 'test3.com', id: '3' },
+    ];
+
+    const addErrorMock = jest.fn();
+
+    const result = hook(testData[1], addErrorMock, testData);
+
+    expect(addErrorMock).toHaveBeenCalledWith('id', {
+      message: 'This id value already exists in your import data',
+      level: 'error',
+    });
+    expect(result).toBe(testData[1]);
+  });
+
+  it('should return row with error if row is not unique - multi fields index', () => {
+    const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+    const testData: ImportedStructuredRow<string>[] = [
+      { name: 'test', employees: '100', id: '1' },
+      { name: 'test', employees: '100', id: '2' },
+      { name: 'test', employees: '101', id: '3' },
+    ];
+
+    const addErrorMock = jest.fn();
+
+    const result = hook(testData[1], addErrorMock, testData);
+
+    expect(addErrorMock).toHaveBeenCalledWith('name', {
+      message: 'This name value already exists in your import data',
+      level: 'error',
+    });
+    expect(addErrorMock).toHaveBeenCalledWith('employees', {
+      message: 'This employees value already exists in your import data',
+      level: 'error',
+    });
+    expect(result).toBe(testData[1]);
+  });
+  it('should not add error if row values are unique', () => {
+    const hook = spreadsheetImportGetUnicityRowHook(mockObjectMetadataItem);
+
+    const testData: ImportedStructuredRow<string>[] = [
+      {
+        name: 'test',
+        'Link URL (domainName)': 'test.com',
+        employees: '100',
+        id: '1',
+      },
+      {
+        name: 'test',
+        'Link URL (domainName)': 'test2.com',
+        employees: '101',
+        id: '2',
+      },
+      {
+        name: 'test',
+        'Link URL (domainName)': 'test3.com',
+        employees: '102',
+        id: '3',
+      },
+    ];
+
+    const addErrorMock = jest.fn();
+
+    const result = hook(testData[1], addErrorMock, testData);
+
+    expect(addErrorMock).not.toHaveBeenCalled();
+    expect(result).toBe(testData[1]);
+  });
+});

packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/spreadsheetImportGetUnicityRowHook.ts

@@ -0,0 +1,87 @@
+import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
+import { isCompositeFieldType } from '@/object-record/object-filter-dropdown/utils/isCompositeFieldType';
+import { getSubFieldOptionKey } from '@/object-record/spreadsheet-import/utils/getSubFieldOptionKey';
+import { SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS } from '@/settings/data-model/constants/SettingsCompositeFieldTypeConfigs';
+import {
+  ImportedStructuredRow,
+  SpreadsheetImportRowHook,
+} from '@/spreadsheet-import/types';
+import { isDefined } from 'twenty-shared/utils';
+
+export const spreadsheetImportGetUnicityRowHook = (
+  objectMetadataItem: ObjectMetadataItem,
+) => {
+  const uniqueConstraints = objectMetadataItem.indexMetadatas.filter(
+    (indexMetadata) => indexMetadata.isUnique,
+  );
+
+  const uniqueConstraintFields = [
+    ['id'],
+    ...uniqueConstraints.map((indexMetadata) =>
+      indexMetadata.indexFieldMetadatas.flatMap((indexField) => {
+        const field = objectMetadataItem.fields.find(
+          (objectField) => objectField.id === indexField.fieldMetadataId,
+        );
+
+        if (!field) {
+          return [];
+        }
+
+        if (isCompositeFieldType(field.type)) {
+          const compositeTypeFieldConfig =
+            SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS[field.type];
+
+          const uniqueSubFields = compositeTypeFieldConfig.subFields.filter(
+            (subField) => subField.isIncludedInUniqueConstraint,
+          );
+
+          return uniqueSubFields.map((subField) =>
+            getSubFieldOptionKey(field, subField.subFieldName),
+          );
+        }
+
+        return [field.name];
+      }),
+    ),
+  ];
+
+  const rowHook: SpreadsheetImportRowHook<string> = (row, addError, table) => {
+    if (uniqueConstraints.length === 0) {
+      return row;
+    }
+
+    uniqueConstraintFields.forEach((uniqueConstraint) => {
+      const rowUniqueValues = getUniqueValues(row, uniqueConstraint);
+
+      const duplicateRows = table.filter(
+        (r) => getUniqueValues(r, uniqueConstraint) === rowUniqueValues,
+      );
+
+      if (duplicateRows.length <= 1) {
+        return row;
+      }
+
+      uniqueConstraint.forEach((field) => {
+        if (isDefined(row[field])) {
+          addError(field, {
+            message: `This ${field} value already exists in your import data`,
+            level: 'error',
+          });
+        }
+      });
+    });
+
+    return row;
+  };
+
+  return rowHook;
+};
+
+const getUniqueValues = (
+  row: ImportedStructuredRow<string>,
+  uniqueConstraint: string[],
+) => {
+  return uniqueConstraint
+    .map((field) => row?.[field]?.toString().trim().toLowerCase())
+    .join('');
+};

packages/twenty-front/src/modules/settings/data-model/constants/SettingsCompositeFieldTypeConfigs.ts

@@ -25,11 +25,14 @@ import {
 } from 'twenty-ui/display';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
 
+//TODO : isIncludedInUniqueConstraint refactor - https://github.com/twentyhq/core-team-issues/issues/1097
+
 type CompositeSubFieldConfig<T> = {
   subFieldName: keyof T;
   subFieldLabel: string;
   isImportable: boolean;
   isFilterable: boolean;
+  isIncludedInUniqueConstraint: boolean;
 };
 
 export type SettingsCompositeFieldTypeConfig<T> = SettingsFieldTypeConfig<T> & {
@@ -54,6 +57,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .amountMicros,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'currencyCode',
@@ -62,6 +66,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .currencyCode,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
     ],
     exampleValues: [
@@ -91,6 +96,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .primaryEmail,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: true,
       },
       {
         subFieldName: 'additionalEmails',
@@ -99,6 +105,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .additionalEmails,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
     ],
     exampleValues: [
@@ -132,6 +139,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .primaryLinkUrl,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: true,
       },
       {
         subFieldName: 'primaryLinkLabel',
@@ -140,6 +148,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .primaryLinkLabel,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'secondaryLinks',
@@ -148,6 +157,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .secondaryLinks,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
     ],
     exampleValues: [
@@ -180,6 +190,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .primaryPhoneCallingCode,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'primaryPhoneCountryCode',
@@ -188,6 +199,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .primaryPhoneCountryCode,
         isImportable: true,
         isFilterable: false,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'primaryPhoneNumber',
@@ -196,6 +208,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .primaryPhoneNumber,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: true,
       },
       {
         subFieldName: 'additionalPhones',
@@ -204,6 +217,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .additionalPhones,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
     ],
     exampleValues: [
@@ -244,6 +258,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .firstName,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: true,
       },
       {
         subFieldName: 'lastName',
@@ -252,6 +267,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .lastName,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: true,
       },
     ],
     exampleValues: [
@@ -272,6 +288,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressStreet1,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'addressStreet2',
@@ -280,6 +297,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressStreet2,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'addressCity',
@@ -288,6 +306,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressCity,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'addressState',
@@ -296,6 +315,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressState,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'addressCountry',
@@ -304,6 +324,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressCountry,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'addressPostcode',
@@ -312,6 +333,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressPostcode,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'addressLat',
@@ -320,6 +342,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressLat,
         isImportable: false,
         isFilterable: false,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'addressLng',
@@ -328,6 +351,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .addressLng,
         isImportable: false,
         isFilterable: false,
+        isIncludedInUniqueConstraint: false,
       },
     ],
     exampleValues: [
@@ -375,6 +399,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
           COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].source,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'name',
@@ -382,6 +407,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
           COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].name,
         isImportable: true,
         isFilterable: true,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'workspaceMemberId',
@@ -390,6 +416,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .workspaceMemberId,
         isImportable: true,
         isFilterable: false,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'context',
@@ -397,6 +424,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
           COMPOSITE_FIELD_SUB_FIELD_LABELS[FieldMetadataType.ACTOR].context,
         isImportable: true,
         isFilterable: false,
+        isIncludedInUniqueConstraint: false,
       },
     ],
     exampleValues: [
@@ -432,6 +460,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .blocknote,
         isImportable: false,
         isFilterable: false,
+        isIncludedInUniqueConstraint: false,
       },
       {
         subFieldName: 'markdown',
@@ -440,6 +469,7 @@ export const SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS = {
             .markdown,
         isImportable: false,
         isFilterable: false,
+        isIncludedInUniqueConstraint: false,
       },
     ],
     exampleValues: [