Import - Increase record import limit (#12627)

<img width="700" alt="Screenshot 2025-06-16 at 15 05 09"
src="https://github.com/user-attachments/assets/a09c3fae-c0ae-4a63-8bda-9d29c97a6a66"
/>


closes https://github.com/twentyhq/twenty/issues/11980

packages/twenty-front/src/modules/object-record/hooks/__tests__/useCreateManyRecords.test.tsx

@@ -76,7 +76,9 @@ describe('useCreateManyRecords', () => {
     );
 
     await act(async () => {
-      const res = await result.current.createManyRecords(input);
+      const res = await result.current.createManyRecords({
+        recordsToCreate: input,
+      });
       expect(res).toEqual(response);
     });
 
@@ -96,7 +98,10 @@ describe('useCreateManyRecords', () => {
     );
 
     await act(async () => {
-      const res = await result.current.createManyRecords(input, true);
+      const res = await result.current.createManyRecords({
+        recordsToCreate: input,
+        upsert: true,
+      });
       expect(res).toEqual(response);
     });
 

packages/twenty-front/src/modules/object-record/hooks/useBatchCreateManyRecords.ts

@@ -0,0 +1,106 @@
+import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
+import { DEFAULT_MUTATION_BATCH_SIZE } from '@/object-record/constants/DefaultMutationBatchSize';
+import {
+  useCreateManyRecords,
+  useCreateManyRecordsProps,
+} from '@/object-record/hooks/useCreateManyRecords';
+import { useRefetchAggregateQueries } from '@/object-record/hooks/useRefetchAggregateQueries';
+import { ObjectRecord } from '@/object-record/types/ObjectRecord';
+import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar';
+import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
+import { ApolloError } from '@apollo/client';
+import { t } from '@lingui/core/macro';
+import { formatNumber } from '~/utils/format/number';
+
+export const useBatchCreateManyRecords = <
+  CreatedObjectRecord extends ObjectRecord = ObjectRecord,
+>({
+  objectNameSingular,
+  recordGqlFields,
+  skipPostOptimisticEffect = false,
+  shouldMatchRootQueryFilter,
+  mutationBatchSize = DEFAULT_MUTATION_BATCH_SIZE,
+  setBatchedRecordsCount,
+  abortController,
+}: useCreateManyRecordsProps & {
+  mutationBatchSize?: number;
+  setBatchedRecordsCount?: (count: number) => void;
+  abortController?: AbortController;
+}) => {
+  const { createManyRecords } = useCreateManyRecords({
+    objectNameSingular,
+    recordGqlFields,
+    skipPostOptimisticEffect,
+    shouldMatchRootQueryFilter,
+    shouldRefetchAggregateQueries: false,
+  });
+
+  const { objectMetadataItem } = useObjectMetadataItem({
+    objectNameSingular,
+  });
+
+  const { refetchAggregateQueries } = useRefetchAggregateQueries({
+    objectMetadataNamePlural: objectMetadataItem.namePlural,
+  });
+
+  const { enqueueSnackBar } = useSnackBar();
+
+  const batchCreateManyRecords = async ({
+    recordsToCreate,
+    upsert,
+  }: {
+    recordsToCreate: Partial<CreatedObjectRecord>[];
+    upsert?: boolean;
+  }) => {
+    const numberOfBatches = Math.ceil(
+      recordsToCreate.length / mutationBatchSize,
+    );
+
+    setBatchedRecordsCount?.(0);
+
+    const allCreatedRecords = [];
+    let createdRecordsCount = 0;
+    try {
+      for (let batchIndex = 0; batchIndex < numberOfBatches; batchIndex++) {
+        const batchedRecordsToCreate = recordsToCreate.slice(
+          batchIndex * mutationBatchSize,
+          (batchIndex + 1) * mutationBatchSize,
+        );
+
+        createdRecordsCount =
+          batchIndex + 1 === numberOfBatches
+            ? recordsToCreate.length
+            : (batchIndex + 1) * mutationBatchSize;
+
+        const createdRecords = await createManyRecords({
+          recordsToCreate: batchedRecordsToCreate,
+          upsert,
+          abortController,
+        });
+
+        setBatchedRecordsCount?.(createdRecordsCount);
+        allCreatedRecords.push(...createdRecords);
+      }
+    } catch (error) {
+      if (error instanceof ApolloError && error.message.includes('aborted')) {
+        const formattedCreatedRecordsCount = formatNumber(createdRecordsCount);
+        enqueueSnackBar(
+          t`Record creation stopped. ${formattedCreatedRecordsCount} records created.`,
+          {
+            variant: SnackBarVariant.Warning,
+            duration: 5000,
+          },
+        );
+      } else {
+        throw error;
+      }
+    }
+
+    await refetchAggregateQueries();
+    return allCreatedRecords;
+  };
+
+  return {
+    batchCreateManyRecords,
+  };
+};

packages/twenty-front/src/modules/object-record/hooks/useCreateManyRecords.ts

@@ -32,11 +32,12 @@ type PartialObjectRecordWithOptionalId = Partial<ObjectRecord> & {
   id?: string;
 };
 
-type useCreateManyRecordsProps = {
+export type useCreateManyRecordsProps = {
   objectNameSingular: string;
   recordGqlFields?: RecordGqlOperationGqlRecordFields;
   skipPostOptimisticEffect?: boolean;
   shouldMatchRootQueryFilter?: boolean;
+  shouldRefetchAggregateQueries?: boolean;
 };
 
 export const useCreateManyRecords = <
@@ -46,6 +47,7 @@ export const useCreateManyRecords = <
   recordGqlFields,
   skipPostOptimisticEffect = false,
   shouldMatchRootQueryFilter,
+  shouldRefetchAggregateQueries = true,
 }: useCreateManyRecordsProps) => {
   const apolloClient = useApolloClient();
 
@@ -76,10 +78,17 @@ export const useCreateManyRecords = <
     objectMetadataNamePlural: objectMetadataItem.namePlural,
   });
 
-  const createManyRecords = async (
-    recordsToCreate: Partial<CreatedObjectRecord>[],
-    upsert?: boolean,
-  ) => {
+  type createManyRecordsProps = {
+    recordsToCreate: Partial<CreatedObjectRecord>[];
+    upsert?: boolean;
+    abortController?: AbortController;
+  };
+
+  const createManyRecords = async ({
+    recordsToCreate,
+    upsert,
+    abortController,
+  }: createManyRecordsProps) => {
     const sanitizedCreateManyRecordsInput: PartialObjectRecordWithOptionalId[] =
       [];
     const recordOptimisticRecordsInput: PartialObjectRecordWithId[] = [];
@@ -169,6 +178,11 @@ export const useCreateManyRecords = <
           data: sanitizedCreateManyRecordsInput,
           upsert: upsert,
         },
+        context: {
+          fetchOptions: {
+            signal: abortController?.signal,
+          },
+        },
         update: (cache, { data }) => {
           const records = data?.[mutationResponseField];
 
@@ -205,7 +219,8 @@ export const useCreateManyRecords = <
         throw error;
       });
 
-    await refetchAggregateQueries();
+    if (shouldRefetchAggregateQueries) await refetchAggregateQueries();
+
     return createdObjects.data?.[mutationResponseField] ?? [];
   };
 

packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreadsheetImportDialog.ts

@@ -1,12 +1,15 @@
 import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
-import { useCreateManyRecords } from '@/object-record/hooks/useCreateManyRecords';
+import { useBatchCreateManyRecords } from '@/object-record/hooks/useBatchCreateManyRecords';
 import { useBuildAvailableFieldsForImport } from '@/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport';
 import { buildRecordFromImportedStructuredRow } from '@/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow';
 import { spreadsheetImportFilterAvailableFieldMetadataItems } from '@/object-record/spreadsheet-import/utils/spreadsheetImportFilterAvailableFieldMetadataItems.ts';
+import { SpreadsheetImportCreateRecordsBatchSize } from '@/spreadsheet-import/constants/SpreadsheetImportCreateRecordsBatchSize';
 import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog';
+import { spreadsheetImportCreatedRecordsProgressState } from '@/spreadsheet-import/states/spreadsheetImportCreatedRecordsProgressState';
 import { SpreadsheetImportDialogOptions } from '@/spreadsheet-import/types';
 import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar';
 import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
+import { useSetRecoilState } from 'recoil';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
 
 export const useOpenObjectRecordsSpreadsheetImportDialog = (
@@ -19,8 +22,17 @@ export const useOpenObjectRecordsSpreadsheetImportDialog = (
     objectNameSingular,
   });
 
-  const { createManyRecords } = useCreateManyRecords({
+  const setCreatedRecordsProgress = useSetRecoilState(
+    spreadsheetImportCreatedRecordsProgressState,
+  );
+
+  const abortController = new AbortController();
+
+  const { batchCreateManyRecords } = useBatchCreateManyRecords({
     objectNameSingular,
+    mutationBatchSize: SpreadsheetImportCreateRecordsBatchSize,
+    setBatchedRecordsCount: setCreatedRecordsProgress,
+    abortController,
   });
 
   const { buildAvailableFieldsForImport } = useBuildAvailableFieldsForImport();
@@ -61,8 +73,10 @@ export const useOpenObjectRecordsSpreadsheetImportDialog = (
         });
 
         try {
-          const upsert = true;
-          await createManyRecords(createInputs, upsert);
+          await batchCreateManyRecords({
+            recordsToCreate: createInputs,
+            upsert: true,
+          });
         } catch (error: any) {
           enqueueSnackBar(error?.message || 'Something went wrong', {
             variant: SnackBarVariant.Error,
@@ -71,6 +85,9 @@ export const useOpenObjectRecordsSpreadsheetImportDialog = (
       },
       fields: availableFieldsForMatching,
       availableFieldMetadataItems: availableFieldMetadataItemsToImport,
+      onAbortSubmit: () => {
+        abortController.abort();
+      },
     });
   };