Custom object import csv (#3756)

* poc custom object import csv * fix fullname * lint * add relation Ids, fix label full name, add simple test * mock missing fields? * - fix test * validate uuid, fix key in column dropdown, don't save non set composite fields, allow only import relations where toRelationMetadata
2024-02-06 16:22:39 +01:00
parent 0096e60489
commit 7b8fffc3b8
13 changed files with 241 additions and 663 deletions
--- a/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx
+++ b/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx
@ -5,8 +5,8 @@ import { Key } from 'ts-key-enum';
 import { RECORD_INDEX_OPTIONS_DROPDOWN_ID } from '@/object-record/record-index/options/constants/RecordIndexOptionsDropdownId';
 import { useRecordIndexOptionsForBoard } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForBoard';
 import { useRecordIndexOptionsForTable } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForTable';
-import { useRecordIndexOptionsImport } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsImport';
 import { TableOptionsHotkeyScope } from '@/object-record/record-table/types/TableOptionsHotkeyScope';
+import { useSpreadsheetRecordImport } from '@/object-record/spreadsheet-import/useSpreadsheetRecordImport';
 import {
  IconBaselineDensitySmall,
  IconChevronLeft,
@ -116,7 +116,8 @@ export const RecordIndexOptionsDropdownContent = ({
      ? handleBoardFieldVisibilityChange
      : handleColumnVisibilityChange;

-  const { handleImport } = useRecordIndexOptionsImport({ objectNameSingular });
+  const { openRecordSpreadsheetImport } =
+    useSpreadsheetRecordImport(objectNameSingular);

  return (
    <>
@ -141,13 +142,11 @@ export const RecordIndexOptionsDropdownContent = ({
              LeftIcon={IconTag}
              text="Fields"
            />
-            {handleImport && (
-              <MenuItem
-                onClick={() => handleImport()}
-                LeftIcon={IconFileImport}
-                text="Import"
-              />
-            )}
+            <MenuItem
+              onClick={() => openRecordSpreadsheetImport()}
+              LeftIcon={IconFileImport}
+              text="Import"
+            />
          </DropdownMenuItemsContainer>
        </>
      )}
--- a/packages/twenty-front/src/modules/object-record/record-index/options/hooks/useRecordIndexOptionsImport.ts
+++ b/packages/twenty-front/src/modules/object-record/record-index/options/hooks/useRecordIndexOptionsImport.ts
@ -1,23 +0,0 @@
-import { useSpreadsheetCompanyImport } from '@/companies/hooks/useSpreadsheetCompanyImport';
-import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular';
-import { useSpreadsheetPersonImport } from '@/people/hooks/useSpreadsheetPersonImport';
-
-type useRecordIndexOptionsImportParams = {
-  objectNameSingular: string;
-};
-
-export const useRecordIndexOptionsImport = ({
-  objectNameSingular,
-}: useRecordIndexOptionsImportParams) => {
-  const { openPersonSpreadsheetImport } = useSpreadsheetPersonImport();
-  const { openCompanySpreadsheetImport } = useSpreadsheetCompanyImport();
-
-  const handleImport =
-    CoreObjectNameSingular.Company === objectNameSingular
-      ? openCompanySpreadsheetImport
-      : CoreObjectNameSingular.Person === objectNameSingular
-        ? openPersonSpreadsheetImport
-        : undefined;
-
-  return { handleImport };
-};
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/tests/useSpreadsheetRecordImport.test.tsx
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/tests/useSpreadsheetRecordImport.test.tsx
@ -0,0 +1,140 @@
+import { ReactNode } from 'react';
+import { gql } from '@apollo/client';
+import { MockedProvider } from '@apollo/client/testing';
+import { act, renderHook, waitFor } from '@testing-library/react';
+import { RecoilRoot, useRecoilValue } from 'recoil';
+
+import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular';
+import { spreadsheetImportState } from '@/spreadsheet-import/states/spreadsheetImportState';
+import { SnackBarProviderScope } from '@/ui/feedback/snack-bar-manager/scopes/SnackBarProviderScope';
+
+import { useSpreadsheetRecordImport } from '../useSpreadsheetRecordImport';
+
+const companyId = 'cb2e9f4b-20c3-4759-9315-4ffeecfaf71a';
+
+jest.mock('uuid', () => ({
+  v4: jest.fn(() => companyId),
+}));
+
+jest.mock('@/object-metadata/hooks/useMapFieldMetadataToGraphQLQuery', () => ({
+  useMapFieldMetadataToGraphQLQuery: () => () => '\n',
+}));
+
+const companyMocks = [
+  {
+    request: {
+      query: gql`
+        mutation CreateCompanies($data: [CompanyCreateInput!]!) {
+          createCompanies(data: $data) {
+            id
+          }
+        }
+      `,
+      variables: {
+        data: [
+          {
+            address: 'test',
+            domainName: 'example.com',
+            employees: 0,
+            idealCustomerProfile: true,
+            name: 'Example Company',
+            id: companyId,
+          },
+        ],
+      },
+    },
+    result: jest.fn(() => ({
+      data: {
+        createCompanies: [
+          {
+            id: companyId,
+          },
+        ],
+      },
+    })),
+  },
+];
+
+const fakeCsv = () => {
+  const csvContent = 'name\nExample Company';
+  const blob = new Blob([csvContent], { type: 'text/csv' });
+  return new File([blob], 'fakeData.csv', { type: 'text/csv' });
+};
+
+const Wrapper = ({ children }: { children: ReactNode }) => (
+  <RecoilRoot>
+    <MockedProvider mocks={companyMocks} addTypename={false}>
+      <SnackBarProviderScope snackBarManagerScopeId="snack-bar-manager">
+        {children}
+      </SnackBarProviderScope>
+    </MockedProvider>
+  </RecoilRoot>
+);
+
+describe('useSpreadsheetCompanyImport', () => {
+  it('should work as expected', async () => {
+    const { result } = renderHook(
+      () => {
+        const spreadsheetImport = useRecoilValue(spreadsheetImportState);
+        const { openRecordSpreadsheetImport } = useSpreadsheetRecordImport(
+          CoreObjectNameSingular.Company,
+        );
+        return { openRecordSpreadsheetImport, spreadsheetImport };
+      },
+      {
+        wrapper: Wrapper,
+      },
+    );
+
+    const { spreadsheetImport, openRecordSpreadsheetImport } = result.current;
+
+    expect(spreadsheetImport.isOpen).toBe(false);
+    expect(spreadsheetImport.options).toBeNull();
+
+    await act(async () => {
+      openRecordSpreadsheetImport();
+    });
+
+    const { spreadsheetImport: updatedImport } = result.current;
+
+    expect(updatedImport.isOpen).toBe(true);
+    expect(updatedImport.options).toHaveProperty('onSubmit');
+    expect(updatedImport.options?.onSubmit).toBeInstanceOf(Function);
+    expect(updatedImport.options).toHaveProperty('fields');
+    expect(Array.isArray(updatedImport.options?.fields)).toBe(true);
+
+    act(() => {
+      updatedImport.options?.onSubmit(
+        {
+          validData: [
+            {
+              id: companyId,
+              name: 'Example Company',
+              domainName: 'example.com',
+              idealCustomerProfile: true,
+              address: 'test',
+              employees: '0',
+            },
+          ],
+          invalidData: [],
+          all: [
+            {
+              id: companyId,
+              name: 'Example Company',
+              domainName: 'example.com',
+              __index: 'cbc3985f-dde9-46d1-bae2-c124141700ac',
+              idealCustomerProfile: true,
+              address: 'test',
+              employees: '0',
+            },
+          ],
+        },
+        fakeCsv(),
+      );
+    });
+
+    await waitFor(() => {
+      expect(companyMocks[0].result).toHaveBeenCalled();
+    });
+  });
+});
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/useSpreadsheetRecordImport.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/useSpreadsheetRecordImport.ts
@ -0,0 +1,166 @@
+import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
+import { useCreateManyRecords } from '@/object-record/hooks/useCreateManyRecords';
+import { getSpreadSheetValidation } from '@/object-record/spreadsheet-import/util/getSpreadSheetValidation';
+import { useSpreadsheetImport } from '@/spreadsheet-import/hooks/useSpreadsheetImport';
+import { SpreadsheetOptions, Validation } from '@/spreadsheet-import/types';
+import { useIcons } from '@/ui/display/icon/hooks/useIcons';
+import { IconComponent } from '@/ui/display/icon/types/IconComponent';
+import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
+import { FieldMetadataType } from '~/generated-metadata/graphql';
+
+const firstName = 'Firstname';
+const lastName = 'Lastname';
+
+export const useSpreadsheetRecordImport = (objectNameSingular: string) => {
+  const { openSpreadsheetImport } = useSpreadsheetImport<any>();
+  const { enqueueSnackBar } = useSnackBar();
+  const { getIcon } = useIcons();
+
+  const { objectMetadataItem } = useObjectMetadataItem({ objectNameSingular });
+  const fields = objectMetadataItem.fields
+    .filter(
+      (x) =>
+        x.isActive &&
+        !x.isSystem &&
+        x.name !== 'createdAt' &&
+        (x.type !== FieldMetadataType.Relation || x.toRelationMetadata),
+    )
+    .sort((a, b) => a.name.localeCompare(b.name));
+
+  const templateFields: {
+    icon: IconComponent;
+    label: string;
+    key: string;
+    fieldType: {
+      type: 'input' | 'checkbox';
+    };
+    validations?: Validation[];
+  }[] = [];
+  for (const field of fields) {
+    if (field.type === FieldMetadataType.FullName) {
+      templateFields.push({
+        icon: getIcon(field.icon),
+        label: `${firstName} (${field.label})`,
+        key: `${firstName} (${field.name})`,
+        fieldType: {
+          type: 'input',
+        },
+        validations: getSpreadSheetValidation(
+          field.type,
+          `${firstName} (${field.label})`,
+        ),
+      });
+      templateFields.push({
+        icon: getIcon(field.icon),
+        label: `${lastName} (${field.label})`,
+        key: `${lastName} (${field.name})`,
+        fieldType: {
+          type: 'input',
+        },
+        validations: getSpreadSheetValidation(
+          field.type,
+          `${lastName} (${field.label})`,
+        ),
+      });
+    } else if (field.type === FieldMetadataType.Relation) {
+      templateFields.push({
+        icon: getIcon(field.icon),
+        label: field.label + ' (ID)',
+        key: field.name,
+        fieldType: {
+          type: 'input',
+        },
+        validations: getSpreadSheetValidation(
+          field.type,
+          field.label + ' (ID)',
+        ),
+      });
+    } else {
+      templateFields.push({
+        icon: getIcon(field.icon),
+        label: field.label,
+        key: field.name,
+        fieldType: {
+          type: 'input',
+        },
+        validations: getSpreadSheetValidation(field.type, field.label),
+      });
+    }
+  }
+
+  const { createManyRecords } = useCreateManyRecords({
+    objectNameSingular,
+  });
+
+  const openRecordSpreadsheetImport = (
+    options?: Omit<SpreadsheetOptions<any>, 'fields' | 'isOpen' | 'onClose'>,
+  ) => {
+    openSpreadsheetImport({
+      ...options,
+      onSubmit: async (data) => {
+        const createInputs = data.validData.map((record) => {
+          const fieldMapping: Record<string, any> = {};
+          for (const field of fields) {
+            const value = record[field.name];
+
+            switch (field.type) {
+              case FieldMetadataType.Boolean:
+                fieldMapping[field.name] = value === 'true' || value === true;
+                break;
+              case FieldMetadataType.Number:
+              case FieldMetadataType.Numeric:
+                fieldMapping[field.name] = Number(value);
+                break;
+              case FieldMetadataType.Currency:
+                if (value !== undefined) {
+                  fieldMapping[field.name] = {
+                    amountMicros: Number(value),
+                    currencyCode: 'USD',
+                  };
+                }
+                break;
+              case FieldMetadataType.Link:
+                if (value !== undefined) {
+                  fieldMapping[field.name] = {
+                    label: field.name,
+                    url: value || null,
+                  };
+                }
+                break;
+              case FieldMetadataType.Relation:
+                if (value) {
+                  fieldMapping[field.name + 'Id'] = value;
+                }
+                break;
+              case FieldMetadataType.FullName:
+                if (
+                  record[`${firstName} (${field.name})`] ||
+                  record[`${lastName} (${field.name})`]
+                ) {
+                  fieldMapping[field.name] = {
+                    firstName: record[`${firstName} (${field.name})`] || '',
+                    lastName: record[`${lastName} (${field.name})`] || '',
+                  };
+                }
+                break;
+              default:
+                fieldMapping[field.name] = value;
+                break;
+            }
+          }
+          return fieldMapping;
+        });
+        try {
+          await createManyRecords(createInputs);
+        } catch (error: any) {
+          enqueueSnackBar(error?.message || 'Something went wrong', {
+            variant: 'error',
+          });
+        }
+      },
+      fields: templateFields,
+    });
+  };
+
+  return { openRecordSpreadsheetImport };
+};
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/util/getSpreadSheetValidation.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/util/getSpreadSheetValidation.ts
@ -0,0 +1,42 @@
+import { isValidPhoneNumber } from 'libphonenumber-js';
+
+import { isValidUuid } from '@/object-record/spreadsheet-import/util/isValidUuid';
+import { Validation } from '@/spreadsheet-import/types';
+import { FieldMetadataType } from '~/generated-metadata/graphql';
+
+export const getSpreadSheetValidation = (
+  type: FieldMetadataType,
+  fieldName: string,
+): Validation[] => {
+  switch (type) {
+    case FieldMetadataType.Number:
+      return [
+        {
+          rule: 'regex',
+          value: '^d+$',
+          errorMessage: fieldName + ' must be a number',
+          level: 'error',
+        },
+      ];
+    case FieldMetadataType.Phone:
+      return [
+        {
+          rule: 'function',
+          isValid: (value: string) => isValidPhoneNumber(value),
+          errorMessage: fieldName + ' is not valid',
+          level: 'error',
+        },
+      ];
+    case FieldMetadataType.Relation:
+      return [
+        {
+          rule: 'function',
+          isValid: (value: string) => isValidUuid(value),
+          errorMessage: fieldName + ' is not valid',
+          level: 'error',
+        },
+      ];
+    default:
+      return [];
+  }
+};
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/util/isValidUuid.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/util/isValidUuid.ts
@ -0,0 +1,5 @@
+export const isValidUuid = (value: string) => {
+  return /^[0-9A-F]{8}-[0-9A-F]{4}-[4][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i.test(
+    value,
+  );
+};