5425 - Introducing support for all Composite Fields Import (#5470)

Adding support for all Composite Fields while using the "import" functionality. This includes: - Currency - Address Edit : - Refactored a lot of types in the spreadsheet import module - Renamed a lot of functions, hooks and types that were not self-explanatory enough --------- Co-authored-by: Charles Bochet <charles@twenty.com> Co-authored-by: Félix Malfait <felix.malfait@gmail.com> Co-authored-by: Lucas Bordeau <bordeau.lucas@gmail.com>
2024-07-23 21:32:23 +05:30
parent 2cc0597ee4
commit 5c8fe027f9
46 changed files with 888 additions and 535 deletions
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx
@ -1,10 +1,10 @@
-import { useCallback, useEffect, useMemo, useState } from 'react';
 import styled from '@emotion/styled';
+import { useCallback, useEffect, useMemo, useState } from 'react';

 import { Heading } from '@/spreadsheet-import/components/Heading';
 import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton';
 import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
-import { Field, RawData } from '@/spreadsheet-import/types';
+import { Field, ImportedRow } from '@/spreadsheet-import/types';
 import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields';
 import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns';
 import { normalizeTableData } from '@/spreadsheet-import/utils/normalizeTableData';
@ -46,9 +46,13 @@ const StyledColumn = styled.span`
 `;

 export type MatchColumnsStepProps<T extends string> = {
-  data: RawData[];
-  headerValues: RawData;
-  onContinue: (data: any[], rawData: RawData[], columns: Columns<T>) => void;
+  data: ImportedRow[];
+  headerValues: ImportedRow;
+  onContinue: (
+    data: any[],
+    rawData: ImportedRow[],
+    columns: Columns<T>,
+  ) => void;
  onBack: () => void;
 };

@ -67,23 +71,27 @@ export type MatchedOptions<T> = {
 };

 type EmptyColumn = { type: ColumnType.empty; index: number; header: string };
+
 type IgnoredColumn = {
  type: ColumnType.ignored;
  index: number;
  header: string;
 };
+
 type MatchedColumn<T> = {
  type: ColumnType.matched;
  index: number;
  header: string;
  value: T;
 };
+
 type MatchedSwitchColumn<T> = {
  type: ColumnType.matchedCheckbox;
  index: number;
  header: string;
  value: T;
 };
+
 export type MatchedSelectColumn<T> = {
  type: ColumnType.matchedSelect;
  index: number;
@ -91,6 +99,7 @@ export type MatchedSelectColumn<T> = {
  value: T;
  matchedOptions: Partial<MatchedOptions<T>>[];
 };
+
 export type MatchedSelectOptionsColumn<T> = {
  type: ColumnType.matchedSelectOptions;
  index: number;
@ -271,7 +280,7 @@ export const MatchColumnsStep = <T extends string>({
          renderUserColumn={(columns, columnIndex) => (
            <UserTableColumn
              column={columns[columnIndex]}
-              entries={dataExample.map(
+              importedRow={dataExample.map(
                (row) => row[columns[columnIndex].index],
              )}
            />
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UserTableColumn.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UserTableColumn.tsx
@ -1,6 +1,6 @@
 import styled from '@emotion/styled';

-import { RawData } from '@/spreadsheet-import/types';
+import { ImportedRow } from '@/spreadsheet-import/types';
 import { isDefined } from '~/utils/isDefined';

 import { Column } from '../MatchColumnsStep';
@ -31,20 +31,22 @@ const StyledExample = styled.span`

 type UserTableColumnProps<T extends string> = {
  column: Column<T>;
-  entries: RawData;
+  importedRow: ImportedRow;
 };

 export const UserTableColumn = <T extends string>({
  column,
-  entries,
+  importedRow,
 }: UserTableColumnProps<T>) => {
  const { header } = column;
-  const entry = entries.find(isDefined);
+  const firstDefinedValue = importedRow.find(isDefined);

  return (
    <StyledContainer>
      <StyledValue>{header}</StyledValue>
-      {entry && <StyledExample>{`ex: ${entry}`}</StyledExample>}
+      {firstDefinedValue && (
+        <StyledExample>{`ex: ${firstDefinedValue}`}</StyledExample>
+      )}
    </StyledContainer>
  );
 };
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/SelectHeaderStep.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/SelectHeaderStep.tsx
@ -1,9 +1,9 @@
-import { useCallback, useState } from 'react';
 import styled from '@emotion/styled';
+import { useCallback, useState } from 'react';

 import { Heading } from '@/spreadsheet-import/components/Heading';
 import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton';
-import { RawData } from '@/spreadsheet-import/types';
+import { ImportedRow } from '@/spreadsheet-import/types';
 import { Modal } from '@/ui/layout/modal/components/Modal';

 import { SelectHeaderTable } from './components/SelectHeaderTable';
@ -19,29 +19,36 @@ const StyledTableContainer = styled.div`
 `;

 type SelectHeaderStepProps = {
-  data: RawData[];
-  onContinue: (headerValues: RawData, data: RawData[]) => Promise<void>;
+  importedRows: ImportedRow[];
+  onContinue: (
+    headerValues: ImportedRow,
+    importedRows: ImportedRow[],
+  ) => Promise<void>;
  onBack: () => void;
 };

 export const SelectHeaderStep = ({
-  data,
+  importedRows,
  onContinue,
  onBack,
 }: SelectHeaderStepProps) => {
-  const [selectedRows, setSelectedRows] = useState<ReadonlySet<number>>(
-    new Set([0]),
-  );
+  const [selectedRowIndexes, setSelectedRowIndexes] = useState<
+    ReadonlySet<number>
+  >(new Set([0]));
+
  const [isLoading, setIsLoading] = useState(false);

  const handleContinue = useCallback(async () => {
-    const [selectedRowIndex] = Array.from(new Set(selectedRows));
+    const [selectedRowIndex] = Array.from(new Set(selectedRowIndexes));
    // We consider data above header to be redundant
-    const trimmedData = data.slice(selectedRowIndex + 1);
+    const trimmedData = importedRows.slice(selectedRowIndex + 1);
+
    setIsLoading(true);
-    await onContinue(data[selectedRowIndex], trimmedData);
+
+    await onContinue(importedRows[selectedRowIndex], trimmedData);
+
    setIsLoading(false);
-  }, [onContinue, data, selectedRows]);
+  }, [onContinue, importedRows, selectedRowIndexes]);

  return (
    <>
@ -49,9 +56,9 @@ export const SelectHeaderStep = ({
        <StyledHeading title="Select header row" />
        <StyledTableContainer>
          <SelectHeaderTable
-            data={data}
-            selectedRows={selectedRows}
-            setSelectedRows={setSelectedRows}
+            importedRows={importedRows}
+            selectedRowIndexes={selectedRowIndexes}
+            setSelectedRowIndexes={setSelectedRowIndexes}
          />
        </StyledTableContainer>
      </Modal.Content>
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/components/SelectColumn.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/components/SelectColumn.tsx
@ -1,7 +1,7 @@
 // @ts-expect-error // Todo: remove usage of react-data-grid
 import { Column, FormatterProps, useRowSelection } from 'react-data-grid';

-import { RawData } from '@/spreadsheet-import/types';
+import { ImportedRow } from '@/spreadsheet-import/types';
 import { Radio } from '@/ui/input/components/Radio';

 const SELECT_COLUMN_KEY = 'select-row';
@ -39,7 +39,7 @@ export const SelectColumn: Column<any, any> = {
  formatter: SelectFormatter,
 };

-export const generateSelectionColumns = (data: RawData[]) => {
+export const generateSelectionColumns = (data: ImportedRow[]) => {
  const longestRowLength = data.reduce(
    (acc, curr) => (acc > curr.length ? acc : curr.length),
    0,
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/components/SelectHeaderTable.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/SelectHeaderStep/components/SelectHeaderTable.tsx
@ -1,41 +1,44 @@
 import { useMemo } from 'react';

 import { Table } from '@/spreadsheet-import/components/Table';
-import { RawData } from '@/spreadsheet-import/types';
+import { ImportedRow } from '@/spreadsheet-import/types';

 import { generateSelectionColumns } from './SelectColumn';

-interface SelectHeaderTableProps {
-  data: RawData[];
-  selectedRows: ReadonlySet<number>;
-  setSelectedRows: (rows: ReadonlySet<number>) => void;
-}
+type SelectHeaderTableProps = {
+  importedRows: ImportedRow[];
+  selectedRowIndexes: ReadonlySet<number>;
+  setSelectedRowIndexes: (rowIndexes: ReadonlySet<number>) => void;
+};

 export const SelectHeaderTable = ({
-  data,
-  selectedRows,
-  setSelectedRows,
+  importedRows,
+  selectedRowIndexes,
+  setSelectedRowIndexes,
 }: SelectHeaderTableProps) => {
-  const columns = useMemo(() => generateSelectionColumns(data), [data]);
+  const columns = useMemo(
+    () => generateSelectionColumns(importedRows),
+    [importedRows],
+  );

  return (
    <Table
      // Todo: remove usage of react-data-grid
-      rowKeyGetter={(row: any) => data.indexOf(row)}
-      rows={data}
+      rowKeyGetter={(row: any) => importedRows.indexOf(row)}
+      rows={importedRows}
      columns={columns}
-      selectedRows={selectedRows}
-      onSelectedRowsChange={(newRows: any) => {
+      selectedRowIndexes={selectedRowIndexes}
+      onSelectedRowIndexesChange={(newRowIndexes: number[]) => {
        // allow selecting only one row
-        newRows.forEach((value: any) => {
-          if (!selectedRows.has(value as number)) {
-            setSelectedRows(new Set([value as number]));
+        newRowIndexes.forEach((value: any) => {
+          if (!selectedRowIndexes.has(value as number)) {
+            setSelectedRowIndexes(new Set([value as number]));
            return;
          }
        });
      }}
      onRowClick={(row: any) => {
-        setSelectedRows(new Set([data.indexOf(row)]));
+        setSelectedRowIndexes(new Set([importedRows.indexOf(row)]));
      }}
      headerRowHeight={0}
    />
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadFlow.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadFlow.tsx
@ -1,10 +1,10 @@
-import { useCallback, useState } from 'react';
 import { useTheme } from '@emotion/react';
 import styled from '@emotion/styled';
+import { useCallback, useState } from 'react';
 import { WorkBook } from 'xlsx-ugnis';

 import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
-import { RawData } from '@/spreadsheet-import/types';
+import { ImportedRow } from '@/spreadsheet-import/types';
 import { exceedsMaxRecords } from '@/spreadsheet-import/utils/exceedsMaxRecords';
 import { mapWorkbook } from '@/spreadsheet-import/utils/mapWorkbook';
 import { CircularProgressBar } from '@/ui/feedback/progress-bar/components/CircularProgressBar';
@ -42,12 +42,12 @@ export type StepState =
    }
  | {
      type: StepType.selectHeader;
-      data: RawData[];
+      data: ImportedRow[];
    }
  | {
      type: StepType.matchColumns;
-      data: RawData[];
-      headerValues: RawData;
+      data: ImportedRow[];
+      headerValues: ImportedRow;
    }
  | {
      type: StepType.validateData;
@ -131,10 +131,8 @@ export const UploadFlow = ({ nextStep, prevStep }: UploadFlowProps) => {
                  // Automatically select first row as header
                  const trimmedData = mappedWorkbook.slice(1);

-                  const { data, headerValues } = await selectHeaderStepHook(
-                    mappedWorkbook[0],
-                    trimmedData,
-                  );
+                  const { importedRows: data, headerRow: headerValues } =
+                    await selectHeaderStepHook(mappedWorkbook[0], trimmedData);

                  setState({
                    type: StepType.matchColumns,
@ -186,12 +184,11 @@ export const UploadFlow = ({ nextStep, prevStep }: UploadFlowProps) => {
    case StepType.selectHeader:
      return (
        <SelectHeaderStep
-          data={state.data}
+          importedRows={state.data}
          onContinue={async (...args) => {
            try {
-              const { data, headerValues } = await selectHeaderStepHook(
-                ...args,
-              );
+              const { importedRows: data, headerRow: headerValues } =
+                await selectHeaderStepHook(...args);
              setState({
                type: StepType.matchColumns,
                data,
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/components/DropZone.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/UploadStep/components/DropZone.tsx
@ -1,6 +1,6 @@
+import styled from '@emotion/styled';
 import { useState } from 'react';
 import { useDropzone } from 'react-dropzone';
-import styled from '@emotion/styled';
 import * as XLSX from 'xlsx-ugnis';

 import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
@ -79,11 +79,7 @@ const StyledText = styled.span`
  font-size: ${({ theme }) => theme.font.size.sm};
  font-weight: ${({ theme }) => theme.font.weight.medium};
  text-align: center;
-`;
-
-const StyledButton = styled(MainButton)`
-  margin-top: ${({ theme }) => theme.spacing(2)};
-  width: 200px;
+  padding: 15px;
 `;

 type DropZoneProps = {
@ -151,7 +147,7 @@ export const DropZone = ({ onContinue, isLoading }: DropZoneProps) => {
      ) : (
        <>
          <StyledText>Upload .xlsx, .xls or .csv file</StyledText>
-          <StyledButton onClick={open} title="Select file" />
+          <MainButton onClick={open} title="Select file" />
        </>
      )}
    </StyledContainer>
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx
@ -1,7 +1,7 @@
+import styled from '@emotion/styled';
 import { useCallback, useMemo, useState } from 'react';
 // @ts-expect-error Todo: remove usage of react-data-grid
 import { RowsChangeData } from 'react-data-grid';
-import styled from '@emotion/styled';
 import { IconTrash } from 'twenty-ui';

 import { Heading } from '@/spreadsheet-import/components/Heading';
@ -12,7 +12,10 @@ import {
  Columns,
  ColumnType,
 } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
-import { Data } from '@/spreadsheet-import/types';
+import {
+  ImportedStructuredRow,
+  ImportValidationResult,
+} from '@/spreadsheet-import/types';
 import { addErrorsAndRunHooks } from '@/spreadsheet-import/utils/dataMutations';
 import { useDialogManager } from '@/ui/feedback/dialog-manager/hooks/useDialogManager';
 import { Button } from '@/ui/input/button/components/Button';
@ -21,7 +24,7 @@ import { Modal } from '@/ui/layout/modal/components/Modal';
 import { isDefined } from '~/utils/isDefined';

 import { generateColumns } from './components/columns';
-import { Meta } from './types';
+import { ImportedStructuredRowMetadata } from './types';

 const StyledContent = styled(Modal.Content)`
  padding-left: ${({ theme }) => theme.spacing(6)};
@ -65,7 +68,7 @@ const StyledNoRowsContainer = styled.div`
 `;

 type ValidationStepProps<T extends string> = {
-  initialData: Data<T>[];
+  initialData: ImportedStructuredRow<T>[];
  importedColumns: Columns<string>;
  file: File;
  onSubmitStart?: () => void;
@ -83,7 +86,9 @@ export const ValidationStep = <T extends string>({
  const { fields, onClose, onSubmit, rowHook, tableHook } =
    useSpreadsheetImportInternal<T>();

-  const [data, setData] = useState<(Data<T> & Meta)[]>(
+  const [data, setData] = useState<
+    (ImportedStructuredRow<T> & ImportedStructuredRowMetadata)[]
+  >(
    useMemo(
      () => addErrorsAndRunHooks<T>(initialData, fields, rowHook, tableHook),
      // eslint-disable-next-line react-hooks/exhaustive-deps
@ -173,7 +178,11 @@ export const ValidationStep = <T extends string>({
    return data;
  }, [data, filterByErrors]);

-  const rowKeyGetter = useCallback((row: Data<T> & Meta) => row.__index, []);
+  const rowKeyGetter = useCallback(
+    (row: ImportedStructuredRow<T> & ImportedStructuredRowMetadata) =>
+      row.__index,
+    [],
+  );

  const submitData = async () => {
    const calculatedData = data.reduce(
@ -182,15 +191,23 @@ export const ValidationStep = <T extends string>({
        if (isDefined(__errors)) {
          for (const key in __errors) {
            if (__errors[key].level === 'error') {
-              acc.invalidData.push(values as unknown as Data<T>);
+              acc.invalidStructuredRows.push(
+                values as unknown as ImportedStructuredRow<T>,
+              );
              return acc;
            }
          }
        }
-        acc.validData.push(values as unknown as Data<T>);
+        acc.validStructuredRows.push(
+          values as unknown as ImportedStructuredRow<T>,
+        );
        return acc;
      },
-      { validData: [] as Data<T>[], invalidData: [] as Data<T>[], all: data },
+      {
+        validStructuredRows: [] as ImportedStructuredRow<T>[],
+        invalidStructuredRows: [] as ImportedStructuredRow<T>[],
+        allStructuredRows: data,
+      } satisfies ImportValidationResult<T>,
    );
    onSubmitStart?.();
    await onSubmit(calculatedData, file);
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/components/columns.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/components/columns.tsx
@ -1,17 +1,17 @@
+import styled from '@emotion/styled';
 // @ts-expect-error // Todo: remove usage of react-data-grid
 import { Column, useRowSelection } from 'react-data-grid';
 import { createPortal } from 'react-dom';
-import styled from '@emotion/styled';
 import { AppTooltip } from 'twenty-ui';

 import { MatchColumnSelect } from '@/spreadsheet-import/components/MatchColumnSelect';
-import { Data, Fields } from '@/spreadsheet-import/types';
+import { Fields, ImportedStructuredRow } from '@/spreadsheet-import/types';
 import { Checkbox, CheckboxVariant } from '@/ui/input/components/Checkbox';
 import { TextInput } from '@/ui/input/components/TextInput';
 import { Toggle } from '@/ui/input/components/Toggle';
 import { isDefined } from '~/utils/isDefined';

-import { Meta } from '../types';
+import { ImportedStructuredRowMetadata } from '../types';

 const StyledHeaderContainer = styled.div`
  align-items: center;
@ -63,7 +63,7 @@ const SELECT_COLUMN_KEY = 'select-row';

 export const generateColumns = <T extends string>(
  fields: Fields<T>,
-): Column<Data<T> & Meta>[] => [
+): Column<ImportedStructuredRow<T> & ImportedStructuredRowMetadata>[] => [
  {
    key: SELECT_COLUMN_KEY,
    name: '',
@ -96,7 +96,9 @@ export const generateColumns = <T extends string>(
    },
  },
  ...fields.map(
-    (column): Column<Data<T> & Meta> => ({
+    (
+      column,
+    ): Column<ImportedStructuredRow<T> & ImportedStructuredRowMetadata> => ({
      key: column.key,
      name: column.label,
      minWidth: 150,
@ -120,7 +122,8 @@ export const generateColumns = <T extends string>(
      editable: column.fieldType.type !== 'checkbox',
      // Todo: remove usage of react-data-grid
      editor: ({ row, onRowChange, onClose }: any) => {
-        const columnKey = column.key as keyof (Data<T> & Meta);
+        const columnKey = column.key as keyof (ImportedStructuredRow<T> &
+          ImportedStructuredRowMetadata);
        let component;

        switch (column.fieldType.type) {
@ -167,7 +170,8 @@ export const generateColumns = <T extends string>(
      },
      // Todo: remove usage of react-data-grid
      formatter: ({ row, onRowChange }: { row: any; onRowChange: any }) => {
-        const columnKey = column.key as keyof (Data<T> & Meta);
+        const columnKey = column.key as keyof (ImportedStructuredRow<T> &
+          ImportedStructuredRowMetadata);
        let component;

        switch (column.fieldType.type) {
@ -226,7 +230,7 @@ export const generateColumns = <T extends string>(

        return component;
      },
-      cellClass: (row: Meta) => {
+      cellClass: (row: ImportedStructuredRowMetadata) => {
        switch (row.__errors?.[column.key]?.level) {
          case 'error':
            return 'rdg-cell-error';
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/types.ts
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/types.ts
@ -1,5 +1,8 @@
 import { Info } from '@/spreadsheet-import/types';

-export type Meta = { __index: string; __errors?: Error | null };
+export type ImportedStructuredRowMetadata = {
+  __index: string;
+  __errors?: Error | null;
+};
 export type Error = { [key: string]: Info };
 export type Errors = { [id: string]: Error };
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/stories/SelectHeader.stories.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/stories/SelectHeader.stories.tsx
@ -24,7 +24,7 @@ export const Default = () => (
    <Providers values={mockRsiValues}>
      <ModalWrapper isOpen={true} onClose={() => null}>
        <SelectHeaderStep
-          data={headerSelectionTableFields}
+          importedRows={headerSelectionTableFields}
          onContinue={() => Promise.resolve()}
          onBack={() => Promise.resolve()}
        />