feat: wip import csv [part 1] (#1033)

* feat: wip import csv

* feat: start implementing twenty UI

* feat: new radio button component

* feat: use new radio button component and fix scroll issue

* fix: max height modal

* feat: wip try to customize react-data-grid to match design

* feat: wip match columns

* feat: wip match column selection

* feat: match column

* feat: clean heading component & try to fix scroll in last step

* feat: validation step

* fix: small cleaning and remove unused component

* feat: clean folder architecture

* feat: remove translations

* feat: remove chackra theme

* feat: remove unused libraries

* feat: use option button to open spreadsheet & fix stories

* Fix lint and fix imports

---------

Co-authored-by: Charles Bochet <charles@twenty.com>

front/src/modules/spreadsheet-import/utils/dataMutations.ts

@@ -0,0 +1,130 @@
+import { v4 } from 'uuid';
+
+import type {
+  Errors,
+  Meta,
+} from '@/spreadsheet-import/components/steps/ValidationStep/types';
+import type {
+  Data,
+  Fields,
+  Info,
+  RowHook,
+  TableHook,
+} from '@/spreadsheet-import/types';
+
+export const addErrorsAndRunHooks = <T extends string>(
+  data: (Data<T> & Partial<Meta>)[],
+  fields: Fields<T>,
+  rowHook?: RowHook<T>,
+  tableHook?: TableHook<T>,
+): (Data<T> & Meta)[] => {
+  const errors: Errors = {};
+
+  const addHookError = (rowIndex: number, fieldKey: T, error: Info) => {
+    errors[rowIndex] = {
+      ...errors[rowIndex],
+      [fieldKey]: error,
+    };
+  };
+
+  if (tableHook) {
+    data = tableHook(data, addHookError);
+  }
+
+  if (rowHook) {
+    data = data.map((value, index) =>
+      rowHook(value, (...props) => addHookError(index, ...props), data),
+    );
+  }
+
+  fields.forEach((field) => {
+    field.validations?.forEach((validation) => {
+      switch (validation.rule) {
+        case 'unique': {
+          const values = data.map((entry) => entry[field.key as T]);
+
+          const taken = new Set(); // Set of items used at least once
+          const duplicates = new Set(); // Set of items used multiple times
+
+          values.forEach((value) => {
+            if (validation.allowEmpty && !value) {
+              // If allowEmpty is set, we will not validate falsy fields such as undefined or empty string.
+              return;
+            }
+
+            if (taken.has(value)) {
+              duplicates.add(value);
+            } else {
+              taken.add(value);
+            }
+          });
+
+          values.forEach((value, index) => {
+            if (duplicates.has(value)) {
+              errors[index] = {
+                ...errors[index],
+                [field.key]: {
+                  level: validation.level || 'error',
+                  message: validation.errorMessage || 'Field must be unique',
+                },
+              };
+            }
+          });
+          break;
+        }
+        case 'required': {
+          data.forEach((entry, index) => {
+            if (
+              entry[field.key as T] === null ||
+              entry[field.key as T] === undefined ||
+              entry[field.key as T] === ''
+            ) {
+              errors[index] = {
+                ...errors[index],
+                [field.key]: {
+                  level: validation.level || 'error',
+                  message: validation.errorMessage || 'Field is required',
+                },
+              };
+            }
+          });
+          break;
+        }
+        case 'regex': {
+          const regex = new RegExp(validation.value, validation.flags);
+          data.forEach((entry, index) => {
+            const value = entry[field.key]?.toString() ?? '';
+            if (!value.match(regex)) {
+              errors[index] = {
+                ...errors[index],
+                [field.key]: {
+                  level: validation.level || 'error',
+                  message:
+                    validation.errorMessage ||
+                    `Field did not match the regex /${validation.value}/${validation.flags} `,
+                },
+              };
+            }
+          });
+          break;
+        }
+      }
+    });
+  });
+
+  return data.map((value, index) => {
+    // This is required only for table. Mutates to prevent needless rerenders
+    if (!('__index' in value)) {
+      value.__index = v4();
+    }
+    const newValue = value as Data<T> & Meta;
+
+    if (errors[index]) {
+      return { ...newValue, __errors: errors[index] };
+    }
+    if (!errors[index] && value?.__errors) {
+      return { ...newValue, __errors: null };
+    }
+    return newValue;
+  });
+};

front/src/modules/spreadsheet-import/utils/exceedsMaxRecords.ts

@@ -0,0 +1,12 @@
+import type XLSX from 'xlsx-ugnis';
+
+export const exceedsMaxRecords = (
+  workSheet: XLSX.WorkSheet,
+  maxRecords: number,
+) => {
+  const [top, bottom] =
+    workSheet['!ref']
+      ?.split(':')
+      .map((position) => parseInt(position.replace(/\D/g, ''), 10)) || [];
+  return bottom - top > maxRecords;
+};

front/src/modules/spreadsheet-import/utils/findMatch.ts

@@ -0,0 +1,31 @@
+import lavenstein from 'js-levenshtein';
+
+import type { Fields } from '@/spreadsheet-import/types';
+
+type AutoMatchAccumulator<T> = {
+  distance: number;
+  value: T;
+};
+
+export const findMatch = <T extends string>(
+  header: string,
+  fields: Fields<T>,
+  autoMapDistance: number,
+): T | undefined => {
+  const smallestValue = fields.reduce<AutoMatchAccumulator<T>>((acc, field) => {
+    const distance = Math.min(
+      ...[
+        lavenstein(field.key, header),
+        ...(field.alternateMatches?.map((alternate) =>
+          lavenstein(alternate, header),
+        ) || []),
+      ],
+    );
+    return distance < acc.distance || acc.distance === undefined
+      ? ({ value: field.key, distance } as AutoMatchAccumulator<T>)
+      : acc;
+  }, {} as AutoMatchAccumulator<T>);
+  return smallestValue.distance <= autoMapDistance
+    ? smallestValue.value
+    : undefined;
+};

front/src/modules/spreadsheet-import/utils/findUnmatchedRequiredFields.ts

@@ -0,0 +1,18 @@
+import type { Columns } from '@/spreadsheet-import/components/steps/MatchColumnsStep/MatchColumnsStep';
+import type { Fields } from '@/spreadsheet-import/types';
+
+export const findUnmatchedRequiredFields = <T extends string>(
+  fields: Fields<T>,
+  columns: Columns<T>,
+) =>
+  fields
+    .filter((field) =>
+      field.validations?.some((validation) => validation.rule === 'required'),
+    )
+    .filter(
+      (field) =>
+        columns.findIndex(
+          (column) => 'value' in column && column.value === field.key,
+        ) === -1,
+    )
+    .map((field) => field.label) || [];

front/src/modules/spreadsheet-import/utils/generateExampleRow.ts

@@ -0,0 +1,14 @@
+import type { Field, Fields } from '@/spreadsheet-import/types';
+
+const titleMap: Record<Field<string>['fieldType']['type'], string> = {
+  checkbox: 'Boolean',
+  select: 'Options',
+  input: 'Text',
+};
+
+export const generateExampleRow = <T extends string>(fields: Fields<T>) => [
+  fields.reduce((acc, field) => {
+    acc[field.key as T] = field.example || titleMap[field.fieldType.type];
+    return acc;
+  }, {} as Record<T, string>),
+];

front/src/modules/spreadsheet-import/utils/getFieldOptions.ts

@@ -0,0 +1,12 @@
+import type { Fields } from '@/spreadsheet-import/types';
+
+export const getFieldOptions = <T extends string>(
+  fields: Fields<T>,
+  fieldKey: string,
+) => {
+  const field = fields.find(({ key }) => fieldKey === key);
+  if (!field) {
+    return [];
+  }
+  return field.fieldType.type === 'select' ? field.fieldType.options : [];
+};

front/src/modules/spreadsheet-import/utils/getMatchedColumns.ts

@@ -0,0 +1,48 @@
+import lavenstein from 'js-levenshtein';
+
+import type {
+  Column,
+  Columns,
+  MatchColumnsProps,
+} from '@/spreadsheet-import/components/steps/MatchColumnsStep/MatchColumnsStep';
+import type { Field, Fields } from '@/spreadsheet-import/types';
+
+import { findMatch } from './findMatch';
+import { setColumn } from './setColumn';
+
+export const getMatchedColumns = <T extends string>(
+  columns: Columns<T>,
+  fields: Fields<T>,
+  data: MatchColumnsProps<T>['data'],
+  autoMapDistance: number,
+) =>
+  columns.reduce<Column<T>[]>((arr, column) => {
+    const autoMatch = findMatch(column.header, fields, autoMapDistance);
+    if (autoMatch) {
+      const field = fields.find((field) => field.key === autoMatch) as Field<T>;
+      const duplicateIndex = arr.findIndex(
+        (column) => 'value' in column && column.value === field.key,
+      );
+      const duplicate = arr[duplicateIndex];
+      if (duplicate && 'value' in duplicate) {
+        return lavenstein(duplicate.value, duplicate.header) <
+          lavenstein(autoMatch, column.header)
+          ? [
+              ...arr.slice(0, duplicateIndex),
+              setColumn(arr[duplicateIndex], field, data),
+              ...arr.slice(duplicateIndex + 1),
+              setColumn(column),
+            ]
+          : [
+              ...arr.slice(0, duplicateIndex),
+              setColumn(arr[duplicateIndex]),
+              ...arr.slice(duplicateIndex + 1),
+              setColumn(column, field, data),
+            ];
+      } else {
+        return [...arr, setColumn(column, field, data)];
+      }
+    } else {
+      return [...arr, column];
+    }
+  }, []);

front/src/modules/spreadsheet-import/utils/mapWorkbook.ts

@@ -0,0 +1,11 @@
+import * as XLSX from 'xlsx-ugnis';
+
+export const mapWorkbook = (workbook: XLSX.WorkBook, sheetName?: string) => {
+  const worksheet = workbook.Sheets[sheetName || workbook.SheetNames[0]];
+  const data = XLSX.utils.sheet_to_json(worksheet, {
+    header: 1,
+    blankrows: false,
+    raw: false,
+  });
+  return data as string[][];
+};

front/src/modules/spreadsheet-import/utils/normalizeCheckboxValue.ts

@@ -0,0 +1,13 @@
+const booleanWhitelist: Record<string, boolean> = {
+  yes: true,
+  no: false,
+  true: true,
+  false: false,
+};
+
+export const normalizeCheckboxValue = (value: string | undefined): boolean => {
+  if (value && value.toLowerCase() in booleanWhitelist) {
+    return booleanWhitelist[value.toLowerCase()];
+  }
+  return false;
+};

front/src/modules/spreadsheet-import/utils/normalizeTableData.ts

@@ -0,0 +1,67 @@
+import {
+  Columns,
+  ColumnType,
+} from '@/spreadsheet-import/components/steps/MatchColumnsStep/MatchColumnsStep';
+import type { Data, Fields, RawData } from '@/spreadsheet-import/types';
+
+import { normalizeCheckboxValue } from './normalizeCheckboxValue';
+
+export const normalizeTableData = <T extends string>(
+  columns: Columns<T>,
+  data: RawData[],
+  fields: Fields<T>,
+) =>
+  data.map((row) =>
+    columns.reduce((acc, column, index) => {
+      const curr = row[index];
+      switch (column.type) {
+        case ColumnType.matchedCheckbox: {
+          const field = fields.find((field) => field.key === column.value);
+
+          if (!field) {
+            return acc;
+          }
+
+          if (
+            'booleanMatches' in field.fieldType &&
+            Object.keys(field.fieldType).length
+          ) {
+            const booleanMatchKey = Object.keys(
+              field.fieldType.booleanMatches || [],
+            ).find((key) => key.toLowerCase() === curr?.toLowerCase());
+
+            if (!booleanMatchKey) {
+              return acc;
+            }
+
+            const booleanMatch =
+              field.fieldType.booleanMatches?.[booleanMatchKey];
+            acc[column.value] = booleanMatchKey
+              ? booleanMatch
+              : normalizeCheckboxValue(curr);
+          } else {
+            acc[column.value] = normalizeCheckboxValue(curr);
+          }
+          return acc;
+        }
+        case ColumnType.matched: {
+          acc[column.value] = curr === '' ? undefined : curr;
+          return acc;
+        }
+        case ColumnType.matchedSelect:
+        case ColumnType.matchedSelectOptions: {
+          const matchedOption = column.matchedOptions.find(
+            ({ entry }) => entry === curr,
+          );
+          acc[column.value] = matchedOption?.value || undefined;
+          return acc;
+        }
+        case ColumnType.empty:
+        case ColumnType.ignored: {
+          return acc;
+        }
+        default:
+          return acc;
+      }
+    }, {} as Data<T>),
+  );

front/src/modules/spreadsheet-import/utils/readFilesAsync.ts

@@ -0,0 +1,13 @@
+export const readFileAsync = (file: File) => {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+
+    reader.onload = () => {
+      resolve(reader.result);
+    };
+
+    reader.onerror = reject;
+
+    reader.readAsArrayBuffer(file);
+  });
+};

front/src/modules/spreadsheet-import/utils/setColumn.ts

@@ -0,0 +1,44 @@
+import {
+  Column,
+  ColumnType,
+  MatchColumnsProps,
+} from '@/spreadsheet-import/components/steps/MatchColumnsStep/MatchColumnsStep';
+import type { Field } from '@/spreadsheet-import/types';
+
+import { uniqueEntries } from './uniqueEntries';
+
+export const setColumn = <T extends string>(
+  oldColumn: Column<T>,
+  field?: Field<T>,
+  data?: MatchColumnsProps<T>['data'],
+): Column<T> => {
+  switch (field?.fieldType.type) {
+    case 'select':
+      return {
+        ...oldColumn,
+        type: ColumnType.matchedSelect,
+        value: field.key,
+        matchedOptions: uniqueEntries(data || [], oldColumn.index),
+      };
+    case 'checkbox':
+      return {
+        index: oldColumn.index,
+        type: ColumnType.matchedCheckbox,
+        value: field.key,
+        header: oldColumn.header,
+      };
+    case 'input':
+      return {
+        index: oldColumn.index,
+        type: ColumnType.matched,
+        value: field.key,
+        header: oldColumn.header,
+      };
+    default:
+      return {
+        index: oldColumn.index,
+        header: oldColumn.header,
+        type: ColumnType.empty,
+      };
+  }
+};

front/src/modules/spreadsheet-import/utils/setIgnoreColumn.ts

@@ -0,0 +1,13 @@
+import {
+  Column,
+  ColumnType,
+} from '@/spreadsheet-import/components/steps/MatchColumnsStep/MatchColumnsStep';
+
+export const setIgnoreColumn = <T extends string>({
+  header,
+  index,
+}: Column<T>): Column<T> => ({
+  header,
+  index,
+  type: ColumnType.ignored,
+});

front/src/modules/spreadsheet-import/utils/setSubColumn.ts

@@ -0,0 +1,30 @@
+import {
+  ColumnType,
+  MatchedOptions,
+  MatchedSelectColumn,
+  MatchedSelectOptionsColumn,
+} from '@/spreadsheet-import/components/steps/MatchColumnsStep/MatchColumnsStep';
+
+export const setSubColumn = <T>(
+  oldColumn: MatchedSelectColumn<T> | MatchedSelectOptionsColumn<T>,
+  entry: string,
+  value: string,
+): MatchedSelectColumn<T> | MatchedSelectOptionsColumn<T> => {
+  const options = oldColumn.matchedOptions.map((option) =>
+    option.entry === entry ? { ...option, value } : option,
+  );
+  const allMathced = options.every(({ value }) => !!value);
+  if (allMathced) {
+    return {
+      ...oldColumn,
+      matchedOptions: options as MatchedOptions<T>[],
+      type: ColumnType.matchedSelectOptions,
+    };
+  } else {
+    return {
+      ...oldColumn,
+      matchedOptions: options as MatchedOptions<T>[],
+      type: ColumnType.matchedSelect,
+    };
+  }
+};

front/src/modules/spreadsheet-import/utils/uniqueEntries.ts

@@ -0,0 +1,15 @@
+import uniqBy from 'lodash/uniqBy';
+
+import type {
+  MatchColumnsProps,
+  MatchedOptions,
+} from '@/spreadsheet-import/components/steps/MatchColumnsStep/MatchColumnsStep';
+
+export const uniqueEntries = <T extends string>(
+  data: MatchColumnsProps<T>['data'],
+  index: number,
+): Partial<MatchedOptions<T>>[] =>
+  uniqBy(
+    data.map((row) => ({ entry: row[index] })),
+    'entry',
+  ).filter(({ entry }) => !!entry);