Changed the auto matching of columns in import (#12181)

This PR changes the way we do automatching in the import feature.

It uses [Fuse.js](https://www.fusejs.io/) to do a fuzzy text search on
fields and sub-fields.

The labels of sub-fields are now derived from the common config constant
we have for sub-fields.

package.json

@@ -94,6 +94,7 @@
     "facepaint": "^1.2.1",
     "file-type": "16.5.4",
     "framer-motion": "^11.18.0",
+    "fuse.js": "^7.1.0",
     "googleapis": "105",
     "graphiql": "^3.1.1",
     "graphql": "16.8.0",

packages/twenty-front/src/modules/object-record/spreadsheet-import/constants/CompositeFieldImportLabels.ts

@@ -9,43 +9,80 @@ import {
   FieldRichTextV2Value,
 } from '@/object-record/record-field/types/FieldMetadata';
 import { CompositeFieldLabels } from '@/object-record/spreadsheet-import/types/CompositeFieldLabels';
+import { SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS } from '@/settings/data-model/constants/SettingsCompositeFieldTypeConfigs';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
 
 export const COMPOSITE_FIELD_IMPORT_LABELS = {
   [FieldMetadataType.FULL_NAME]: {
-    firstNameLabel: 'First Name',
+    firstNameLabel:
-    lastNameLabel: 'Last Name',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.FULL_NAME.labelBySubField.firstName,
+    lastNameLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.FULL_NAME.labelBySubField.lastName,
   } satisfies CompositeFieldLabels<FieldFullNameValue>,
   [FieldMetadataType.CURRENCY]: {
-    currencyCodeLabel: 'Currency Code',
+    currencyCodeLabel:
-    amountMicrosLabel: 'Amount',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.CURRENCY.labelBySubField
+        .currencyCode,
+    amountMicrosLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.CURRENCY.labelBySubField
+        .amountMicros,
   } satisfies CompositeFieldLabels<FieldCurrencyValue>,
   [FieldMetadataType.ADDRESS]: {
-    addressStreet1Label: 'Address 1',
+    addressStreet1Label:
-    addressStreet2Label: 'Address 2',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
-    addressCityLabel: 'City',
+        .addressStreet1,
-    addressPostcodeLabel: 'Post Code',
+    addressStreet2Label:
-    addressStateLabel: 'State',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
-    addressCountryLabel: 'Country',
+        .addressStreet2,
-    addressLatLabel: 'Latitude',
+    addressCityLabel:
-    addressLngLabel: 'Longitude',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField.addressCity,
-  } satisfies CompositeFieldLabels<FieldAddressValue>,
+    addressPostcodeLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressPostcode,
+    addressStateLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressState,
+    addressCountryLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressCountry,
+  } satisfies Omit<
+    CompositeFieldLabels<FieldAddressValue>,
+    'addressLatLabel' | 'addressLngLabel'
+  >,
   [FieldMetadataType.LINKS]: {
-    // primaryLinkLabelLabel excluded from composite field import labels since it's not used in Links input
+    primaryLinkUrlLabel:
-    primaryLinkUrlLabel: 'Link URL',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.LINKS.labelBySubField
+        .primaryLinkUrl,
+    secondaryLinksLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.LINKS.labelBySubField
+        .secondaryLinks,
   } satisfies Partial<CompositeFieldLabels<FieldLinksValue>>,
   [FieldMetadataType.EMAILS]: {
-    primaryEmailLabel: 'Email',
+    primaryEmailLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.EMAILS.labelBySubField.primaryEmail,
+    additionalEmailsLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.EMAILS.labelBySubField
+        .additionalEmails,
   } satisfies Partial<CompositeFieldLabels<FieldEmailsValue>>,
   [FieldMetadataType.PHONES]: {
-    primaryPhoneCountryCodeLabel: 'Phone country code',
+    primaryPhoneCountryCodeLabel:
-    primaryPhoneNumberLabel: 'Phone number',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.PHONES.labelBySubField
+        .primaryPhoneCountryCode,
+    primaryPhoneNumberLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.PHONES.labelBySubField
+        .primaryPhoneNumber,
   } satisfies Partial<CompositeFieldLabels<FieldPhonesValue>>,
   [FieldMetadataType.RICH_TEXT_V2]: {
-    blocknoteLabel: 'BlockNote',
+    blocknoteLabel:
-    markdownLabel: 'Markdown',
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.RICH_TEXT_V2.labelBySubField
+        .blocknote,
+    markdownLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.RICH_TEXT_V2.labelBySubField
+        .markdown,
   } satisfies Partial<CompositeFieldLabels<FieldRichTextV2Value>>,
   [FieldMetadataType.ACTOR]: {
-    sourceLabel: 'Source',
+    sourceLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ACTOR.labelBySubField.source,
+    nameLabel: SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ACTOR.labelBySubField.name,
   } satisfies Partial<CompositeFieldLabels<FieldActorValue>>,
 };

packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts

@@ -30,8 +30,6 @@ export const buildRecordFromImportedStructuredRow = ({
     ADDRESS: {
       addressCityLabel,
       addressCountryLabel,
-      addressLatLabel,
-      addressLngLabel,
       addressPostcodeLabel,
       addressStateLabel,
       addressStreet1Label,
@@ -88,9 +86,7 @@ export const buildRecordFromImportedStructuredRow = ({
                 `${addressPostcodeLabel} (${field.name})`
               ] ||
               importedStructuredRow[`${addressStateLabel} (${field.name})`] ||
-              importedStructuredRow[`${addressCountryLabel} (${field.name})`] ||
+              importedStructuredRow[`${addressCountryLabel} (${field.name})`],
-              importedStructuredRow[`${addressLatLabel} (${field.name})`] ||
-              importedStructuredRow[`${addressLngLabel} (${field.name})`],
           )
         ) {
           recordToBuild[field.name] = {
@@ -112,13 +108,7 @@ export const buildRecordFromImportedStructuredRow = ({
             addressCountry: castToString(
               importedStructuredRow[`${addressCountryLabel} (${field.name})`],
             ),
-            addressLat: Number(
+          } satisfies Partial<FieldAddressValue>;
-              importedStructuredRow[`${addressLatLabel} (${field.name})`],
-            ),
-            addressLng: Number(
-              importedStructuredRow[`${addressLngLabel} (${field.name})`],
-            ),
-          } satisfies FieldAddressValue;
         }
         break;
       }

packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/getSpreadSheetFieldValidationDefinitions.ts

@@ -1,3 +1,4 @@
+import { FieldLinksValue } from '@/object-record/record-field/types/FieldMetadata';
 import { SpreadsheetImportFieldValidationDefinition } from '@/spreadsheet-import/types';
 import { absoluteUrlSchema, isDefined, isValidUuid } from 'twenty-shared/utils';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
@@ -50,9 +51,11 @@ export const getSpreadSheetFieldValidationDefinitions = (
     case FieldMetadataType.LINKS:
       return [
         {
-          rule: 'function',
+          rule: 'object',
-          isValid: (value: string) =>
+          isValid: ({
-            absoluteUrlSchema.safeParse(value).success,
+            primaryLinkUrl,
+          }: Pick<FieldLinksValue, 'primaryLinkUrl' | 'secondaryLinks'>) =>
+            absoluteUrlSchema.safeParse(primaryLinkUrl).success,
           errorMessage: fieldName + ' is not valid',
           level: 'error',
         },

packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx

@@ -6,7 +6,6 @@ import { StepNavigationButton } from '@/spreadsheet-import/components/StepNaviga
 import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
 import { ImportedRow, ImportedStructuredRow } from '@/spreadsheet-import/types';
 import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields';
-import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns';
 import { normalizeTableData } from '@/spreadsheet-import/utils/normalizeTableData';
 import { setColumn } from '@/spreadsheet-import/utils/setColumn';
 import { setIgnoreColumn } from '@/spreadsheet-import/utils/setIgnoreColumn';
@@ -26,6 +25,7 @@ import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn'
 import { SpreadsheetColumnType } from '@/spreadsheet-import/types/SpreadsheetColumnType';
 import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
 import { SpreadsheetImportField } from '@/spreadsheet-import/types/SpreadsheetImportField';
+import { getMatchedColumnsWithFuse } from '@/spreadsheet-import/utils/getMatchedColumnsWithFuse';
 import { ScrollWrapper } from '@/ui/utilities/scroll/components/ScrollWrapper';
 import { Trans, useLingui } from '@lingui/react/macro';
 import { useRecoilState } from 'recoil';
@@ -82,8 +82,7 @@ export const MatchColumnsStep = <T extends string>({
   const { enqueueDialog } = useDialogManager();
   const { enqueueSnackBar } = useSnackBar();
   const dataExample = data.slice(0, 2);
-  const { fields, autoMapHeaders, autoMapDistance } =
+  const { fields, autoMapHeaders } = useSpreadsheetImportInternal<T>();
-    useSpreadsheetImportInternal<T>();
   const [isLoading, setIsLoading] = useState(false);
   const [columns, setColumns] = useRecoilState(
     initialComputedColumnsSelector(headerValues),
@@ -264,7 +263,13 @@ export const MatchColumnsStep = <T extends string>({
       (column) => column.type === SpreadsheetColumnType.empty,
     );
     if (autoMapHeaders && isInitialColumnsState) {
-      setColumns(getMatchedColumns(columns, fields, data, autoMapDistance));
+      const { matchedColumns } = getMatchedColumnsWithFuse(
+        columns,
+        fields,
+        data,
+      );
+
+      setColumns(matchedColumns);
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
@@ -275,7 +280,7 @@ export const MatchColumnsStep = <T extends string>({
         <StyledContent>
           <Heading
             title={t`Match Columns`}
-            description={t`Select the correct field for each column you'd like to import.`}
+            description={t`⚠️ Please verify the auto mapping of the columns. You can also ignore or change the mapping of the columns.`}
           />
           <ColumnGrid
             columns={columns}

packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn.tsx

@@ -20,7 +20,7 @@ const getExpandableContainerTitle = <T extends string>(
 
   return `Match ${fieldLabel} (${
     'matchedOptions' in column &&
-    column.matchedOptions.filter((option) => !isDefined(option.value)).length
+    column.matchedOptions?.filter((option) => !isDefined(option.value)).length
   } Unmatched)`;
 };
 
@@ -70,7 +70,7 @@ export const UnmatchColumn = <T extends string>({
         containAnimation
       >
         <StyledContentWrapper>
-          {column.matchedOptions.map((option) => (
+          {column.matchedOptions?.map((option) => (
             <SubMatchingSelectRow
               option={option}
               column={column}

packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx

@@ -25,10 +25,10 @@ import {
 // @ts-expect-error Todo: remove usage of react-data-grid`
 import { RowsChangeData } from 'react-data-grid';
 import { isDefined } from 'twenty-shared/utils';
+import { IconTrash } from 'twenty-ui/display';
+import { Button, Toggle } from 'twenty-ui/input';
 import { generateColumns } from './components/columns';
 import { ImportedStructuredRowMetadata } from './types';
-import { Button, Toggle } from 'twenty-ui/input';
-import { IconTrash } from 'twenty-ui/display';
 
 const StyledContent = styled(Modal.Content)`
   padding-left: ${({ theme }) => theme.spacing(6)};

packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumnsWithFuse.ts

@@ -0,0 +1,38 @@
+import { MatchColumnsStepProps } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
+
+import { SpreadsheetImportFields } from '@/spreadsheet-import/types';
+import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn';
+import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
+import { setColumn } from '@/spreadsheet-import/utils/setColumn';
+import Fuse from 'fuse.js';
+import { isDefined } from 'twenty-shared/utils';
+
+export const getMatchedColumnsWithFuse = <T extends string>(
+  columns: SpreadsheetColumns<T>,
+  fields: SpreadsheetImportFields<T>,
+  data: MatchColumnsStepProps['data'],
+) => {
+  const matchedColumns: SpreadsheetColumn<T>[] = [];
+
+  const fieldsToSearch = new Fuse(fields, {
+    keys: ['label'],
+    includeScore: true,
+    threshold: 0.3,
+  });
+
+  for (const column of columns) {
+    const fieldsThatMatch = fieldsToSearch.search(column.header);
+
+    const firstMatch = fieldsThatMatch[0]?.item ?? null;
+
+    if (isDefined(firstMatch)) {
+      const newColumn = setColumn(column, firstMatch as any, data);
+
+      matchedColumns.push(newColumn);
+    } else {
+      matchedColumns.push(column);
+    }
+  }
+
+  return { matchedColumns };
+};

yarn.lock

@@ -35813,7 +35813,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fuse.js@npm:^7.0.0":
+"fuse.js@npm:^7.0.0, fuse.js@npm:^7.1.0":
   version: 7.1.0
   resolution: "fuse.js@npm:7.1.0"
   checksum: 10c0/c0d1b1d192a4bdf3eade897453ddd28aff96b70bf3e49161a45880f9845ebaee97265595db633776700a5bcf8942223c752754a848d70c508c3c9fd997faad1e
@@ -55988,6 +55988,7 @@ __metadata:
     facepaint: "npm:^1.2.1"
     file-type: "npm:16.5.4"
     framer-motion: "npm:^11.18.0"
+    fuse.js: "npm:^7.1.0"
     googleapis: "npm:105"
     graphiql: "npm:^3.1.1"
     graphql: "npm:16.8.0"