Changed the auto matching of columns in import (#12181)

This PR changes the way we do automatching in the import feature. It uses [Fuse.js](https://www.fusejs.io/) to do a fuzzy text search on fields and sub-fields. The labels of sub-fields are now derived from the common config constant we have for sub-fields.
2025-05-23 18:33:18 +02:00
parent f7ccb5d207
commit 371fdba1f8
9 changed files with 121 additions and 46 deletions
--- a/package.json
+++ b/package.json
@ -94,6 +94,7 @@
    "facepaint": "^1.2.1",
    "file-type": "16.5.4",
    "framer-motion": "^11.18.0",
+    "fuse.js": "^7.1.0",
    "googleapis": "105",
    "graphiql": "^3.1.1",
    "graphql": "16.8.0",
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/constants/CompositeFieldImportLabels.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/constants/CompositeFieldImportLabels.ts
@ -9,43 +9,80 @@ import {
  FieldRichTextV2Value,
 } from '@/object-record/record-field/types/FieldMetadata';
 import { CompositeFieldLabels } from '@/object-record/spreadsheet-import/types/CompositeFieldLabels';
+import { SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS } from '@/settings/data-model/constants/SettingsCompositeFieldTypeConfigs';
 import { FieldMetadataType } from '~/generated-metadata/graphql';

 export const COMPOSITE_FIELD_IMPORT_LABELS = {
  [FieldMetadataType.FULL_NAME]: {
-    firstNameLabel: 'First Name',
-    lastNameLabel: 'Last Name',
+    firstNameLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.FULL_NAME.labelBySubField.firstName,
+    lastNameLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.FULL_NAME.labelBySubField.lastName,
  } satisfies CompositeFieldLabels<FieldFullNameValue>,
  [FieldMetadataType.CURRENCY]: {
-    currencyCodeLabel: 'Currency Code',
-    amountMicrosLabel: 'Amount',
+    currencyCodeLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.CURRENCY.labelBySubField
+        .currencyCode,
+    amountMicrosLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.CURRENCY.labelBySubField
+        .amountMicros,
  } satisfies CompositeFieldLabels<FieldCurrencyValue>,
  [FieldMetadataType.ADDRESS]: {
-    addressStreet1Label: 'Address 1',
-    addressStreet2Label: 'Address 2',
-    addressCityLabel: 'City',
-    addressPostcodeLabel: 'Post Code',
-    addressStateLabel: 'State',
-    addressCountryLabel: 'Country',
-    addressLatLabel: 'Latitude',
-    addressLngLabel: 'Longitude',
-  } satisfies CompositeFieldLabels<FieldAddressValue>,
+    addressStreet1Label:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressStreet1,
+    addressStreet2Label:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressStreet2,
+    addressCityLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField.addressCity,
+    addressPostcodeLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressPostcode,
+    addressStateLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressState,
+    addressCountryLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField
+        .addressCountry,
+  } satisfies Omit<
+    CompositeFieldLabels<FieldAddressValue>,
+    'addressLatLabel' | 'addressLngLabel'
+  >,
  [FieldMetadataType.LINKS]: {
-    // primaryLinkLabelLabel excluded from composite field import labels since it's not used in Links input
-    primaryLinkUrlLabel: 'Link URL',
+    primaryLinkUrlLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.LINKS.labelBySubField
+        .primaryLinkUrl,
+    secondaryLinksLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.LINKS.labelBySubField
+        .secondaryLinks,
  } satisfies Partial<CompositeFieldLabels<FieldLinksValue>>,
  [FieldMetadataType.EMAILS]: {
-    primaryEmailLabel: 'Email',
+    primaryEmailLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.EMAILS.labelBySubField.primaryEmail,
+    additionalEmailsLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.EMAILS.labelBySubField
+        .additionalEmails,
  } satisfies Partial<CompositeFieldLabels<FieldEmailsValue>>,
  [FieldMetadataType.PHONES]: {
-    primaryPhoneCountryCodeLabel: 'Phone country code',
-    primaryPhoneNumberLabel: 'Phone number',
+    primaryPhoneCountryCodeLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.PHONES.labelBySubField
+        .primaryPhoneCountryCode,
+    primaryPhoneNumberLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.PHONES.labelBySubField
+        .primaryPhoneNumber,
  } satisfies Partial<CompositeFieldLabels<FieldPhonesValue>>,
  [FieldMetadataType.RICH_TEXT_V2]: {
-    blocknoteLabel: 'BlockNote',
-    markdownLabel: 'Markdown',
+    blocknoteLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.RICH_TEXT_V2.labelBySubField
+        .blocknote,
+    markdownLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.RICH_TEXT_V2.labelBySubField
+        .markdown,
  } satisfies Partial<CompositeFieldLabels<FieldRichTextV2Value>>,
  [FieldMetadataType.ACTOR]: {
-    sourceLabel: 'Source',
+    sourceLabel:
+      SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ACTOR.labelBySubField.source,
+    nameLabel: SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ACTOR.labelBySubField.name,
  } satisfies Partial<CompositeFieldLabels<FieldActorValue>>,
 };
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts
@ -30,8 +30,6 @@ export const buildRecordFromImportedStructuredRow = ({
    ADDRESS: {
      addressCityLabel,
      addressCountryLabel,
-      addressLatLabel,
-      addressLngLabel,
      addressPostcodeLabel,
      addressStateLabel,
      addressStreet1Label,
@ -88,9 +86,7 @@ export const buildRecordFromImportedStructuredRow = ({
                `${addressPostcodeLabel} (${field.name})`
              ] ||
              importedStructuredRow[`${addressStateLabel} (${field.name})`] ||
-              importedStructuredRow[`${addressCountryLabel} (${field.name})`] ||
-              importedStructuredRow[`${addressLatLabel} (${field.name})`] ||
-              importedStructuredRow[`${addressLngLabel} (${field.name})`],
+              importedStructuredRow[`${addressCountryLabel} (${field.name})`],
          )
        ) {
          recordToBuild[field.name] = {
@ -112,13 +108,7 @@ export const buildRecordFromImportedStructuredRow = ({
            addressCountry: castToString(
              importedStructuredRow[`${addressCountryLabel} (${field.name})`],
            ),
-            addressLat: Number(
-              importedStructuredRow[`${addressLatLabel} (${field.name})`],
-            ),
-            addressLng: Number(
-              importedStructuredRow[`${addressLngLabel} (${field.name})`],
-            ),
-          } satisfies FieldAddressValue;
+          } satisfies Partial<FieldAddressValue>;
        }
        break;
      }
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/getSpreadSheetFieldValidationDefinitions.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/getSpreadSheetFieldValidationDefinitions.ts
@ -1,3 +1,4 @@
+import { FieldLinksValue } from '@/object-record/record-field/types/FieldMetadata';
 import { SpreadsheetImportFieldValidationDefinition } from '@/spreadsheet-import/types';
 import { absoluteUrlSchema, isDefined, isValidUuid } from 'twenty-shared/utils';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
@ -50,9 +51,11 @@ export const getSpreadSheetFieldValidationDefinitions = (
    case FieldMetadataType.LINKS:
      return [
        {
-          rule: 'function',
-          isValid: (value: string) =>
-            absoluteUrlSchema.safeParse(value).success,
+          rule: 'object',
+          isValid: ({
+            primaryLinkUrl,
+          }: Pick<FieldLinksValue, 'primaryLinkUrl' | 'secondaryLinks'>) =>
+            absoluteUrlSchema.safeParse(primaryLinkUrl).success,
          errorMessage: fieldName + ' is not valid',
          level: 'error',
        },
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx
@ -6,7 +6,6 @@ import { StepNavigationButton } from '@/spreadsheet-import/components/StepNaviga
 import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
 import { ImportedRow, ImportedStructuredRow } from '@/spreadsheet-import/types';
 import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields';
-import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns';
 import { normalizeTableData } from '@/spreadsheet-import/utils/normalizeTableData';
 import { setColumn } from '@/spreadsheet-import/utils/setColumn';
 import { setIgnoreColumn } from '@/spreadsheet-import/utils/setIgnoreColumn';
@ -26,6 +25,7 @@ import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn'
 import { SpreadsheetColumnType } from '@/spreadsheet-import/types/SpreadsheetColumnType';
 import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
 import { SpreadsheetImportField } from '@/spreadsheet-import/types/SpreadsheetImportField';
+import { getMatchedColumnsWithFuse } from '@/spreadsheet-import/utils/getMatchedColumnsWithFuse';
 import { ScrollWrapper } from '@/ui/utilities/scroll/components/ScrollWrapper';
 import { Trans, useLingui } from '@lingui/react/macro';
 import { useRecoilState } from 'recoil';
@ -82,8 +82,7 @@ export const MatchColumnsStep = <T extends string>({
  const { enqueueDialog } = useDialogManager();
  const { enqueueSnackBar } = useSnackBar();
  const dataExample = data.slice(0, 2);
-  const { fields, autoMapHeaders, autoMapDistance } =
-    useSpreadsheetImportInternal<T>();
+  const { fields, autoMapHeaders } = useSpreadsheetImportInternal<T>();
  const [isLoading, setIsLoading] = useState(false);
  const [columns, setColumns] = useRecoilState(
    initialComputedColumnsSelector(headerValues),
@ -264,7 +263,13 @@ export const MatchColumnsStep = <T extends string>({
      (column) => column.type === SpreadsheetColumnType.empty,
    );
    if (autoMapHeaders && isInitialColumnsState) {
-      setColumns(getMatchedColumns(columns, fields, data, autoMapDistance));
+      const { matchedColumns } = getMatchedColumnsWithFuse(
+        columns,
+        fields,
+        data,
+      );
+
+      setColumns(matchedColumns);
    }
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, []);
@ -275,7 +280,7 @@ export const MatchColumnsStep = <T extends string>({
        <StyledContent>
          <Heading
            title={t`Match Columns`}
-            description={t`Select the correct field for each column you'd like to import.`}
+            description={t`⚠️ Please verify the auto mapping of the columns. You can also ignore or change the mapping of the columns.`}
          />
          <ColumnGrid
            columns={columns}
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn.tsx
@ -20,7 +20,7 @@ const getExpandableContainerTitle = <T extends string>(

  return `Match ${fieldLabel} (${
    'matchedOptions' in column &&
-    column.matchedOptions.filter((option) => !isDefined(option.value)).length
+    column.matchedOptions?.filter((option) => !isDefined(option.value)).length
  } Unmatched)`;
 };

@ -70,7 +70,7 @@ export const UnmatchColumn = <T extends string>({
        containAnimation
      >
        <StyledContentWrapper>
-          {column.matchedOptions.map((option) => (
+          {column.matchedOptions?.map((option) => (
            <SubMatchingSelectRow
              option={option}
              column={column}
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/ValidationStep/ValidationStep.tsx
@ -25,10 +25,10 @@ import {
 // @ts-expect-error Todo: remove usage of react-data-grid`
 import { RowsChangeData } from 'react-data-grid';
 import { isDefined } from 'twenty-shared/utils';
+import { IconTrash } from 'twenty-ui/display';
+import { Button, Toggle } from 'twenty-ui/input';
 import { generateColumns } from './components/columns';
 import { ImportedStructuredRowMetadata } from './types';
-import { Button, Toggle } from 'twenty-ui/input';
-import { IconTrash } from 'twenty-ui/display';

 const StyledContent = styled(Modal.Content)`
  padding-left: ${({ theme }) => theme.spacing(6)};
--- a/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumnsWithFuse.ts
+++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumnsWithFuse.ts
@ -0,0 +1,38 @@
+import { MatchColumnsStepProps } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
+
+import { SpreadsheetImportFields } from '@/spreadsheet-import/types';
+import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn';
+import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
+import { setColumn } from '@/spreadsheet-import/utils/setColumn';
+import Fuse from 'fuse.js';
+import { isDefined } from 'twenty-shared/utils';
+
+export const getMatchedColumnsWithFuse = <T extends string>(
+  columns: SpreadsheetColumns<T>,
+  fields: SpreadsheetImportFields<T>,
+  data: MatchColumnsStepProps['data'],
+) => {
+  const matchedColumns: SpreadsheetColumn<T>[] = [];
+
+  const fieldsToSearch = new Fuse(fields, {
+    keys: ['label'],
+    includeScore: true,
+    threshold: 0.3,
+  });
+
+  for (const column of columns) {
+    const fieldsThatMatch = fieldsToSearch.search(column.header);
+
+    const firstMatch = fieldsThatMatch[0]?.item ?? null;
+
+    if (isDefined(firstMatch)) {
+      const newColumn = setColumn(column, firstMatch as any, data);
+
+      matchedColumns.push(newColumn);
+    } else {
+      matchedColumns.push(column);
+    }
+  }
+
+  return { matchedColumns };
+};
--- a/yarn.lock
+++ b/yarn.lock
@ -35813,7 +35813,7 @@ __metadata:
  languageName: node
  linkType: hard

-"fuse.js@npm:^7.0.0":
+"fuse.js@npm:^7.0.0, fuse.js@npm:^7.1.0":
  version: 7.1.0
  resolution: "fuse.js@npm:7.1.0"
  checksum: 10c0/c0d1b1d192a4bdf3eade897453ddd28aff96b70bf3e49161a45880f9845ebaee97265595db633776700a5bcf8942223c752754a848d70c508c3c9fd997faad1e
@ -55988,6 +55988,7 @@ __metadata:
    facepaint: "npm:^1.2.1"
    file-type: "npm:16.5.4"
    framer-motion: "npm:^11.18.0"
+    fuse.js: "npm:^7.1.0"
    googleapis: "npm:105"
    graphiql: "npm:^3.1.1"
    graphql: "npm:16.8.0"