From 371fdba1f871c9720972e65c32b395f9d3fb5c6a Mon Sep 17 00:00:00 2001 From: Lucas Bordeau Date: Fri, 23 May 2025 18:33:18 +0200 Subject: [PATCH] Changed the auto matching of columns in import (#12181) This PR changes the way we do automatching in the import feature. It uses [Fuse.js](https://www.fusejs.io/) to do a fuzzy text search on fields and sub-fields. The labels of sub-fields are now derived from the common config constant we have for sub-fields. --- package.json | 1 + .../constants/CompositeFieldImportLabels.ts | 79 ++++++++++++++----- .../buildRecordFromImportedStructuredRow.ts | 14 +--- ...etSpreadSheetFieldValidationDefinitions.ts | 9 ++- .../MatchColumnsStep/MatchColumnsStep.tsx | 15 ++-- .../components/UnmatchColumn.tsx | 4 +- .../ValidationStep/ValidationStep.tsx | 4 +- .../utils/getMatchedColumnsWithFuse.ts | 38 +++++++++ yarn.lock | 3 +- 9 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumnsWithFuse.ts diff --git a/package.json b/package.json index 00fc48d94..27ccdf430 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "facepaint": "^1.2.1", "file-type": "16.5.4", "framer-motion": "^11.18.0", + "fuse.js": "^7.1.0", "googleapis": "105", "graphiql": "^3.1.1", "graphql": "16.8.0", diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/constants/CompositeFieldImportLabels.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/constants/CompositeFieldImportLabels.ts index 41d19c33b..26c630d31 100644 --- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/constants/CompositeFieldImportLabels.ts +++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/constants/CompositeFieldImportLabels.ts @@ -9,43 +9,80 @@ import { FieldRichTextV2Value, } from '@/object-record/record-field/types/FieldMetadata'; import { CompositeFieldLabels } from '@/object-record/spreadsheet-import/types/CompositeFieldLabels'; +import { SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS } from '@/settings/data-model/constants/SettingsCompositeFieldTypeConfigs'; import { FieldMetadataType } from '~/generated-metadata/graphql'; export const COMPOSITE_FIELD_IMPORT_LABELS = { [FieldMetadataType.FULL_NAME]: { - firstNameLabel: 'First Name', - lastNameLabel: 'Last Name', + firstNameLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.FULL_NAME.labelBySubField.firstName, + lastNameLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.FULL_NAME.labelBySubField.lastName, } satisfies CompositeFieldLabels, [FieldMetadataType.CURRENCY]: { - currencyCodeLabel: 'Currency Code', - amountMicrosLabel: 'Amount', + currencyCodeLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.CURRENCY.labelBySubField + .currencyCode, + amountMicrosLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.CURRENCY.labelBySubField + .amountMicros, } satisfies CompositeFieldLabels, [FieldMetadataType.ADDRESS]: { - addressStreet1Label: 'Address 1', - addressStreet2Label: 'Address 2', - addressCityLabel: 'City', - addressPostcodeLabel: 'Post Code', - addressStateLabel: 'State', - addressCountryLabel: 'Country', - addressLatLabel: 'Latitude', - addressLngLabel: 'Longitude', - } satisfies CompositeFieldLabels, + addressStreet1Label: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField + .addressStreet1, + addressStreet2Label: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField + .addressStreet2, + addressCityLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField.addressCity, + addressPostcodeLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField + .addressPostcode, + addressStateLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField + .addressState, + addressCountryLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ADDRESS.labelBySubField + .addressCountry, + } satisfies Omit< + CompositeFieldLabels, + 'addressLatLabel' | 'addressLngLabel' + >, [FieldMetadataType.LINKS]: { - // primaryLinkLabelLabel excluded from composite field import labels since it's not used in Links input - primaryLinkUrlLabel: 'Link URL', + primaryLinkUrlLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.LINKS.labelBySubField + .primaryLinkUrl, + secondaryLinksLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.LINKS.labelBySubField + .secondaryLinks, } satisfies Partial>, [FieldMetadataType.EMAILS]: { - primaryEmailLabel: 'Email', + primaryEmailLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.EMAILS.labelBySubField.primaryEmail, + additionalEmailsLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.EMAILS.labelBySubField + .additionalEmails, } satisfies Partial>, [FieldMetadataType.PHONES]: { - primaryPhoneCountryCodeLabel: 'Phone country code', - primaryPhoneNumberLabel: 'Phone number', + primaryPhoneCountryCodeLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.PHONES.labelBySubField + .primaryPhoneCountryCode, + primaryPhoneNumberLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.PHONES.labelBySubField + .primaryPhoneNumber, } satisfies Partial>, [FieldMetadataType.RICH_TEXT_V2]: { - blocknoteLabel: 'BlockNote', - markdownLabel: 'Markdown', + blocknoteLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.RICH_TEXT_V2.labelBySubField + .blocknote, + markdownLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.RICH_TEXT_V2.labelBySubField + .markdown, } satisfies Partial>, [FieldMetadataType.ACTOR]: { - sourceLabel: 'Source', + sourceLabel: + SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ACTOR.labelBySubField.source, + nameLabel: SETTINGS_COMPOSITE_FIELD_TYPE_CONFIGS.ACTOR.labelBySubField.name, } satisfies Partial>, }; diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts index cbacd4581..156bcc4ba 100644 --- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts +++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts @@ -30,8 +30,6 @@ export const buildRecordFromImportedStructuredRow = ({ ADDRESS: { addressCityLabel, addressCountryLabel, - addressLatLabel, - addressLngLabel, addressPostcodeLabel, addressStateLabel, addressStreet1Label, @@ -88,9 +86,7 @@ export const buildRecordFromImportedStructuredRow = ({ `${addressPostcodeLabel} (${field.name})` ] || importedStructuredRow[`${addressStateLabel} (${field.name})`] || - importedStructuredRow[`${addressCountryLabel} (${field.name})`] || - importedStructuredRow[`${addressLatLabel} (${field.name})`] || - importedStructuredRow[`${addressLngLabel} (${field.name})`], + importedStructuredRow[`${addressCountryLabel} (${field.name})`], ) ) { recordToBuild[field.name] = { @@ -112,13 +108,7 @@ export const buildRecordFromImportedStructuredRow = ({ addressCountry: castToString( importedStructuredRow[`${addressCountryLabel} (${field.name})`], ), - addressLat: Number( - importedStructuredRow[`${addressLatLabel} (${field.name})`], - ), - addressLng: Number( - importedStructuredRow[`${addressLngLabel} (${field.name})`], - ), - } satisfies FieldAddressValue; + } satisfies Partial; } break; } diff --git a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/getSpreadSheetFieldValidationDefinitions.ts b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/getSpreadSheetFieldValidationDefinitions.ts index 069e694e3..4ff296a22 100644 --- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/getSpreadSheetFieldValidationDefinitions.ts +++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/getSpreadSheetFieldValidationDefinitions.ts @@ -1,3 +1,4 @@ +import { FieldLinksValue } from '@/object-record/record-field/types/FieldMetadata'; import { SpreadsheetImportFieldValidationDefinition } from '@/spreadsheet-import/types'; import { absoluteUrlSchema, isDefined, isValidUuid } from 'twenty-shared/utils'; import { FieldMetadataType } from '~/generated-metadata/graphql'; @@ -50,9 +51,11 @@ export const getSpreadSheetFieldValidationDefinitions = ( case FieldMetadataType.LINKS: return [ { - rule: 'function', - isValid: (value: string) => - absoluteUrlSchema.safeParse(value).success, + rule: 'object', + isValid: ({ + primaryLinkUrl, + }: Pick) => + absoluteUrlSchema.safeParse(primaryLinkUrl).success, errorMessage: fieldName + ' is not valid', level: 'error', }, diff --git a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx index 56c6f0d8a..b4aae976c 100644 --- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx +++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx @@ -6,7 +6,6 @@ import { StepNavigationButton } from '@/spreadsheet-import/components/StepNaviga import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; import { ImportedRow, ImportedStructuredRow } from '@/spreadsheet-import/types'; import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields'; -import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns'; import { normalizeTableData } from '@/spreadsheet-import/utils/normalizeTableData'; import { setColumn } from '@/spreadsheet-import/utils/setColumn'; import { setIgnoreColumn } from '@/spreadsheet-import/utils/setIgnoreColumn'; @@ -26,6 +25,7 @@ import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn' import { SpreadsheetColumnType } from '@/spreadsheet-import/types/SpreadsheetColumnType'; import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns'; import { SpreadsheetImportField } from '@/spreadsheet-import/types/SpreadsheetImportField'; +import { getMatchedColumnsWithFuse } from '@/spreadsheet-import/utils/getMatchedColumnsWithFuse'; import { ScrollWrapper } from '@/ui/utilities/scroll/components/ScrollWrapper'; import { Trans, useLingui } from '@lingui/react/macro'; import { useRecoilState } from 'recoil'; @@ -82,8 +82,7 @@ export const MatchColumnsStep = ({ const { enqueueDialog } = useDialogManager(); const { enqueueSnackBar } = useSnackBar(); const dataExample = data.slice(0, 2); - const { fields, autoMapHeaders, autoMapDistance } = - useSpreadsheetImportInternal(); + const { fields, autoMapHeaders } = useSpreadsheetImportInternal(); const [isLoading, setIsLoading] = useState(false); const [columns, setColumns] = useRecoilState( initialComputedColumnsSelector(headerValues), @@ -264,7 +263,13 @@ export const MatchColumnsStep = ({ (column) => column.type === SpreadsheetColumnType.empty, ); if (autoMapHeaders && isInitialColumnsState) { - setColumns(getMatchedColumns(columns, fields, data, autoMapDistance)); + const { matchedColumns } = getMatchedColumnsWithFuse( + columns, + fields, + data, + ); + + setColumns(matchedColumns); } // eslint-disable-next-line react-hooks/exhaustive-deps }, []); @@ -275,7 +280,7 @@ export const MatchColumnsStep = ({ ( return `Match ${fieldLabel} (${ 'matchedOptions' in column && - column.matchedOptions.filter((option) => !isDefined(option.value)).length + column.matchedOptions?.filter((option) => !isDefined(option.value)).length } Unmatched)`; }; @@ -70,7 +70,7 @@ export const UnmatchColumn = ({ containAnimation > - {column.matchedOptions.map((option) => ( + {column.matchedOptions?.map((option) => ( theme.spacing(6)}; diff --git a/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumnsWithFuse.ts b/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumnsWithFuse.ts new file mode 100644 index 000000000..7a3748285 --- /dev/null +++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/getMatchedColumnsWithFuse.ts @@ -0,0 +1,38 @@ +import { MatchColumnsStepProps } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; + +import { SpreadsheetImportFields } from '@/spreadsheet-import/types'; +import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn'; +import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns'; +import { setColumn } from '@/spreadsheet-import/utils/setColumn'; +import Fuse from 'fuse.js'; +import { isDefined } from 'twenty-shared/utils'; + +export const getMatchedColumnsWithFuse = ( + columns: SpreadsheetColumns, + fields: SpreadsheetImportFields, + data: MatchColumnsStepProps['data'], +) => { + const matchedColumns: SpreadsheetColumn[] = []; + + const fieldsToSearch = new Fuse(fields, { + keys: ['label'], + includeScore: true, + threshold: 0.3, + }); + + for (const column of columns) { + const fieldsThatMatch = fieldsToSearch.search(column.header); + + const firstMatch = fieldsThatMatch[0]?.item ?? null; + + if (isDefined(firstMatch)) { + const newColumn = setColumn(column, firstMatch as any, data); + + matchedColumns.push(newColumn); + } else { + matchedColumns.push(column); + } + } + + return { matchedColumns }; +}; diff --git a/yarn.lock b/yarn.lock index 712aa86c7..e5f789ede 100644 --- a/yarn.lock +++ b/yarn.lock @@ -35813,7 +35813,7 @@ __metadata: languageName: node linkType: hard -"fuse.js@npm:^7.0.0": +"fuse.js@npm:^7.0.0, fuse.js@npm:^7.1.0": version: 7.1.0 resolution: "fuse.js@npm:7.1.0" checksum: 10c0/c0d1b1d192a4bdf3eade897453ddd28aff96b70bf3e49161a45880f9845ebaee97265595db633776700a5bcf8942223c752754a848d70c508c3c9fd997faad1e @@ -55988,6 +55988,7 @@ __metadata: facepaint: "npm:^1.2.1" file-type: "npm:16.5.4" framer-motion: "npm:^11.18.0" + fuse.js: "npm:^7.1.0" googleapis: "npm:105" graphiql: "npm:^3.1.1" graphql: "npm:16.8.0"