Refactor spreadsheet import (#11250)

Mostly renaming objects to avoid conflicts (it was painful because names
were too generic so you could cmd+replace easily)

Also refactoring `useBuildAvailableFieldsForImport`
This commit is contained in:
Félix Malfait
2025-03-28 07:56:51 +01:00
committed by GitHub
parent 9af2628264
commit e9e33c4d29
84 changed files with 960 additions and 916 deletions

View File

@ -5,9 +5,9 @@ import { Heading } from '@/spreadsheet-import/components/Heading';
import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton';
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
import {
Field,
ImportedRow,
ImportedStructuredRow,
SpreadsheetImportField,
} from '@/spreadsheet-import/types';
import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields';
import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns';
@ -25,6 +25,9 @@ import { initialComputedColumnsSelector } from '@/spreadsheet-import/steps/compo
import { UnmatchColumn } from '@/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumn';
import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep';
import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType';
import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn';
import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
import { SpreadsheetColumnType } from '@/spreadsheet-import/types/SpreadsheetColumnType';
import { ScrollWrapper } from '@/ui/utilities/scroll/components/ScrollWrapper';
import { Trans, useLingui } from '@lingui/react/macro';
import { useRecoilState } from 'recoil';
@ -68,68 +71,6 @@ export type MatchColumnsStepProps = {
onError: (message: string) => void;
};
export enum ColumnType {
empty,
ignored,
matched,
matchedCheckbox,
matchedSelect,
matchedSelectOptions,
}
export type MatchedOptions<T> = {
entry: string;
value?: T;
};
type EmptyColumn = { type: ColumnType.empty; index: number; header: string };
type IgnoredColumn = {
type: ColumnType.ignored;
index: number;
header: string;
};
type MatchedColumn<T> = {
type: ColumnType.matched;
index: number;
header: string;
value: T;
};
type MatchedSwitchColumn<T> = {
type: ColumnType.matchedCheckbox;
index: number;
header: string;
value: T;
};
export type MatchedSelectColumn<T> = {
type: ColumnType.matchedSelect;
index: number;
header: string;
value: T;
matchedOptions: Partial<MatchedOptions<T>>[];
};
export type MatchedSelectOptionsColumn<T> = {
type: ColumnType.matchedSelectOptions;
index: number;
header: string;
value: T;
matchedOptions: MatchedOptions<T>[];
};
export type Column<T extends string> =
| EmptyColumn
| IgnoredColumn
| MatchedColumn<T>
| MatchedSwitchColumn<T>
| MatchedSelectColumn<T>
| MatchedSelectOptionsColumn<T>;
export type Columns<T extends string> = Column<T>[];
export const MatchColumnsStep = <T extends string>({
data,
headerValues,
@ -179,7 +120,7 @@ export const MatchColumnsStep = <T extends string>({
const onChange = useCallback(
(value: T, columnIndex: number) => {
if (value === 'do-not-import') {
if (columns[columnIndex].type === ColumnType.ignored) {
if (columns[columnIndex].type === SpreadsheetColumnType.ignored) {
onRevertIgnore(columnIndex);
} else {
onIgnore(columnIndex);
@ -187,12 +128,12 @@ export const MatchColumnsStep = <T extends string>({
} else {
const field = fields.find(
(field) => field.key === value,
) as unknown as Field<T>;
) as unknown as SpreadsheetImportField<T>;
const existingFieldIndex = columns.findIndex(
(column) => 'value' in column && column.value === field.key,
);
setColumns(
columns.map<Column<string>>((column, index) => {
columns.map<SpreadsheetColumn<string>>((column, index) => {
if (columnIndex === index) {
return setColumn(column, field, data);
} else if (index === existingFieldIndex) {
@ -223,7 +164,7 @@ export const MatchColumnsStep = <T extends string>({
async (
values: ImportedStructuredRow<string>[],
rawData: ImportedRow[],
columns: Columns<string>,
columns: SpreadsheetColumns<string>,
) => {
try {
const data = await matchColumnsStepHook(values, rawData, columns);
@ -322,7 +263,7 @@ export const MatchColumnsStep = <T extends string>({
useEffect(() => {
const isInitialColumnsState = columns.every(
(column) => column.type === ColumnType.empty,
(column) => column.type === SpreadsheetColumnType.empty,
);
if (autoMapHeaders && isInitialColumnsState) {
setColumns(getMatchedColumns(columns, fields, data, autoMapDistance));

View File

@ -1,8 +1,7 @@
import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
import styled from '@emotion/styled';
import React from 'react';
import { Columns } from '../MatchColumnsStep';
const StyledGridContainer = styled.div`
align-items: center;
display: flex;
@ -93,17 +92,17 @@ const StyledGridHeader = styled.div<PositionProps>`
`;
type ColumnGridProps<T extends string> = {
columns: Columns<T>;
columns: SpreadsheetColumns<T>;
renderUserColumn: (
columns: Columns<T>,
columns: SpreadsheetColumns<T>,
columnIndex: number,
) => React.ReactNode;
renderTemplateColumn: (
columns: Columns<T>,
columns: SpreadsheetColumns<T>,
columnIndex: number,
) => React.ReactNode;
renderUnmatchedColumn: (
columns: Columns<T>,
columns: SpreadsheetColumns<T>,
columnIndex: number,
) => React.ReactNode;
};

View File

@ -2,20 +2,19 @@ import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
import { SelectOption } from '@/spreadsheet-import/types';
import { getFieldOptions } from '@/spreadsheet-import/utils/getFieldOptions';
import { SelectFieldHotkeyScope } from '@/object-record/select/types/SelectFieldHotkeyScope';
import {
SpreadsheetMatchedSelectColumn,
SpreadsheetMatchedSelectOptionsColumn,
} from '@/spreadsheet-import/types/SpreadsheetColumn';
import { SpreadsheetMatchedOptions } from '@/spreadsheet-import/types/SpreadsheetMatchedOptions';
import { SelectInput } from '@/ui/input/components/SelectInput';
import { useSetHotkeyScope } from '@/ui/utilities/hotkey/hooks/useSetHotkeyScope';
import { useEffect, useState } from 'react';
import { IconChevronDown, Tag, TagColor } from 'twenty-ui';
import {
MatchedOptions,
MatchedSelectColumn,
MatchedSelectOptionsColumn,
} from '../MatchColumnsStep';
import { IconChevronDown, SelectOption, Tag, TagColor } from 'twenty-ui';
const StyledContainer = styled.div`
align-items: center;
@ -58,11 +57,15 @@ const StyledIconChevronDown = styled(IconChevronDown)`
`;
interface SubMatchingSelectProps<T> {
option: MatchedOptions<T> | Partial<MatchedOptions<T>>;
column: MatchedSelectColumn<T> | MatchedSelectOptionsColumn<T>;
option: SpreadsheetMatchedOptions<T> | Partial<SpreadsheetMatchedOptions<T>>;
column:
| SpreadsheetMatchedSelectColumn<T>
| SpreadsheetMatchedSelectOptionsColumn<T>;
onSubChange: (val: T, index: number, option: string) => void;
placeholder: string;
selectedOption?: MatchedOptions<T> | Partial<MatchedOptions<T>>;
selectedOption?:
| SpreadsheetMatchedOptions<T>
| Partial<SpreadsheetMatchedOptions<T>>;
}
export const SubMatchingSelect = <T extends string>({

View File

@ -3,9 +3,10 @@ import { IconForbid } from 'twenty-ui';
import { MatchColumnSelect } from '@/spreadsheet-import/components/MatchColumnSelect';
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
import { SpreadsheetColumnType } from '@/spreadsheet-import/types/SpreadsheetColumnType';
import { useLingui } from '@lingui/react/macro';
import { FieldMetadataType } from 'twenty-shared/types';
import { Columns, ColumnType } from '../MatchColumnsStep';
const StyledContainer = styled.div`
display: flex;
@ -15,7 +16,7 @@ const StyledContainer = styled.div`
`;
type TemplateColumnProps<T extends string> = {
columns: Columns<string>;
columns: SpreadsheetColumns<string>;
columnIndex: number;
onChange: (val: T, index: number) => void;
};
@ -27,13 +28,13 @@ export const TemplateColumn = <T extends string>({
}: TemplateColumnProps<T>) => {
const { fields } = useSpreadsheetImportInternal<T>();
const column = columns[columnIndex];
const isIgnored = column.type === ColumnType.ignored;
const isIgnored = column.type === SpreadsheetColumnType.ignored;
const { t } = useLingui();
const fieldOptions = fields
.filter((field) => field.fieldMetadataType !== FieldMetadataType.RICH_TEXT)
.map(({ icon, label, key }) => {
.map(({ Icon, label, key }) => {
const isSelected =
columns.findIndex((column) => {
if ('value' in column) {
@ -43,7 +44,7 @@ export const TemplateColumn = <T extends string>({
}) !== -1;
return {
icon: icon,
Icon: Icon,
value: key,
label: label,
disabled: isSelected,
@ -52,7 +53,7 @@ export const TemplateColumn = <T extends string>({
const selectOptions = [
{
icon: IconForbid,
Icon: IconForbid,
value: 'do-not-import',
label: t`Do not import`,
},

View File

@ -1,8 +1,9 @@
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
import { SubMatchingSelect } from '@/spreadsheet-import/steps/components/MatchColumnsStep/components/SubMatchingSelect';
import { UnmatchColumnBanner } from '@/spreadsheet-import/steps/components/MatchColumnsStep/components/UnmatchColumnBanner';
import { Column } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { Fields } from '@/spreadsheet-import/types';
import { SpreadsheetImportFields } from '@/spreadsheet-import/types';
import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn';
import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
import styled from '@emotion/styled';
import { useLingui } from '@lingui/react/macro';
import { useState } from 'react';
@ -10,8 +11,8 @@ import { isDefined } from 'twenty-shared/utils';
import { AnimatedExpandableContainer } from 'twenty-ui';
const getExpandableContainerTitle = <T extends string>(
fields: Fields<T>,
column: Column<T>,
fields: SpreadsheetImportFields<T>,
column: SpreadsheetColumn<T>,
) => {
const fieldLabel = fields.find(
(field) => 'value' in column && field.key === column.value,
@ -24,7 +25,7 @@ const getExpandableContainerTitle = <T extends string>(
};
type UnmatchColumnProps<T extends string> = {
columns: Column<T>[];
columns: SpreadsheetColumns<T>;
columnIndex: number;
onSubChange: (val: T, index: number, option: string) => void;
};

View File

@ -2,7 +2,7 @@ import styled from '@emotion/styled';
import { ImportedRow } from '@/spreadsheet-import/types';
import { Column } from '../MatchColumnsStep';
import { SpreadsheetColumn } from '@/spreadsheet-import/types/SpreadsheetColumn';
import { isDefined } from 'twenty-shared/utils';
const StyledContainer = styled.div`
@ -30,7 +30,7 @@ const StyledExample = styled.span`
`;
type UserTableColumnProps<T extends string> = {
column: Column<T>;
column: SpreadsheetColumn<T>;
importedRow: ImportedRow;
};

View File

@ -1,34 +1,32 @@
import {
Columns,
ColumnType,
} from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { ImportedRow } from '@/spreadsheet-import/types';
import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
import { SpreadsheetColumnType } from '@/spreadsheet-import/types/SpreadsheetColumnType';
import { atom, selectorFamily } from 'recoil';
export const matchColumnsState = atom({
key: 'MatchColumnsState',
default: [] as Columns<string>,
default: [] as SpreadsheetColumns<string>,
});
export const initialComputedColumnsSelector = selectorFamily<
Columns<string>,
SpreadsheetColumns<string>,
ImportedRow
>({
key: 'initialComputedColumnsSelector',
get:
(headerValues: ImportedRow) =>
({ get }) => {
const currentState = get(matchColumnsState) as Columns<string>;
const currentState = get(matchColumnsState) as SpreadsheetColumns<string>;
if (currentState.length === 0) {
// Do not remove spread, it indexes empty array elements, otherwise map() skips over them
const initialState = ([...headerValues] as string[]).map(
(value, index) => ({
type: ColumnType.empty,
type: SpreadsheetColumnType.empty,
index,
header: value ?? '',
}),
);
return initialState as Columns<string>;
return initialState as SpreadsheetColumns<string>;
} else {
return currentState;
}
@ -36,6 +34,6 @@ export const initialComputedColumnsSelector = selectorFamily<
set:
() =>
({ set }, newValue) => {
set(matchColumnsState, newValue as Columns<string>);
set(matchColumnsState, newValue as SpreadsheetColumns<string>);
},
});

View File

@ -1,13 +1,13 @@
import { useMemo } from 'react';
import { SpreadsheetImportTable } from '@/spreadsheet-import/components/SpreadsheetImportTable';
import { Fields } from '@/spreadsheet-import/types';
import { SpreadsheetImportFields } from '@/spreadsheet-import/types';
import { generateExampleRow } from '@/spreadsheet-import/utils/generateExampleRow';
import { generateColumns } from './columns';
interface ExampleTableProps<T extends string> {
fields: Fields<T>;
fields: SpreadsheetImportFields<T>;
}
export const ExampleTable = <T extends string>({

View File

@ -1,10 +1,10 @@
import styled from '@emotion/styled';
// @ts-expect-error // Todo: remove usage of react-data-grid
import { Column } from 'react-data-grid';
import { createPortal } from 'react-dom';
import styled from '@emotion/styled';
import { AppTooltip } from 'twenty-ui';
import { Fields } from '@/spreadsheet-import/types';
import { SpreadsheetImportFields } from '@/spreadsheet-import/types';
const StyledHeaderContainer = styled.div`
align-items: center;
@ -27,7 +27,9 @@ const StyledDefaultContainer = styled.div`
text-overflow: ellipsis;
`;
export const generateColumns = <T extends string>(fields: Fields<T>) =>
export const generateColumns = <T extends string>(
fields: SpreadsheetImportFields<T>,
) =>
fields.map(
(column): Column<any> => ({
key: column.key,

View File

@ -2,16 +2,14 @@ import { Heading } from '@/spreadsheet-import/components/Heading';
import { SpreadsheetImportTable } from '@/spreadsheet-import/components/SpreadsheetImportTable';
import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton';
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
import {
ColumnType,
Columns,
} from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { SpreadsheetImportStep } from '@/spreadsheet-import/steps/types/SpreadsheetImportStep';
import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType';
import {
ImportValidationResult,
ImportedStructuredRow,
SpreadsheetImportImportValidationResult,
} from '@/spreadsheet-import/types';
import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
import { SpreadsheetColumnType } from '@/spreadsheet-import/types/SpreadsheetColumnType';
import { addErrorsAndRunHooks } from '@/spreadsheet-import/utils/dataMutations';
import { useDialogManager } from '@/ui/feedback/dialog-manager/hooks/useDialogManager';
import { Modal } from '@/ui/layout/modal/components/Modal';
@ -74,7 +72,7 @@ const StyledNoRowsContainer = styled.div`
type ValidationStepProps<T extends string> = {
initialData: ImportedStructuredRow<T>[];
importedColumns: Columns<string>;
importedColumns: SpreadsheetColumns<string>;
file: File;
onBack: () => void;
setCurrentStepState: Dispatch<SetStateAction<SpreadsheetImportStep>>;
@ -153,13 +151,14 @@ export const ValidationStep = <T extends string>({
const hasBeenImported =
importedColumns.filter(
(importColumn) =>
(importColumn.type === ColumnType.matched &&
(importColumn.type === SpreadsheetColumnType.matched &&
importColumn.value === column.key) ||
(importColumn.type === ColumnType.matchedSelect &&
(importColumn.type === SpreadsheetColumnType.matchedSelect &&
importColumn.value === column.key) ||
(importColumn.type === ColumnType.matchedSelectOptions &&
(importColumn.type ===
SpreadsheetColumnType.matchedSelectOptions &&
importColumn.value === column.key) ||
(importColumn.type === ColumnType.matchedCheckbox &&
(importColumn.type === SpreadsheetColumnType.matchedCheckbox &&
importColumn.value === column.key) ||
column.key === 'select-row',
).length > 0;
@ -214,7 +213,7 @@ export const ValidationStep = <T extends string>({
validStructuredRows: [] as ImportedStructuredRow<T>[],
invalidStructuredRows: [] as ImportedStructuredRow<T>[],
allStructuredRows: data,
} satisfies ImportValidationResult<T>,
} satisfies SpreadsheetImportImportValidationResult<T>,
);
setCurrentStepState({

View File

@ -5,11 +5,14 @@ import { createPortal } from 'react-dom';
import { AppTooltip, Checkbox, CheckboxVariant, Toggle } from 'twenty-ui';
import { MatchColumnSelect } from '@/spreadsheet-import/components/MatchColumnSelect';
import { Fields, ImportedStructuredRow } from '@/spreadsheet-import/types';
import {
ImportedStructuredRow,
SpreadsheetImportFields,
} from '@/spreadsheet-import/types';
import { TextInput } from '@/ui/input/components/TextInput';
import { ImportedStructuredRowMetadata } from '../types';
import { isDefined } from 'twenty-shared/utils';
import { ImportedStructuredRowMetadata } from '../types';
const StyledHeaderContainer = styled.div`
align-items: center;
@ -60,7 +63,7 @@ const StyledDefaultContainer = styled.div`
const SELECT_COLUMN_KEY = 'select-row';
export const generateColumns = <T extends string>(
fields: Fields<T>,
fields: SpreadsheetImportFields<T>,
): Column<ImportedStructuredRow<T> & ImportedStructuredRowMetadata>[] => [
{
key: SELECT_COLUMN_KEY,
@ -135,7 +138,7 @@ export const generateColumns = <T extends string>(
value={
value
? ({
icon: undefined,
Icon: undefined,
...value,
} as const)
: value

View File

@ -1,8 +1,8 @@
import { Info } from '@/spreadsheet-import/types';
import { SpreadsheetImportInfo } from '@/spreadsheet-import/types';
export type ImportedStructuredRowMetadata = {
__index: string;
__errors?: Error | null;
};
export type Error = { [key: string]: Info };
export type Error = { [key: string]: SpreadsheetImportInfo };
export type Errors = { [id: string]: Error };

View File

@ -1,6 +1,6 @@
import { Columns } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { SpreadsheetImportStepType } from '@/spreadsheet-import/steps/types/SpreadsheetImportStepType';
import { ImportedRow } from '@/spreadsheet-import/types';
import { SpreadsheetColumns } from '@/spreadsheet-import/types/SpreadsheetColumns';
import { WorkBook } from 'xlsx-ugnis';
export type SpreadsheetImportStep =
@ -23,7 +23,7 @@ export type SpreadsheetImportStep =
| {
type: SpreadsheetImportStepType.validateData;
data: any[];
importedColumns: Columns<string>;
importedColumns: SpreadsheetColumns<string>;
}
| {
type: SpreadsheetImportStepType.loading;