CSV importing and exporting fixes (#8824)

Fixes issue https://github.com/twentyhq/twenty/issues/5793 (and duplicate https://github.com/twentyhq/twenty/issues/8822) - Fix importing multi-select and array fields. - Fix exporting and importing RAW_JSON fields. --------- Co-authored-by: ad-elias <elias@autodiligence.com>
2024-12-05 18:44:53 +01:00
parent 815e5dfa16
commit f60ce384c6
10 changed files with 181 additions and 28 deletions
--- a/packages/twenty-front/src/modules/object-record/object-options-dropdown/hooks/useExportProcessRecordsForCSV.ts
+++ b/packages/twenty-front/src/modules/object-record/object-options-dropdown/hooks/useExportProcessRecordsForCSV.ts
@ -11,28 +11,36 @@ export const useExportProcessRecordsForCSV = (objectNameSingular: string) => {
  });

  const processRecordsForCSVExport = (records: ObjectRecord[]) => {
-    return records.map((record) => {
-      const currencyFields = objectMetadataItem.fields.filter(
-        (field) => field.type === FieldMetadataType.Currency,
-      );
+    return records.map((record) =>
+      objectMetadataItem.fields.reduce(
+        (processedRecord, field) => {
+          if (!isDefined(record[field.name])) {
+            return processedRecord;
+          }

-      const processedRecord = {
-        ...record,
-      };
-
-      for (const currencyField of currencyFields) {
-        if (isDefined(record[currencyField.name])) {
-          processedRecord[currencyField.name] = {
-            amountMicros: convertCurrencyMicrosToCurrencyAmount(
-              record[currencyField.name].amountMicros,
-            ),
-            currencyCode: record[currencyField.name].currencyCode,
-          } satisfies FieldCurrencyValue;
-        }
-      }
-
-      return processedRecord;
-    });
+          switch (field.type) {
+            case FieldMetadataType.Currency:
+              return {
+                ...processedRecord,
+                [field.name]: {
+                  amountMicros: convertCurrencyMicrosToCurrencyAmount(
+                    record[field.name].amountMicros,
+                  ),
+                  currencyCode: record[field.name].currencyCode,
+                } satisfies FieldCurrencyValue,
+              };
+            case FieldMetadataType.RawJson:
+              return {
+                ...processedRecord,
+                [field.name]: JSON.stringify(record[field.name]),
+              };
+            default:
+              return processedRecord;
+          }
+        },
+        { ...record },
+      ),
+    );
  };

  return { processRecordsForCSVExport };
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/tests/useOpenObjectRecordsSpreadsheetImportDialog.test.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/tests/useOpenObjectRecordsSpreadsheetImportDialog.test.ts
@ -290,6 +290,8 @@ const companyMocks = [
            name: 'Example Company',
            id: companyId,
            visaSponsorship: false,
+            deletedAt: undefined,
+            workPolicy: [],
          },
        ],
        upsert: true,
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport.ts
@ -143,6 +143,25 @@ export const useBuildAvailableFieldsForImport = () => {
            fieldMetadataItem.label + ' (ID)',
          ),
        });
+      } else if (fieldMetadataItem.type === FieldMetadataType.MultiSelect) {
+        availableFieldsForImport.push({
+          icon: getIcon(fieldMetadataItem.icon),
+          label: fieldMetadataItem.label,
+          key: fieldMetadataItem.name,
+          fieldType: {
+            type: 'multiSelect',
+            options:
+              fieldMetadataItem.options?.map((option) => ({
+                label: option.label,
+                value: option.value,
+                color: option.color,
+              })) || [],
+          },
+          fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
+            fieldMetadataItem.type,
+            fieldMetadataItem.label + ' (ID)',
+          ),
+        });
      } else if (fieldMetadataItem.type === FieldMetadataType.Boolean) {
        availableFieldsForImport.push({
          icon: getIcon(fieldMetadataItem.icon),
--- a/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts
+++ b/packages/twenty-front/src/modules/object-record/spreadsheet-import/utils/buildRecordFromImportedStructuredRow.ts
@ -9,6 +9,7 @@ import { COMPOSITE_FIELD_IMPORT_LABELS } from '@/object-record/spreadsheet-impor
 import { ImportedStructuredRow } from '@/spreadsheet-import/types';
 import { isNonEmptyString } from '@sniptt/guards';
 import { isDefined } from 'twenty-ui';
+import { z } from 'zod';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
 import { castToString } from '~/utils/castToString';
 import { convertCurrencyAmountToCurrencyMicros } from '~/utils/convertCurrencyToCurrencyMicros';
@ -203,6 +204,35 @@ export const buildRecordFromImportedStructuredRow = (
          source: 'IMPORT',
        };
        break;
+      case FieldMetadataType.Array:
+      case FieldMetadataType.MultiSelect: {
+        const stringArrayJSONSchema = z
+          .preprocess((value) => {
+            try {
+              if (typeof value !== 'string') {
+                return [];
+              }
+              return JSON.parse(value);
+            } catch {
+              return [];
+            }
+          }, z.array(z.string()))
+          .catch([]);
+
+        recordToBuild[field.name] =
+          stringArrayJSONSchema.parse(importedFieldValue);
+        break;
+      }
+      case FieldMetadataType.RawJson: {
+        if (typeof importedFieldValue === 'string') {
+          try {
+            recordToBuild[field.name] = JSON.parse(importedFieldValue);
+          } catch {
+            break;
+          }
+        }
+        break;
+      }
      default:
        recordToBuild[field.name] = importedFieldValue;
        break;
--- a/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx
+++ b/packages/twenty-front/src/modules/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep.tsx
@ -77,7 +77,7 @@ export enum ColumnType {

 export type MatchedOptions<T> = {
  entry: string;
-  value: T;
+  value?: T;
 };

 type EmptyColumn = { type: ColumnType.empty; index: number; header: string };
--- a/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts
+++ b/packages/twenty-front/src/modules/spreadsheet-import/types/index.ts
@ -79,6 +79,11 @@ export type Select = {
  options: SelectOption[];
 };

+export type MultiSelect = {
+  type: 'multiSelect';
+  options: SelectOption[];
+};
+
 export type SelectOption = {
  // Icon
  icon?: IconComponent | null;
@ -96,7 +101,11 @@ export type Input = {
  type: 'input';
 };

-export type SpreadsheetImportFieldType = Checkbox | Select | Input;
+export type SpreadsheetImportFieldType =
+  | Checkbox
+  | Select
+  | MultiSelect
+  | Input;

 export type Field<T extends string> = {
  // Icon
--- a/packages/twenty-front/src/modules/spreadsheet-import/utils/generateExampleRow.ts
+++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/generateExampleRow.ts
@ -3,6 +3,7 @@ import { Field, Fields } from '@/spreadsheet-import/types';
 const titleMap: Record<Field<string>['fieldType']['type'], string> = {
  checkbox: 'Boolean',
  select: 'Options',
+  multiSelect: 'Options',
  input: 'Text',
 };

--- a/packages/twenty-front/src/modules/spreadsheet-import/utils/getFieldOptions.ts
+++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/getFieldOptions.ts
@ -8,5 +8,8 @@ export const getFieldOptions = <T extends string>(
  if (!field) {
    return [];
  }
-  return field.fieldType.type === 'select' ? field.fieldType.options : [];
+  return field.fieldType.type === 'select' ||
+    field.fieldType.type === 'multiSelect'
+    ? field.fieldType.options
+    : [];
 };
--- a/packages/twenty-front/src/modules/spreadsheet-import/utils/normalizeTableData.ts
+++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/normalizeTableData.ts
@ -8,6 +8,8 @@ import {
  ImportedStructuredRow,
 } from '@/spreadsheet-import/types';

+import { isDefined } from '@ui/utilities/isDefined';
+import { z } from 'zod';
 import { normalizeCheckboxValue } from './normalizeCheckboxValue';

 export const normalizeTableData = <T extends string>(
@ -54,10 +56,45 @@ export const normalizeTableData = <T extends string>(
        }
        case ColumnType.matchedSelect:
        case ColumnType.matchedSelectOptions: {
-          const matchedOption = column.matchedOptions.find(
-            ({ entry }) => entry === curr,
-          );
-          acc[column.value] = matchedOption?.value || undefined;
+          const field = fields.find((field) => field.key === column.value);
+
+          if (!field) {
+            return acc;
+          }
+
+          if (field.fieldType.type === 'multiSelect' && isDefined(curr)) {
+            const currentOptionsSchema = z.preprocess(
+              (value) => JSON.parse(z.string().parse(value)),
+              z.array(z.unknown()),
+            );
+
+            const rawCurrentOptions = currentOptionsSchema.safeParse(curr).data;
+
+            const matchedOptionValues = [
+              ...new Set(
+                rawCurrentOptions
+                  ?.map(
+                    (option) =>
+                      column.matchedOptions.find(
+                        (matchedOption) => matchedOption.entry === option,
+                      )?.value,
+                  )
+                  .filter(isDefined),
+              ),
+            ];
+
+            const fieldValue =
+              matchedOptionValues && matchedOptionValues.length > 0
+                ? JSON.stringify(matchedOptionValues)
+                : undefined;
+
+            acc[column.value] = fieldValue;
+          } else {
+            const matchedOption = column.matchedOptions.find(
+              ({ entry }) => entry === curr,
+            );
+            acc[column.value] = matchedOption?.value || undefined;
+          }
          return acc;
        }
        case ColumnType.empty:
--- a/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts
+++ b/packages/twenty-front/src/modules/spreadsheet-import/utils/setColumn.ts
@ -6,6 +6,7 @@ import {
 } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
 import { Field } from '@/spreadsheet-import/types';

+import { z } from 'zod';
 import { uniqueEntries } from './uniqueEntries';

 export const setColumn = <T extends string>(
@ -19,6 +20,7 @@ export const setColumn = <T extends string>(
      data || [],
      oldColumn.index,
    ) as MatchedOptions<T>[];
+
    const matchedOptions = uniqueData.map((record) => {
      const value = fieldOptions.find(
        (fieldOption) =>
@ -42,6 +44,48 @@ export const setColumn = <T extends string>(
    };
  }

+  if (field?.fieldType.type === 'multiSelect') {
+    const fieldOptions = field.fieldType.options;
+
+    const entries = [
+      ...new Set(
+        data
+          ?.flatMap((row) => {
+            try {
+              const value = row[oldColumn.index];
+              const options = JSON.parse(z.string().parse(value));
+              return z.array(z.string()).parse(options);
+            } catch {
+              return [];
+            }
+          })
+          .filter((entry) => typeof entry === 'string'),
+      ),
+    ];
+
+    const matchedOptions = entries.map((entry) => {
+      const value = fieldOptions.find(
+        (fieldOption) =>
+          fieldOption.value === entry || fieldOption.label === entry,
+      )?.value;
+      return value
+        ? ({ entry, value } as MatchedOptions<T>)
+        : ({ entry } as MatchedOptions<T>);
+    });
+    const areAllMatched =
+      matchedOptions.filter((option) => option.value).length ===
+      entries?.length;
+
+    return {
+      ...oldColumn,
+      type: areAllMatched
+        ? ColumnType.matchedSelectOptions
+        : ColumnType.matchedSelect,
+      value: field.key,
+      matchedOptions,
+    };
+  }
+
  if (field?.fieldType.type === 'checkbox') {
    return {
      index: oldColumn.index,