Fix record board export not taking filters into account (#8505)

Fix Export CSV action not taking into account the filters applied on the
Kanban index view

packages/twenty-front/src/modules/object-record/record-index/components/RecordIndexContainer.tsx

@@ -26,6 +26,7 @@ import { RecordIndexActionMenu } from '@/action-menu/components/RecordIndexActio
 import { contextStoreTargetedRecordsRuleComponentState } from '@/context-store/states/contextStoreTargetedRecordsRuleComponentState';
 import { useRecordBoard } from '@/object-record/record-board/hooks/useRecordBoard';
 import { recordGroupDefinitionsComponentState } from '@/object-record/record-group/states/recordGroupDefinitionsComponentState';
+import { RecordIndexFiltersToContextStoreEffect } from '@/object-record/record-index/components/RecordIndexFiltersToContextStoreEffect';
 import { recordIndexViewFilterGroupsState } from '@/object-record/record-index/states/recordIndexViewFilterGroupsState';
 import { useRecoilComponentCallbackStateV2 } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentCallbackStateV2';
 import { useSetRecoilComponentStateV2 } from '@/ui/utilities/state/component-state/hooks/useSetRecoilComponentStateV2';
@@ -207,7 +208,7 @@ export const RecordIndexContainer = () => {
             viewBarId={recordIndexId}
           />
         </SpreadsheetImportProvider>
-
+        <RecordIndexFiltersToContextStoreEffect />
         {recordIndexViewType === ViewType.Table && (
           <>
             <RecordIndexTableContainer

packages/twenty-front/src/modules/object-record/record-index/components/RecordIndexFiltersToContextStoreEffect.tsx

@@ -0,0 +1,76 @@
+import { useContext, useEffect } from 'react';
+
+import { contextStoreFiltersComponentState } from '@/context-store/states/contextStoreFiltersComponentState';
+import { contextStoreTargetedRecordsRuleComponentState } from '@/context-store/states/contextStoreTargetedRecordsRuleComponentState';
+import { RecordIndexRootPropsContext } from '@/object-record/record-index/contexts/RecordIndexRootPropsContext';
+import { recordIndexFiltersState } from '@/object-record/record-index/states/recordIndexFiltersState';
+import { hasUserSelectedAllRowsComponentState } from '@/object-record/record-table/record-table-row/states/hasUserSelectedAllRowsFamilyState';
+import { selectedRowIdsComponentSelector } from '@/object-record/record-table/states/selectors/selectedRowIdsComponentSelector';
+import { unselectedRowIdsComponentSelector } from '@/object-record/record-table/states/selectors/unselectedRowIdsComponentSelector';
+import { useRecoilComponentValueV2 } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentValueV2';
+import { useSetRecoilComponentStateV2 } from '@/ui/utilities/state/component-state/hooks/useSetRecoilComponentStateV2';
+import { useRecoilValue } from 'recoil';
+
+export const RecordIndexFiltersToContextStoreEffect = () => {
+  const { recordIndexId } = useContext(RecordIndexRootPropsContext);
+
+  const recordIndexFilters = useRecoilValue(recordIndexFiltersState);
+
+  const setContextStoreTargetedRecords = useSetRecoilComponentStateV2(
+    contextStoreTargetedRecordsRuleComponentState,
+  );
+
+  const hasUserSelectedAllRows = useRecoilComponentValueV2(
+    hasUserSelectedAllRowsComponentState,
+    recordIndexId,
+  );
+
+  const selectedRowIds = useRecoilComponentValueV2(
+    selectedRowIdsComponentSelector,
+    recordIndexId,
+  );
+  const unselectedRowIds = useRecoilComponentValueV2(
+    unselectedRowIdsComponentSelector,
+    recordIndexId,
+  );
+
+  useEffect(() => {
+    if (hasUserSelectedAllRows) {
+      setContextStoreTargetedRecords({
+        mode: 'exclusion',
+        excludedRecordIds: unselectedRowIds,
+      });
+    } else {
+      setContextStoreTargetedRecords({
+        mode: 'selection',
+        selectedRecordIds: selectedRowIds,
+      });
+    }
+
+    return () => {
+      setContextStoreTargetedRecords({
+        mode: 'selection',
+        selectedRecordIds: [],
+      });
+    };
+  }, [
+    hasUserSelectedAllRows,
+    selectedRowIds,
+    setContextStoreTargetedRecords,
+    unselectedRowIds,
+  ]);
+
+  const setContextStoreFilters = useSetRecoilComponentStateV2(
+    contextStoreFiltersComponentState,
+  );
+
+  useEffect(() => {
+    setContextStoreFilters(recordIndexFilters);
+
+    return () => {
+      setContextStoreFilters([]);
+    };
+  }, [recordIndexFilters, setContextStoreFilters]);
+
+  return <></>;
+};

packages/twenty-front/src/modules/object-record/record-index/components/RecordIndexTableContainerEffect.tsx

@@ -1,21 +1,12 @@
 import { useContext, useEffect } from 'react';
 
-import { contextStoreFiltersComponentState } from '@/context-store/states/contextStoreFiltersComponentState';
-import { contextStoreTargetedRecordsRuleComponentState } from '@/context-store/states/contextStoreTargetedRecordsRuleComponentState';
 import { useColumnDefinitionsFromFieldMetadata } from '@/object-metadata/hooks/useColumnDefinitionsFromFieldMetadata';
 import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
 import { RecordIndexRootPropsContext } from '@/object-record/record-index/contexts/RecordIndexRootPropsContext';
 import { useHandleToggleColumnFilter } from '@/object-record/record-index/hooks/useHandleToggleColumnFilter';
 import { useHandleToggleColumnSort } from '@/object-record/record-index/hooks/useHandleToggleColumnSort';
-import { recordIndexFiltersState } from '@/object-record/record-index/states/recordIndexFiltersState';
 import { useRecordTable } from '@/object-record/record-table/hooks/useRecordTable';
-import { hasUserSelectedAllRowsComponentState } from '@/object-record/record-table/record-table-row/states/hasUserSelectedAllRowsFamilyState';
-import { selectedRowIdsComponentSelector } from '@/object-record/record-table/states/selectors/selectedRowIdsComponentSelector';
-import { unselectedRowIdsComponentSelector } from '@/object-record/record-table/states/selectors/unselectedRowIdsComponentSelector';
-import { useRecoilComponentValueV2 } from '@/ui/utilities/state/component-state/hooks/useRecoilComponentValueV2';
-import { useSetRecoilComponentStateV2 } from '@/ui/utilities/state/component-state/hooks/useSetRecoilComponentStateV2';
 import { useSetRecordCountInCurrentView } from '@/views/hooks/useSetRecordCountInCurrentView';
-import { useRecoilValue } from 'recoil';
 
 export const RecordIndexTableContainerEffect = () => {
   const { recordIndexId, objectNameSingular } = useContext(
@@ -77,61 +68,5 @@ export const RecordIndexTableContainerEffect = () => {
     );
   }, [setRecordCountInCurrentView, setOnEntityCountChange]);
 
-  const setContextStoreTargetedRecords = useSetRecoilComponentStateV2(
-    contextStoreTargetedRecordsRuleComponentState,
-  );
-  const hasUserSelectedAllRows = useRecoilComponentValueV2(
-    hasUserSelectedAllRowsComponentState,
-    recordIndexId,
-  );
-  const selectedRowIds = useRecoilComponentValueV2(
-    selectedRowIdsComponentSelector,
-    recordIndexId,
-  );
-  const unselectedRowIds = useRecoilComponentValueV2(
-    unselectedRowIdsComponentSelector,
-    recordIndexId,
-  );
-
-  const recordIndexFilters = useRecoilValue(recordIndexFiltersState);
-
-  useEffect(() => {
-    if (hasUserSelectedAllRows) {
-      setContextStoreTargetedRecords({
-        mode: 'exclusion',
-        excludedRecordIds: unselectedRowIds,
-      });
-    } else {
-      setContextStoreTargetedRecords({
-        mode: 'selection',
-        selectedRecordIds: selectedRowIds,
-      });
-    }
-
-    return () => {
-      setContextStoreTargetedRecords({
-        mode: 'selection',
-        selectedRecordIds: [],
-      });
-    };
-  }, [
-    hasUserSelectedAllRows,
-    selectedRowIds,
-    setContextStoreTargetedRecords,
-    unselectedRowIds,
-  ]);
-
-  const setContextStoreFilters = useSetRecoilComponentStateV2(
-    contextStoreFiltersComponentState,
-  );
-
-  useEffect(() => {
-    setContextStoreFilters(recordIndexFilters);
-
-    return () => {
-      setContextStoreFilters([]);
-    };
-  }, [recordIndexFilters, setContextStoreFilters]);
-
   return <></>;
 };

packages/twenty-front/src/modules/object-record/record-index/export/hooks/__tests__/useExportFetchRecords.test.ts

@@ -1,6 +1,10 @@
 import { renderHook, waitFor } from '@testing-library/react';
 import { act } from 'react';
-import { percentage, sleep, useRecordData } from '../useRecordData';
+import {
+  percentage,
+  sleep,
+  useExportFetchRecords,
+} from '../useExportFetchRecords';
 
 import { PERSON_FRAGMENT_WITH_DEPTH_ZERO_RELATIONS } from '@/object-record/hooks/__mocks__/personFragments';
 import { useRecordBoard } from '@/object-record/record-board/hooks/useRecordBoard';
@@ -9,6 +13,7 @@ import { useRecordIndexOptionsForBoard } from '@/object-record/record-index/opti
 import { extractComponentState } from '@/ui/utilities/state/component-state/utils/extractComponentState';
 import { ViewType } from '@/views/types/ViewType';
 import { MockedResponse } from '@apollo/client/testing';
+import { expect } from '@storybook/test';
 import gql from 'graphql-tag';
 import { useRecoilValue } from 'recoil';
 import { getJestMetadataAndApolloMocksAndContextStoreWrapper } from '~/testing/jest/getJestMetadataAndApolloMocksAndContextStoreWrapper';
@@ -178,7 +183,7 @@ describe('useRecordData', () => {
 
       const { result } = renderHook(
         () =>
-          useRecordData({
+          useExportFetchRecords({
             recordIndexId,
             objectMetadataItem,
             pageSize: 30,
@@ -204,7 +209,7 @@ describe('useRecordData', () => {
       const callback = jest.fn();
       const { result } = renderHook(
         () =>
-          useRecordData({
+          useExportFetchRecords({
             recordIndexId,
             objectMetadataItem,
             callback,
@@ -232,7 +237,7 @@ describe('useRecordData', () => {
             recordIndexId,
           );
           return {
-            tableData: useRecordData({
+            tableData: useExportFetchRecords({
               recordIndexId,
               objectMetadataItem,
               callback,
@@ -322,7 +327,7 @@ describe('useRecordData', () => {
             recordIndexId,
           );
           return {
-            tableData: useRecordData({
+            tableData: useExportFetchRecords({
               recordIndexId,
               objectMetadataItem,
               callback,

packages/twenty-front/src/modules/object-record/record-index/export/hooks/__tests__/useExportRecords.test.ts

@@ -0,0 +1,107 @@
+import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata';
+import { ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition';
+
+import { RelationDefinitionType } from '~/generated-metadata/graphql';
+import {
+  csvDownloader,
+  displayedExportProgress,
+  download,
+  generateCsv,
+} from '../useExportRecords';
+
+jest.useFakeTimers();
+
+describe('download', () => {
+  it('creates a download link and clicks it', () => {
+    const link = document.createElement('a');
+    document.createElement = jest.fn().mockReturnValue(link);
+    const appendChild = jest.spyOn(document.body, 'appendChild');
+    const click = jest.spyOn(link, 'click');
+
+    URL.createObjectURL = jest.fn().mockReturnValue('fake-url');
+    download(new Blob(['test'], { type: 'text/plain' }), 'test.txt');
+
+    expect(appendChild).toHaveBeenCalledWith(link);
+    expect(link.href).toEqual('http://localhost/fake-url');
+    expect(link.getAttribute('download')).toEqual('test.txt');
+    expect(click).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('generateCsv', () => {
+  it('generates a csv with formatted headers', async () => {
+    const columns = [
+      { label: 'Foo', metadata: { fieldName: 'foo' } },
+      { label: 'Empty', metadata: { fieldName: 'empty' } },
+      { label: 'Nested', metadata: { fieldName: 'nested' } },
+      {
+        label: 'Relation',
+        metadata: {
+          fieldName: 'relation',
+          relationType: RelationDefinitionType.ManyToOne,
+        },
+      },
+    ] as ColumnDefinition<FieldMetadata>[];
+    const rows = [
+      {
+        id: '1',
+        bar: 'another field',
+        empty: null,
+        foo: 'some field',
+        nested: { __typename: 'type', foo: 'foo', nested: 'nested' },
+        relation: 'a relation',
+      },
+    ];
+    const csv = generateCsv({ columns, rows });
+    expect(csv).toEqual(`Id,Foo,Empty,Nested Foo,Nested Nested,Relation
+1,some field,,foo,nested,a relation`);
+  });
+});
+
+describe('csvDownloader', () => {
+  it('downloads a csv', () => {
+    const filename = 'test.csv';
+    const data = {
+      rows: [
+        { id: 1, name: 'John' },
+        { id: 2, name: 'Alice' },
+      ],
+      columns: [],
+      objectNameSingular: '',
+    };
+
+    const link = document.createElement('a');
+    document.createElement = jest.fn().mockReturnValue(link);
+    const createObjectURL = jest.spyOn(URL, 'createObjectURL');
+
+    csvDownloader(filename, data);
+
+    expect(link.getAttribute('download')).toEqual('test.csv');
+    expect(createObjectURL).toHaveBeenCalledWith(expect.any(Blob));
+    expect(createObjectURL).toHaveBeenCalledWith(
+      expect.objectContaining({ type: 'text/csv' }),
+    );
+  });
+});
+
+describe('displayedExportProgress', () => {
+  it.each([
+    [undefined, undefined, 'percentage', 'Export View as CSV'],
+    [20, 50, 'percentage', 'Export (40%)'],
+    [0, 100, 'number', 'Export (0)'],
+    [10, 10, 'percentage', 'Export (100%)'],
+    [10, 10, 'number', 'Export (10)'],
+    [7, 9, 'percentage', 'Export (78%)'],
+  ])(
+    'displays the export progress',
+    (exportedRecordCount, totalRecordCount, displayType, expected) => {
+      expect(
+        displayedExportProgress('all', {
+          exportedRecordCount,
+          totalRecordCount,
+          displayType: displayType as 'percentage' | 'number',
+        }),
+      ).toEqual(expected);
+    },
+  );
+});

packages/twenty-front/src/modules/object-record/record-index/export/hooks/useExportFetchRecords.ts

@@ -45,7 +45,7 @@ type ExportProgress = {
   displayType: 'percentage' | 'number';
 };
 
-export const useRecordData = ({
+export const useExportFetchRecords = ({
   objectMetadataItem,
   delayMs,
   maximumRequests = 100,

packages/twenty-front/src/modules/object-record/record-index/export/hooks/useExportProcessRecordsForCSV.ts

@@ -5,7 +5,7 @@ import { isDefined } from 'twenty-ui';
 import { FieldMetadataType } from '~/generated-metadata/graphql';
 import { convertCurrencyMicrosToCurrencyAmount } from '~/utils/convertCurrencyToCurrencyMicros';
 
-export const useProcessRecordsForCSVExport = (objectNameSingular: string) => {
+export const useExportProcessRecordsForCSV = (objectNameSingular: string) => {
   const { objectMetadataItem } = useObjectMetadataItem({
     objectNameSingular,
   });

packages/twenty-front/src/modules/object-record/record-index/export/hooks/useExportRecords.ts

@@ -0,0 +1,178 @@
+import { json2csv } from 'json-2-csv';
+import { useMemo } from 'react';
+
+import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata';
+import {
+  UseRecordDataOptions,
+  useExportFetchRecords,
+} from '@/object-record/record-index/export/hooks/useExportFetchRecords';
+import { useExportProcessRecordsForCSV } from '@/object-record/record-index/export/hooks/useExportProcessRecordsForCSV';
+import { EXPORT_TABLE_DATA_DEFAULT_PAGE_SIZE } from '@/object-record/record-index/options/constants/ExportTableDataDefaultPageSize';
+import { ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition';
+import { ObjectRecord } from '@/object-record/types/ObjectRecord';
+import { RelationDefinitionType } from '~/generated-metadata/graphql';
+import { FieldMetadataType } from '~/generated/graphql';
+import { isDefined } from '~/utils/isDefined';
+import { isUndefinedOrNull } from '~/utils/isUndefinedOrNull';
+
+export const download = (blob: Blob, filename: string) => {
+  const url = URL.createObjectURL(blob);
+  const link = document.createElement('a');
+  link.href = url;
+  link.setAttribute('download', filename);
+  document.body.appendChild(link);
+  link.click();
+  link.parentNode?.removeChild(link);
+};
+
+type GenerateExportOptions = {
+  columns: ColumnDefinition<FieldMetadata>[];
+  rows: object[];
+};
+
+type GenerateExport = (data: GenerateExportOptions) => string;
+
+type ExportProgress = {
+  exportedRecordCount?: number;
+  totalRecordCount?: number;
+  displayType: 'percentage' | 'number';
+};
+
+export const generateCsv: GenerateExport = ({
+  columns,
+  rows,
+}: GenerateExportOptions): string => {
+  const columnsToExport = columns.filter(
+    (col) =>
+      !('relationType' in col.metadata && col.metadata.relationType) ||
+      col.metadata.relationType === RelationDefinitionType.ManyToOne,
+  );
+
+  const objectIdColumn: ColumnDefinition<FieldMetadata> = {
+    fieldMetadataId: '',
+    type: FieldMetadataType.Uuid,
+    iconName: '',
+    label: `Id`,
+    metadata: {
+      fieldName: 'id',
+    },
+    position: 0,
+    size: 0,
+  };
+
+  const columnsToExportWithIdColumn = [objectIdColumn, ...columnsToExport];
+
+  const keys = columnsToExportWithIdColumn.flatMap((col) => {
+    const column = {
+      field: `${col.metadata.fieldName}${col.type === 'RELATION' ? 'Id' : ''}`,
+      title: [col.label, col.type === 'RELATION' ? 'Id' : null]
+        .filter(isDefined)
+        .join(' '),
+    };
+
+    const fieldsWithSubFields = rows.find((row) => {
+      const fieldValue = (row as any)[column.field];
+
+      const hasSubFields =
+        fieldValue &&
+        typeof fieldValue === 'object' &&
+        !Array.isArray(fieldValue);
+
+      return hasSubFields;
+    });
+
+    if (isDefined(fieldsWithSubFields)) {
+      const nestedFieldsWithoutTypename = Object.keys(
+        (fieldsWithSubFields as any)[column.field],
+      )
+        .filter((key) => key !== '__typename')
+        .map((key) => ({
+          field: `${column.field}.${key}`,
+          title: `${column.title} ${key[0].toUpperCase() + key.slice(1)}`,
+        }));
+
+      return nestedFieldsWithoutTypename;
+    }
+
+    return [column];
+  });
+
+  return json2csv(rows, {
+    keys,
+    emptyFieldValue: '',
+  });
+};
+
+const percentage = (part: number, whole: number): number => {
+  return Math.round((part / whole) * 100);
+};
+
+export const displayedExportProgress = (
+  mode: 'all' | 'selection' = 'all',
+  progress?: ExportProgress,
+): string => {
+  if (isUndefinedOrNull(progress?.exportedRecordCount)) {
+    return mode === 'all' ? 'Export View as CSV' : 'Export Selection as CSV';
+  }
+
+  if (
+    progress.displayType === 'percentage' &&
+    isDefined(progress?.totalRecordCount)
+  ) {
+    return `Export (${percentage(
+      progress.exportedRecordCount,
+      progress.totalRecordCount,
+    )}%)`;
+  }
+
+  return `Export (${progress.exportedRecordCount})`;
+};
+
+const downloader = (mimeType: string, generator: GenerateExport) => {
+  return (filename: string, data: GenerateExportOptions) => {
+    const blob = new Blob([generator(data)], { type: mimeType });
+    download(blob, filename);
+  };
+};
+
+export const csvDownloader = downloader('text/csv', generateCsv);
+
+type UseExportTableDataOptions = Omit<UseRecordDataOptions, 'callback'> & {
+  filename: string;
+};
+
+export const useExportRecords = ({
+  delayMs,
+  filename,
+  maximumRequests = 100,
+  objectMetadataItem,
+  pageSize = EXPORT_TABLE_DATA_DEFAULT_PAGE_SIZE,
+  recordIndexId,
+  viewType,
+}: UseExportTableDataOptions) => {
+  const { processRecordsForCSVExport } = useExportProcessRecordsForCSV(
+    objectMetadataItem.nameSingular,
+  );
+
+  const downloadCsv = useMemo(
+    () =>
+      (records: ObjectRecord[], columns: ColumnDefinition<FieldMetadata>[]) => {
+        const recordsProcessedForExport = processRecordsForCSVExport(records);
+
+        csvDownloader(filename, { rows: recordsProcessedForExport, columns });
+      },
+    [filename, processRecordsForCSVExport],
+  );
+
+  const { getTableData: download, progress } = useExportFetchRecords({
+    delayMs,
+    maximumRequests,
+    objectMetadataItem,
+    pageSize,
+    recordIndexId,
+    callback: downloadCsv,
+    viewType,
+  });
+
+  return { progress, download };
+};

packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx

@@ -20,15 +20,15 @@ import { useObjectNamePluralFromSingular } from '@/object-metadata/hooks/useObje
 import { useHandleToggleTrashColumnFilter } from '@/object-record/record-index/hooks/useHandleToggleTrashColumnFilter';
 import { RECORD_INDEX_OPTIONS_DROPDOWN_ID } from '@/object-record/record-index/options/constants/RecordIndexOptionsDropdownId';
 
-import {
-  displayedExportProgress,
-  useExportRecordData,
-} from '@/action-menu/hooks/useExportRecordData';
 import { contextStoreNumberOfSelectedRecordsComponentState } from '@/context-store/states/contextStoreNumberOfSelectedRecordsComponentState';
 import { ObjectMetadataItem } from '@/object-metadata/types/ObjectMetadataItem';
 import { useRecordGroupReorder } from '@/object-record/record-group/hooks/useRecordGroupReorder';
-import { useRecordGroups } from '@/object-record/record-group/hooks/useRecordGroups';
 import { useRecordGroupVisibility } from '@/object-record/record-group/hooks/useRecordGroupVisibility';
+import { useRecordGroups } from '@/object-record/record-group/hooks/useRecordGroups';
+import {
+  displayedExportProgress,
+  useExportRecords,
+} from '@/object-record/record-index/export/hooks/useExportRecords';
 import { useRecordIndexOptionsForBoard } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForBoard';
 import { useRecordIndexOptionsForTable } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForTable';
 import { TableOptionsHotkeyScope } from '@/object-record/record-table/types/TableOptionsHotkeyScope';
@@ -167,7 +167,7 @@ export const RecordIndexOptionsDropdownContent = ({
   const { openObjectRecordsSpreasheetImportDialog } =
     useOpenObjectRecordsSpreasheetImportDialog(objectMetadataItem.nameSingular);
 
-  const { progress, download } = useExportRecordData({
+  const { progress, download } = useExportRecords({
     delayMs: 100,
     filename: `${objectMetadataItem.nameSingular}.csv`,
     objectMetadataItem,