From fede721ba852d87cdb2a4b6f57704ea33e1b3e1c Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Thu, 23 May 2024 22:36:50 +0200 Subject: [PATCH] Add sorter for distant tables (#5546) As title --- .../distant-table/distant-table.service.ts | 2 +- .../__tests__/sort-distant-tables.spec.ts | 53 ++++++++++++ .../utils/sort-distant-tables.util.ts | 28 +++++++ .../stripe-distant-tables.util.ts | 0 .../remote-table-schema-update.service.ts | 54 +------------ .../remote-table/remote-table.resolver.ts | 2 +- .../remote-table/remote-table.service.ts | 81 ++++++++++++++----- 7 files changed, 147 insertions(+), 73 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/__tests__/sort-distant-tables.spec.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util.ts rename packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/{util => utils}/stripe-distant-tables.util.ts (100%) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts index 7aefdbf25..bd1762ca2 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts @@ -10,7 +10,7 @@ import { } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; -import { STRIPE_DISTANT_TABLES } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/util/stripe-distant-tables.util'; +import { STRIPE_DISTANT_TABLES } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/stripe-distant-tables.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; @Injectable() diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/__tests__/sort-distant-tables.spec.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/__tests__/sort-distant-tables.spec.ts new file mode 100644 index 000000000..a318352e3 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/__tests__/sort-distant-tables.spec.ts @@ -0,0 +1,53 @@ +import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; +import { sortDistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util'; + +const table1 = { + status: RemoteTableStatus.SYNCED, + name: 'table1', +}; + +const table2 = { + status: RemoteTableStatus.NOT_SYNCED, + name: 'table2', +}; + +describe('sortDistantTables', () => { + it('should return -1 when first param status is SYNCED and second param status is NOT_SYNCED', () => { + const result = sortDistantTables(table1, table2); + + expect(result).toBe(-1); + }); + + it('should return 1 when first param status is NOT_SYNCED and second param status is SYNCED', () => { + const result = sortDistantTables(table2, table1); + + expect(result).toBe(1); + }); + + it('should return -1 when same status and first param name is smaller than second param name', () => { + const result = sortDistantTables( + { ...table1, status: RemoteTableStatus.NOT_SYNCED }, + table2, + ); + + expect(result).toBe(-1); + }); + + it('should return 1 when same status and second param name is smaller than first param name', () => { + const result = sortDistantTables(table2, { + ...table1, + status: RemoteTableStatus.NOT_SYNCED, + }); + + expect(result).toBe(1); + }); + + it('should be case insensitive', () => { + const result = sortDistantTables( + { ...table1, name: 'table1', status: RemoteTableStatus.NOT_SYNCED }, + { ...table2, name: 'Table2' }, + ); + + expect(result).toBe(-1); + }); +}); diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util.ts new file mode 100644 index 000000000..718f53401 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util.ts @@ -0,0 +1,28 @@ +import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; + +export const sortDistantTables = ( + table1: { + status: RemoteTableStatus; + name: string; + }, + table2: { + status: RemoteTableStatus; + name: string; + }, +) => { + if ( + table1.status === RemoteTableStatus.SYNCED && + table2.status === RemoteTableStatus.NOT_SYNCED + ) { + return -1; + } + + if ( + table1.status === RemoteTableStatus.NOT_SYNCED && + table2.status === RemoteTableStatus.SYNCED + ) { + return 1; + } + + return table1.name.toUpperCase() > table2.name.toUpperCase() ? 1 : -1; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/util/stripe-distant-tables.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/stripe-distant-tables.util.ts similarity index 100% rename from packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/util/stripe-distant-tables.util.ts rename to packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/stripe-distant-tables.util.ts diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts index afbd64f8f..fd27ad05e 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service.ts @@ -2,10 +2,7 @@ import { Injectable } from '@nestjs/common'; import { getForeignTableColumnName as convertToForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util'; import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; -import { - RemoteTableStatus, - DistantTableUpdate, -} from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; +import { DistantTableUpdate } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util'; import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column'; @@ -23,53 +20,6 @@ export class RemoteTableSchemaUpdateService { private readonly workspaceDataSourceService: WorkspaceDataSourceService, ) {} - public async getDistantTablesWithUpdates({ - remoteServerSchema, - workspaceId, - remoteTables, - distantTables, - }: { - remoteServerSchema: string; - workspaceId: string; - remoteTables: RemoteTableEntity[]; - distantTables: DistantTables; - }) { - const schemaPendingUpdates = - await this.getSchemaUpdatesBetweenForeignAndDistantTables({ - workspaceId, - remoteTables, - distantTables, - }); - - const remoteTablesDistantNames = new Set( - remoteTables.map((remoteTable) => remoteTable.distantTableName), - ); - - const distantTablesWithUpdates = Object.keys(distantTables).map( - (tableName) => ({ - name: tableName, - schema: remoteServerSchema, - status: remoteTablesDistantNames.has(tableName) - ? RemoteTableStatus.SYNCED - : RemoteTableStatus.NOT_SYNCED, - schemaPendingUpdates: schemaPendingUpdates[tableName] || [], - }), - ); - - const deletedTables = Object.entries(schemaPendingUpdates) - .filter(([_tableName, updates]) => - updates.includes(DistantTableUpdate.TABLE_DELETED), - ) - .map(([tableName, updates]) => ({ - name: tableName, - schema: remoteServerSchema, - status: RemoteTableStatus.SYNCED, - schemaPendingUpdates: updates, - })); - - return [...distantTablesWithUpdates, ...deletedTables]; - } - public computeForeignTableColumnsUpdates = ( foreignTableColumns: PostgresTableSchemaColumn[], distantTableColumns: PostgresTableSchemaColumn[], @@ -94,7 +44,7 @@ export class RemoteTableSchemaUpdateService { return [...columnsAddedUpdates, ...columnsDeletedUpdates]; }; - private async getSchemaUpdatesBetweenForeignAndDistantTables({ + public async getSchemaUpdatesBetweenForeignAndDistantTables({ workspaceId, remoteTables, distantTables, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts index b49d913a7..77c56966a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts @@ -19,7 +19,7 @@ export class RemoteTableResolver { @Args('input') input: FindManyRemoteTablesInput, @AuthWorkspace() { id: workspaceId }: Workspace, ) { - return this.remoteTableService.findDistantTablesWithStatusByServerId( + return this.remoteTableService.findDistantTablesWithStatus( input.id, workspaceId, input.shouldFetchPendingSchemaUpdates, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index 564afde06..ed7062442 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -9,7 +9,10 @@ import { RemoteServerType, RemoteServerEntity, } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; -import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; +import { + DistantTableUpdate, + RemoteTableStatus, +} from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; import { mapUdtNameToFieldType, mapUdtNameToFieldSettings, @@ -31,6 +34,7 @@ import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-se import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util'; import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service'; import { RemoteTableSchemaUpdateService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service'; +import { sortDistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util'; export class RemoteTableService { private readonly logger = new Logger(RemoteTableService.name); @@ -52,7 +56,7 @@ export class RemoteTableService { private readonly remoteTableSchemaUpdateService: RemoteTableSchemaUpdateService, ) {} - public async findDistantTablesWithStatusByServerId( + public async findDistantTablesWithStatus( id: string, workspaceId: string, shouldFetchPendingSchemaUpdates?: boolean, @@ -82,26 +86,37 @@ export class RemoteTableService { workspaceId, ); - if (currentRemoteTables.length === 0 || !shouldFetchPendingSchemaUpdates) { - const distantTablesWithStatus = Object.keys(distantTables).map( - (tableName) => ({ - name: tableName, - schema: remoteServer.schema, - status: currentRemoteTableDistantNames.includes(tableName) - ? RemoteTableStatus.SYNCED - : RemoteTableStatus.NOT_SYNCED, - }), - ); + const distantTablesWithStatus = Object.keys(distantTables).map( + (tableName) => ({ + name: tableName, + schema: remoteServer.schema, + status: currentRemoteTableDistantNames.includes(tableName) + ? RemoteTableStatus.SYNCED + : RemoteTableStatus.NOT_SYNCED, + }), + ); - return distantTablesWithStatus; + if (!shouldFetchPendingSchemaUpdates) { + return distantTablesWithStatus.sort(sortDistantTables); } - return this.remoteTableSchemaUpdateService.getDistantTablesWithUpdates({ - remoteServerSchema: remoteServer.schema, - workspaceId, - remoteTables: currentRemoteTables, - distantTables, - }); + const schemaPendingUpdates = + await this.remoteTableSchemaUpdateService.getSchemaUpdatesBetweenForeignAndDistantTables( + { + workspaceId, + remoteTables: currentRemoteTables, + distantTables, + }, + ); + + const distantTablesWithPendingUpdates = + this.getDistantTablesWithPendingUpdates( + schemaPendingUpdates, + distantTablesWithStatus, + remoteServer.schema, + ); + + return distantTablesWithPendingUpdates.sort(sortDistantTables); } public async findRemoteTablesByServerId({ @@ -442,4 +457,32 @@ export class RemoteTableService { } } } + + private getDistantTablesWithPendingUpdates( + schemaPendingUpdates: { [tablename: string]: DistantTableUpdate[] }, + distantTablesWithStatus: { + name: string; + schema: string; + status: RemoteTableStatus; + }[], + remoteServerSchema: string, + ) { + const distantTablesWithUpdates = distantTablesWithStatus.map((table) => ({ + ...table, + schemaPendingUpdates: schemaPendingUpdates[table.name] || [], + })); + + const deletedTables = Object.entries(schemaPendingUpdates) + .filter(([_tableName, updates]) => + updates.includes(DistantTableUpdate.TABLE_DELETED), + ) + .map(([tableName, updates]) => ({ + name: tableName, + schema: remoteServerSchema, + status: RemoteTableStatus.SYNCED, + schemaPendingUpdates: updates, + })); + + return [...distantTablesWithUpdates, ...deletedTables]; + } }