Minor refacto and fixes on Remotes updates (#5438)

In this PR - Code refactoring - v0 of adding "updates available" info in Connection sync status <img width="835" alt="Capture d’écran 2024-05-16 à 17 02 07" src="https://github.com/twentyhq/twenty/assets/51697796/9674d3ca-bed2-4520-a5a6-ba37bc242d06"> - fix distant table columns with not-camel case names are always considered as new
2024-05-16 17:31:34 +02:00
parent 9bc9513845
commit d741f4a5bd
15 changed files with 122 additions and 104 deletions
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts
@ -9,9 +9,9 @@ import {
  RemoteServerType,
 } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
 import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
-import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column';
 import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table';
 import { STRIPE_DISTANT_TABLES } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/util/stripe-distant-tables.util';
+import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';

@Injectable()
 export class DistantTableService {
@ -26,7 +26,7 @@ export class DistantTableService {
  public getDistantTableColumns(
    remoteServer: RemoteServerEntity<RemoteServerType>,
    tableName: string,
-  ): DistantTableColumn[] {
+  ): PostgresTableSchemaColumn[] {
    if (!remoteServer.availableTables) {
      throw new BadRequestException(
        'Remote server available tables are not defined',
@ -39,13 +39,8 @@ export class DistantTableService {
  public async fetchDistantTables(
    remoteServer: RemoteServerEntity<RemoteServerType>,
    workspaceId: string,
-    refreshData?: boolean,
  ): Promise<DistantTables> {
-    if (!refreshData && remoteServer.availableTables) {
-      return remoteServer.availableTables;
-    }
-
-    return await this.createAvailableTables(remoteServer, workspaceId);
+    return this.createAvailableTables(remoteServer, workspaceId);
  }

  private async createAvailableTables(
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column.ts
@ -1,6 +0,0 @@
-// Type will evolve as we add more remote table types
-export type DistantTableColumn = {
-  columnName: string;
-  dataType: string;
-  udtName: string;
-};
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts
@ -1,5 +1,5 @@
-import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column';
+import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';

 export type DistantTables = {
-  [tableName: string]: DistantTableColumn[];
+  [tableName: string]: PostgresTableSchemaColumn[];
 };
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/find-many-remote-tables-input.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/find-many-remote-tables-input.ts
@ -1,17 +1,9 @@
-import { InputType, ID, Field } from '@nestjs/graphql';
+import { InputType, ID } from '@nestjs/graphql';

 import { IDField } from '@ptc-org/nestjs-query-graphql';
-import { IsOptional } from 'class-validator';

@InputType()
 export class FindManyRemoteTablesInput {
  @IDField(() => ID, { description: 'The id of the remote server.' })
  id!: string;
-
-  @IsOptional()
-  @Field(() => Boolean, {
-    description: 'Indicates if data from distant tables should be refreshed.',
-    nullable: true,
-  })
-  refreshData?: boolean;
 }
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts
@ -10,11 +10,11 @@ export enum RemoteTableStatus {
  NOT_SYNCED = 'NOT_SYNCED',
 }

-export enum TableUpdate {
+export enum DistantTableUpdate {
  TABLE_DELETED = 'TABLE_DELETED',
-  COLUMNS_DELETED = 'COLUMN_DELETED',
-  COLUMNS_ADDED = 'COLUMN_ADDED',
-  COLUMNS_TYPE_CHANGED = 'COLUMN_TYPE_CHANGED',
+  COLUMNS_DELETED = 'COLUMNS_DELETED',
+  COLUMNS_ADDED = 'COLUMNS_ADDED',
+  COLUMNS_TYPE_CHANGED = 'COLUMNS_TYPE_CHANGED',
 }

 registerEnumType(RemoteTableStatus, {
@ -22,8 +22,8 @@ registerEnumType(RemoteTableStatus, {
  description: 'Status of the table',
 });

-registerEnumType(TableUpdate, {
-  name: 'TableUpdate',
+registerEnumType(DistantTableUpdate, {
+  name: 'DistantTableUpdate',
  description: 'Schema update on a table',
 });

@ -43,6 +43,6 @@ export class RemoteTableDTO {
  schema?: string;

  @IsOptional()
-  @Field(() => [TableUpdate], { nullable: true })
-  schemaPendingUpdates?: [TableUpdate];
+  @Field(() => [DistantTableUpdate], { nullable: true })
+  schemaPendingUpdates?: [DistantTableUpdate];
 }
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts
@ -22,7 +22,6 @@ export class RemoteTableResolver {
    return this.remoteTableService.findDistantTablesByServerId(
      input.id,
      workspaceId,
-      input.refreshData,
    );
  }

--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts
@ -10,7 +10,7 @@ import {
 } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
 import {
  RemoteTableStatus,
-  TableUpdate,
+  DistantTableUpdate,
 } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
 import {
  mapUdtNameToFieldType,
@ -38,9 +38,9 @@ import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/work
 import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
 import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util';
 import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service';
-import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column';
 import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table';
-import { RemoteTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/remote-table/types/remote-table-schema-column';
+import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-foreign-table-column-name.util';
+import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';

 export class RemoteTableService {
  private readonly logger = new Logger(RemoteTableService.name);
@ -62,11 +62,7 @@ export class RemoteTableService {
    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
  ) {}

-  public async findDistantTablesByServerId(
-    id: string,
-    workspaceId: string,
-    refreshData?: boolean,
-  ) {
+  public async findDistantTablesByServerId(id: string, workspaceId: string) {
    const remoteServer = await this.remoteServerRepository.findOne({
      where: {
        id,
@ -90,10 +86,9 @@ export class RemoteTableService {
    const distantTables = await this.distantTableService.fetchDistantTables(
      remoteServer,
      workspaceId,
-      refreshData,
    );

-    if (!refreshData || currentRemoteTables.length === 0) {
+    if (currentRemoteTables.length === 0) {
      const distantTablesWithStatus = Object.keys(distantTables).map(
        (tableName) => ({
          name: tableName,
@ -107,18 +102,41 @@ export class RemoteTableService {
      return distantTablesWithStatus;
    }

+    return this.getDistantTablesWithUpdates({
+      remoteServerSchema: remoteServer.schema,
+      workspaceId,
+      remoteTables: currentRemoteTables,
+      distantTables,
+    });
+  }
+
+  private async getDistantTablesWithUpdates({
+    remoteServerSchema,
+    workspaceId,
+    remoteTables,
+    distantTables,
+  }: {
+    remoteServerSchema: string;
+    workspaceId: string;
+    remoteTables: RemoteTableEntity[];
+    distantTables: DistantTables;
+  }) {
    const schemaPendingUpdates =
-      await this.getSchemaUpdatesBetweenRemoteAndDistantTables({
+      await this.getSchemaUpdatesBetweenForeignAndDistantTables({
        workspaceId,
-        remoteTables: currentRemoteTables,
+        remoteTables,
        distantTables,
      });

+    const remoteTablesDistantNames = remoteTables.map(
+      (remoteTable) => remoteTable.distantTableName,
+    );
+
    const distantTablesWithUpdates = Object.keys(distantTables).map(
      (tableName) => ({
        name: tableName,
-        schema: remoteServer.schema,
-        status: currentRemoteTableDistantNames.includes(tableName)
+        schema: remoteServerSchema,
+        status: remoteTablesDistantNames.includes(tableName)
          ? RemoteTableStatus.SYNCED
          : RemoteTableStatus.NOT_SYNCED,
        schemaPendingUpdates: schemaPendingUpdates[tableName],
@ -127,11 +145,11 @@ export class RemoteTableService {

    const deletedTables = Object.entries(schemaPendingUpdates)
      .filter(([_tableName, updates]) =>
-        updates.includes(TableUpdate.TABLE_DELETED),
+        updates.includes(DistantTableUpdate.TABLE_DELETED),
      )
      .map(([tableName, updates]) => ({
        name: tableName,
-        schema: remoteServer.schema,
+        schema: remoteServerSchema,
        status: RemoteTableStatus.SYNCED,
        schemaPendingUpdates: updates,
      }));
@ -139,7 +157,7 @@ export class RemoteTableService {
    return distantTablesWithUpdates.concat(deletedTables);
  }

-  private async getSchemaUpdatesBetweenRemoteAndDistantTables({
+  private async getSchemaUpdatesBetweenForeignAndDistantTables({
    workspaceId,
    remoteTables,
    distantTables,
@ -147,7 +165,7 @@ export class RemoteTableService {
    workspaceId: string;
    remoteTables: RemoteTableEntity[];
    distantTables: DistantTables;
-  }): Promise<{ [tablename: string]: TableUpdate[] }> {
+  }): Promise<{ [tablename: string]: DistantTableUpdate[] }> {
    const updates = {};

    for (const remoteTable of remoteTables) {
@ -155,37 +173,39 @@ export class RemoteTableService {
      const tableName = remoteTable.distantTableName;

      if (!distantTable) {
-        updates[tableName] = [TableUpdate.TABLE_DELETED];
+        updates[tableName] = [DistantTableUpdate.TABLE_DELETED];
        continue;
      }

-      const distantColumnNames = new Set(
-        distantTable.map((column) => column.columnName),
+      const distantTableColumnNames = new Set(
+        distantTable.map((column) =>
+          getForeignTableColumnName(column.columnName),
+        ),
      );
-      const localColumnNames = new Set(
+      const foreignTableColumnNames = new Set(
        (
          await this.fetchTableColumns(workspaceId, remoteTable.localTableName)
-        ).map((column) => column.column_name),
+        ).map((column) => column.columnName),
      );

-      const columnsAdded = [...distantColumnNames].filter(
-        (columnName) => !localColumnNames.has(columnName),
+      const columnsAdded = [...distantTableColumnNames].filter(
+        (columnName) => !foreignTableColumnNames.has(columnName),
      );

-      const columnsDeleted = [...localColumnNames].filter(
-        (columnName) => !distantColumnNames.has(columnName),
+      const columnsDeleted = [...foreignTableColumnNames].filter(
+        (columnName) => !distantTableColumnNames.has(columnName),
      );

      if (columnsAdded.length > 0) {
        updates[tableName] = [
          ...(updates[tableName] || []),
-          TableUpdate.COLUMNS_ADDED,
+          DistantTableUpdate.COLUMNS_ADDED,
        ];
      }
      if (columnsDeleted.length > 0) {
        updates[tableName] = [
          ...(updates[tableName] || []),
-          TableUpdate.COLUMNS_DELETED,
+          DistantTableUpdate.COLUMNS_DELETED,
        ];
      }
    }
@ -431,7 +451,7 @@ export class RemoteTableService {
  private async fetchTableColumns(
    workspaceId: string,
    tableName: string,
-  ): Promise<RemoteTableSchemaColumn[]> {
+  ): Promise<PostgresTableSchemaColumn[]> {
    const workspaceDataSource =
      await this.workspaceDataSourceService.connectToWorkspaceDataSource(
        workspaceId,
@ -440,11 +460,17 @@ export class RemoteTableService {
    const schemaName =
      this.workspaceDataSourceService.getSchemaName(workspaceId);

-    return await workspaceDataSource.query(
-      `SELECT column_name, data_type
+    const res = await workspaceDataSource.query(
+      `SELECT column_name, data_type, udt_name
        FROM information_schema.columns
        WHERE table_schema = '${schemaName}' AND table_name = '${tableName}'`,
    );
+
+    return res.map((column) => ({
+      columnName: column.column_name,
+      dataType: column.data_type,
+      udtName: column.udt_name,
+    }));
  }

  private async createForeignTable(
@ -452,7 +478,7 @@ export class RemoteTableService {
    localTableName: string,
    remoteTableInput: RemoteTableInput,
    remoteServer: RemoteServerEntity<RemoteServerType>,
-    distantTableColumns: DistantTableColumn[],
+    distantTableColumns: PostgresTableSchemaColumn[],
  ) {
    const referencedTable: ReferencedTable = this.buildReferencedTable(
      remoteServer,
@ -471,7 +497,7 @@ export class RemoteTableService {
              columns: distantTableColumns.map(
                (column) =>
                  ({
-                    columnName: camelCase(column.columnName),
+                    columnName: getForeignTableColumnName(column.columnName),
                    columnType: column.dataType,
                    distantColumnName: column.columnName,
                  }) satisfies WorkspaceMigrationForeignColumnDefinition,
@ -500,8 +526,8 @@ export class RemoteTableService {
  private async createRemoteTableMetadata(
    workspaceId: string,
    localTableName: string,
-    distantTableColumns: DistantTableColumn[],
-    distantTableIdColumn: DistantTableColumn,
+    distantTableColumns: PostgresTableSchemaColumn[],
+    distantTableIdColumn: PostgresTableSchemaColumn,
    dataSourceMetadataId: string,
  ) {
    const objectMetadata = await this.objectMetadataService.createOne({
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table-schema-column.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table-schema-column.ts
@ -1,4 +0,0 @@
-export type RemoteTableSchemaColumn = {
-  column_name: string;
-  data_type: string;
-};
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-foreign-table-column-name.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/get-foreign-table-column-name.util.ts
@ -0,0 +1,5 @@
+import camelCase from 'lodash.camelcase';
+
+export const getForeignTableColumnName = (distantTableColumnName: string) => {
+  return camelCase(distantTableColumnName);
+};
--- a/packages/twenty-server/src/engine/metadata-modules/remote-server/types/postgres-table-schema-column.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/types/postgres-table-schema-column.ts
@ -0,0 +1,5 @@
+export type PostgresTableSchemaColumn = {
+  columnName: string;
+  dataType: string;
+  udtName: string;
+};