Fetch available remote tables (#4665)

* Build remote table module

* Use transactions

* Export url builder in util

---------

Co-authored-by: Thomas Trompette <thomast@twenty.com>

packages/twenty-server/src/engine/core-modules/auth/auth.util.ts

@@ -16,11 +16,11 @@ export const compareHash = async (password: string, passwordHash: string) => {
   return bcrypt.compare(password, passwordHash);
 };
 
-export const encryptText = async (
+export const encryptText = (
   textToEncrypt: string,
   key: string,
   iv: string,
-): Promise<string> => {
+): string => {
   const keyHash = createHash('sha512')
     .update(key)
     .digest('hex')
@@ -35,11 +35,11 @@ export const encryptText = async (
   );
 };
 
-export const decryptText = async (
+export const decryptText = (
   textToDecrypt: string,
   key: string,
   iv: string,
-) => {
+): string => {
   const keyHash = createHash('sha512')
     .update(key)
     .digest('hex')

packages/twenty-server/src/engine/core-modules/core-engine.module.ts

@@ -10,7 +10,6 @@ import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timel
 import { TimelineCalendarEventModule } from 'src/engine/core-modules/calendar/timeline-calendar-event.module';
 import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
 import { HealthModule } from 'src/engine/core-modules/health/health.module';
-import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
 
 import { AnalyticsModule } from './analytics/analytics.module';
 import { FileModule } from './file/file.module';
@@ -31,7 +30,6 @@ import { ClientConfigModule } from './client-config/client-config.module';
     TimelineCalendarEventModule,
     UserModule,
     WorkspaceModule,
-    RemoteServerModule,
   ],
   exports: [
     AnalyticsModule,

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts

@@ -1,22 +1,21 @@
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 
-import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
 import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
 import { RemoteServerResolver } from 'src/engine/metadata-modules/remote-server/remote-server.resolver';
 import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
+import { RemoteTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.module';
 
 @Module({
   imports: [
-    TypeORMModule,
     TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
+    RemoteTableModule,
   ],
   providers: [
     RemoteServerService,
     RemoteServerResolver,
     ForeignDataWrapperQueryFactory,
   ],
-  exports: [RemoteServerService],
 })
 export class RemoteServerModule {}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts

@@ -1,10 +1,9 @@
 import { Injectable, NotFoundException } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
 
 import { v4 } from 'uuid';
-import { Repository } from 'typeorm';
+import { DataSource, EntityManager, Repository } from 'typeorm';
 
-import { TypeORMService } from 'src/database/typeorm/typeorm.service';
 import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/create-remote-server.input';
 import {
   RemoteServerEntity,
@@ -25,7 +24,8 @@ export class RemoteServerService<T extends RemoteServerType> {
     private readonly remoteServerRepository: Repository<
       RemoteServerEntity<RemoteServerType>
     >,
-    private readonly typeORMService: TypeORMService,
+    @InjectDataSource('metadata')
+    private readonly metadataDataSource: DataSource,
     private readonly environmentService: EnvironmentService,
     private readonly foreignDataWrapperQueryFactory: ForeignDataWrapperQueryFactory,
   ) {}
@@ -40,7 +40,6 @@ export class RemoteServerService<T extends RemoteServerType> {
       validateObject(remoteServerInput.userMappingOptions);
     }
 
-    const mainDatasource = this.typeORMService.getMainDataSource();
     const foreignDataWrapperId = v4();
 
     let remoteServerToCreate = {
@@ -67,31 +66,37 @@ export class RemoteServerService<T extends RemoteServerType> {
       };
     }
 
-    const createdRemoteServer =
+    return this.metadataDataSource.transaction(
-      await this.remoteServerRepository.create(remoteServerToCreate);
+      async (entityManager: EntityManager) => {
-
+        const createdRemoteServer = await entityManager.create(
-    const foreignDataWrapperQuery =
+          RemoteServerEntity,
-      this.foreignDataWrapperQueryFactory.createForeignDataWrapper(
+          remoteServerToCreate,
-        createdRemoteServer.foreignDataWrapperId,
-        remoteServerInput.foreignDataWrapperType,
-        remoteServerInput.foreignDataWrapperOptions,
-      );
-
-    await mainDatasource.query(foreignDataWrapperQuery);
-
-    if (remoteServerInput.userMappingOptions) {
-      const userMappingQuery =
-        this.foreignDataWrapperQueryFactory.createUserMapping(
-          createdRemoteServer.foreignDataWrapperId,
-          remoteServerInput.userMappingOptions,
         );
 
-      await mainDatasource.query(userMappingQuery);
+        const foreignDataWrapperQuery =
-    }
+          this.foreignDataWrapperQueryFactory.createForeignDataWrapper(
+            createdRemoteServer.foreignDataWrapperId,
+            remoteServerInput.foreignDataWrapperType,
+            remoteServerInput.foreignDataWrapperOptions,
+          );
 
-    await this.remoteServerRepository.save(createdRemoteServer);
+        await entityManager.query(foreignDataWrapperQuery);
 
-    return createdRemoteServer;
+        if (remoteServerInput.userMappingOptions) {
+          const userMappingQuery =
+            this.foreignDataWrapperQueryFactory.createUserMapping(
+              createdRemoteServer.foreignDataWrapperId,
+              remoteServerInput.userMappingOptions,
+            );
+
+          await entityManager.query(userMappingQuery);
+        }
+
+        await entityManager.save(RemoteServerEntity, createdRemoteServer);
+
+        return createdRemoteServer;
+      },
+    );
   }
 
   async deleteOneRemoteServer(
@@ -111,14 +116,16 @@ export class RemoteServerService<T extends RemoteServerType> {
       throw new NotFoundException('Object does not exist');
     }
 
-    const mainDatasource = this.typeORMService.getMainDataSource();
+    return this.metadataDataSource.transaction(
+      async (entityManager: EntityManager) => {
+        await entityManager.query(
+          `DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`,
+        );
+        await entityManager.delete(RemoteServerEntity, id);
 
-    await mainDatasource.query(
+        return remoteServer;
-      `DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`,
+      },
     );
-    await this.remoteServerRepository.delete(id);
-
-    return remoteServer;
   }
 
   public async findOneByIdWithinWorkspace(id: string, workspaceId: string) {

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts

@@ -0,0 +1,26 @@
+import { ObjectType, Field, registerEnumType } from '@nestjs/graphql';
+
+import { IsEnum } from 'class-validator';
+
+export enum RemoteTableStatus {
+  SYNCED = 'SYNCED',
+  NOT_SYNCED = 'NOT_SYNCED',
+}
+
+registerEnumType(RemoteTableStatus, {
+  name: 'RemoteTableStatus',
+  description: 'Status of the table',
+});
+
+@ObjectType('RemoteTable')
+export class RemoteTableDTO {
+  @Field(() => String)
+  name: string;
+
+  @IsEnum(RemoteTableStatus)
+  @Field(() => RemoteTableStatus)
+  status: RemoteTableStatus;
+
+  @Field(() => String)
+  schema: string;
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver';
+import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
+    WorkspaceDataSourceModule,
+  ],
+  providers: [RemoteTableService, RemoteTableResolver],
+})
+export class RemoteTableModule {}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts

@@ -0,0 +1,26 @@
+import { UseGuards } from '@nestjs/common';
+import { Args, Query, Resolver } from '@nestjs/graphql';
+
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
+import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
+import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
+import { RemoteTableDTO } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
+import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
+
+@UseGuards(JwtAuthGuard)
+@Resolver(() => RemoteTableDTO)
+export class RemoteTableResolver {
+  constructor(private readonly remoteTableService: RemoteTableService) {}
+
+  @Query(() => [RemoteTableDTO])
+  async findAvailableRemoteTablesByServerId(
+    @Args('input') { id }: RemoteServerIdInput,
+    @AuthWorkspace() { id: workspaceId }: Workspace,
+  ) {
+    return this.remoteTableService.findAvailableRemoteTablesByServerId(
+      id,
+      workspaceId,
+    );
+  }
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts

@@ -0,0 +1,112 @@
+import { NotFoundException } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { DataSource, Repository } from 'typeorm';
+
+import {
+  RemoteServerType,
+  RemoteServerEntity,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
+import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
+import {
+  EXCLUDED_POSTGRES_SCHEMAS,
+  buildPostgresUrl,
+} from 'src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util';
+
+export class RemoteTableService {
+  constructor(
+    @InjectRepository(RemoteServerEntity, 'metadata')
+    private readonly remoteServerRepository: Repository<
+      RemoteServerEntity<RemoteServerType>
+    >,
+    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
+    private readonly environmentService: EnvironmentService,
+  ) {}
+
+  public async findAvailableRemoteTablesByServerId(
+    id: string,
+    workspaceId: string,
+  ) {
+    const remoteServer = await this.remoteServerRepository.findOne({
+      where: {
+        id,
+        workspaceId,
+      },
+    });
+
+    if (!remoteServer) {
+      throw new NotFoundException('Remote server does not exist');
+    }
+
+    switch (remoteServer.foreignDataWrapperType) {
+      case RemoteServerType.POSTGRES_FDW:
+        return this.findAvailableRemotePostgresTables(
+          workspaceId,
+          remoteServer,
+        );
+      default:
+        throw new Error('Unsupported foreign data wrapper type');
+    }
+  }
+
+  // TODO: may be moved into a separated postgres table service once we have more use cases
+  private async findAvailableRemotePostgresTables(
+    workspaceId: string,
+    remoteServer: RemoteServerEntity<RemoteServerType>,
+  ) {
+    const remotePostgresTables =
+      await this.fetchTablesFromRemotePostgresSchema(remoteServer);
+
+    const workspaceDataSource =
+      await this.workspaceDataSourceService.connectToWorkspaceDataSource(
+        workspaceId,
+      );
+
+    const currentForeignTableNames = (
+      await workspaceDataSource.query(
+        `SELECT foreign_table_name FROM information_schema.foreign_tables`,
+      )
+    ).map((foreignTable) => foreignTable.foreign_table_name);
+
+    return remotePostgresTables.map((remoteTable) => ({
+      name: remoteTable.table_name,
+      schema: remoteTable.table_schema,
+      status: currentForeignTableNames.includes(remoteTable.table_name)
+        ? RemoteTableStatus.SYNCED
+        : RemoteTableStatus.NOT_SYNCED,
+    }));
+  }
+
+  private async fetchTablesFromRemotePostgresSchema(
+    remoteServer: RemoteServerEntity<RemoteServerType>,
+  ) {
+    const dataSource = new DataSource({
+      url: buildPostgresUrl(
+        this.environmentService.get('LOGIN_TOKEN_SECRET'),
+        remoteServer,
+      ),
+      type: 'postgres',
+      logging: true,
+    });
+
+    await dataSource.initialize();
+
+    const schemaNames = await dataSource.query(
+      `SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map(
+        (schema) => `'${schema}'`,
+      ).join(', ')} ) order by schema_name limit 1`,
+    );
+
+    const remotePostgresTables = await dataSource.query(
+      `SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames
+        .map((schemaName) => `'${schemaName.schema_name}'`)
+        .join(', ')})`,
+    );
+
+    await dataSource.destroy();
+
+    return remotePostgresTables;
+  }
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts

@@ -0,0 +1,29 @@
+import { decryptText } from 'src/engine/core-modules/auth/auth.util';
+import {
+  RemoteServerEntity,
+  RemoteServerType,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+
+export const EXCLUDED_POSTGRES_SCHEMAS = [
+  'information_schema',
+  'pg_catalog',
+  'pg_toast',
+];
+
+export const buildPostgresUrl = (
+  secretKey: string,
+  remoteServer: RemoteServerEntity<RemoteServerType>,
+): string => {
+  const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions;
+  const userMappingOptions = remoteServer.userMappingOptions;
+
+  const password = decryptText(
+    userMappingOptions.password,
+    secretKey,
+    secretKey,
+  );
+
+  const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`;
+
+  return url;
+};