Fetch available remote tables (#4665)

* Build remote table module

* Use transactions

* Export url builder in util

---------

Co-authored-by: Thomas Trompette <thomast@twenty.com>

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts

@@ -0,0 +1,26 @@
+import { ObjectType, Field, registerEnumType } from '@nestjs/graphql';
+
+import { IsEnum } from 'class-validator';
+
+export enum RemoteTableStatus {
+  SYNCED = 'SYNCED',
+  NOT_SYNCED = 'NOT_SYNCED',
+}
+
+registerEnumType(RemoteTableStatus, {
+  name: 'RemoteTableStatus',
+  description: 'Status of the table',
+});
+
+@ObjectType('RemoteTable')
+export class RemoteTableDTO {
+  @Field(() => String)
+  name: string;
+
+  @IsEnum(RemoteTableStatus)
+  @Field(() => RemoteTableStatus)
+  status: RemoteTableStatus;
+
+  @Field(() => String)
+  schema: string;
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts

@@ -0,0 +1,16 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver';
+import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
+    WorkspaceDataSourceModule,
+  ],
+  providers: [RemoteTableService, RemoteTableResolver],
+})
+export class RemoteTableModule {}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts

@@ -0,0 +1,26 @@
+import { UseGuards } from '@nestjs/common';
+import { Args, Query, Resolver } from '@nestjs/graphql';
+
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
+import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
+import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
+import { RemoteTableDTO } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
+import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
+
+@UseGuards(JwtAuthGuard)
+@Resolver(() => RemoteTableDTO)
+export class RemoteTableResolver {
+  constructor(private readonly remoteTableService: RemoteTableService) {}
+
+  @Query(() => [RemoteTableDTO])
+  async findAvailableRemoteTablesByServerId(
+    @Args('input') { id }: RemoteServerIdInput,
+    @AuthWorkspace() { id: workspaceId }: Workspace,
+  ) {
+    return this.remoteTableService.findAvailableRemoteTablesByServerId(
+      id,
+      workspaceId,
+    );
+  }
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts

@@ -0,0 +1,112 @@
+import { NotFoundException } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { DataSource, Repository } from 'typeorm';
+
+import {
+  RemoteServerType,
+  RemoteServerEntity,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
+import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
+import {
+  EXCLUDED_POSTGRES_SCHEMAS,
+  buildPostgresUrl,
+} from 'src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util';
+
+export class RemoteTableService {
+  constructor(
+    @InjectRepository(RemoteServerEntity, 'metadata')
+    private readonly remoteServerRepository: Repository<
+      RemoteServerEntity<RemoteServerType>
+    >,
+    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
+    private readonly environmentService: EnvironmentService,
+  ) {}
+
+  public async findAvailableRemoteTablesByServerId(
+    id: string,
+    workspaceId: string,
+  ) {
+    const remoteServer = await this.remoteServerRepository.findOne({
+      where: {
+        id,
+        workspaceId,
+      },
+    });
+
+    if (!remoteServer) {
+      throw new NotFoundException('Remote server does not exist');
+    }
+
+    switch (remoteServer.foreignDataWrapperType) {
+      case RemoteServerType.POSTGRES_FDW:
+        return this.findAvailableRemotePostgresTables(
+          workspaceId,
+          remoteServer,
+        );
+      default:
+        throw new Error('Unsupported foreign data wrapper type');
+    }
+  }
+
+  // TODO: may be moved into a separated postgres table service once we have more use cases
+  private async findAvailableRemotePostgresTables(
+    workspaceId: string,
+    remoteServer: RemoteServerEntity<RemoteServerType>,
+  ) {
+    const remotePostgresTables =
+      await this.fetchTablesFromRemotePostgresSchema(remoteServer);
+
+    const workspaceDataSource =
+      await this.workspaceDataSourceService.connectToWorkspaceDataSource(
+        workspaceId,
+      );
+
+    const currentForeignTableNames = (
+      await workspaceDataSource.query(
+        `SELECT foreign_table_name FROM information_schema.foreign_tables`,
+      )
+    ).map((foreignTable) => foreignTable.foreign_table_name);
+
+    return remotePostgresTables.map((remoteTable) => ({
+      name: remoteTable.table_name,
+      schema: remoteTable.table_schema,
+      status: currentForeignTableNames.includes(remoteTable.table_name)
+        ? RemoteTableStatus.SYNCED
+        : RemoteTableStatus.NOT_SYNCED,
+    }));
+  }
+
+  private async fetchTablesFromRemotePostgresSchema(
+    remoteServer: RemoteServerEntity<RemoteServerType>,
+  ) {
+    const dataSource = new DataSource({
+      url: buildPostgresUrl(
+        this.environmentService.get('LOGIN_TOKEN_SECRET'),
+        remoteServer,
+      ),
+      type: 'postgres',
+      logging: true,
+    });
+
+    await dataSource.initialize();
+
+    const schemaNames = await dataSource.query(
+      `SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map(
+        (schema) => `'${schema}'`,
+      ).join(', ')} ) order by schema_name limit 1`,
+    );
+
+    const remotePostgresTables = await dataSource.query(
+      `SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames
+        .map((schemaName) => `'${schemaName.schema_name}'`)
+        .join(', ')})`,
+    );
+
+    await dataSource.destroy();
+
+    return remotePostgresTables;
+  }
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts

@@ -0,0 +1,29 @@
+import { decryptText } from 'src/engine/core-modules/auth/auth.util';
+import {
+  RemoteServerEntity,
+  RemoteServerType,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+
+export const EXCLUDED_POSTGRES_SCHEMAS = [
+  'information_schema',
+  'pg_catalog',
+  'pg_toast',
+];
+
+export const buildPostgresUrl = (
+  secretKey: string,
+  remoteServer: RemoteServerEntity<RemoteServerType>,
+): string => {
+  const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions;
+  const userMappingOptions = remoteServer.userMappingOptions;
+
+  const password = decryptText(
+    userMappingOptions.password,
+    secretKey,
+    secretKey,
+  );
+
+  const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`;
+
+  return url;
+};