From 279d99487cc563d6320bc85df0b32881e365fc3e Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Tue, 26 Mar 2024 15:50:41 +0100 Subject: [PATCH] Fetch available remote tables (#4665) * Build remote table module * Use transactions * Export url builder in util --------- Co-authored-by: Thomas Trompette --- .../src/engine/core-modules/auth/auth.util.ts | 8 +- .../engine/core-modules/core-engine.module.ts | 2 - .../remote-server/remote-server.module.ts | 5 +- .../remote-server/remote-server.service.ts | 71 ++++++----- .../remote-table/dtos/remote-table.dto.ts | 26 ++++ .../remote-table/remote-table.module.ts | 16 +++ .../remote-table/remote-table.resolver.ts | 26 ++++ .../remote-table/remote-table.service.ts | 112 ++++++++++++++++++ .../utils/remote-table-postgres.util.ts | 29 +++++ 9 files changed, 254 insertions(+), 41 deletions(-) create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts diff --git a/packages/twenty-server/src/engine/core-modules/auth/auth.util.ts b/packages/twenty-server/src/engine/core-modules/auth/auth.util.ts index 0a3346be4..2cbc7aaea 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/auth.util.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/auth.util.ts @@ -16,11 +16,11 @@ export const compareHash = async (password: string, passwordHash: string) => { return bcrypt.compare(password, passwordHash); }; -export const encryptText = async ( +export const encryptText = ( textToEncrypt: string, key: string, iv: string, -): Promise => { +): string => { const keyHash = createHash('sha512') .update(key) .digest('hex') @@ -35,11 +35,11 @@ export const encryptText = async ( ); }; -export const decryptText = async ( +export const decryptText = ( textToDecrypt: string, key: string, iv: string, -) => { +): string => { const keyHash = createHash('sha512') .update(key) .digest('hex') diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index a74284d09..c61f55b2f 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -10,7 +10,6 @@ import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timel import { TimelineCalendarEventModule } from 'src/engine/core-modules/calendar/timeline-calendar-event.module'; import { BillingModule } from 'src/engine/core-modules/billing/billing.module'; import { HealthModule } from 'src/engine/core-modules/health/health.module'; -import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module'; import { AnalyticsModule } from './analytics/analytics.module'; import { FileModule } from './file/file.module'; @@ -31,7 +30,6 @@ import { ClientConfigModule } from './client-config/client-config.module'; TimelineCalendarEventModule, UserModule, WorkspaceModule, - RemoteServerModule, ], exports: [ AnalyticsModule, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts index 4041b4dc8..9fc4bdbcb 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts @@ -1,22 +1,21 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory'; import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; import { RemoteServerResolver } from 'src/engine/metadata-modules/remote-server/remote-server.resolver'; import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service'; +import { RemoteTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.module'; @Module({ imports: [ - TypeORMModule, TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'), + RemoteTableModule, ], providers: [ RemoteServerService, RemoteServerResolver, ForeignDataWrapperQueryFactory, ], - exports: [RemoteServerService], }) export class RemoteServerModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts index 9064fc733..c16b048a3 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts @@ -1,10 +1,9 @@ import { Injectable, NotFoundException } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; import { v4 } from 'uuid'; -import { Repository } from 'typeorm'; +import { DataSource, EntityManager, Repository } from 'typeorm'; -import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/create-remote-server.input'; import { RemoteServerEntity, @@ -25,7 +24,8 @@ export class RemoteServerService { private readonly remoteServerRepository: Repository< RemoteServerEntity >, - private readonly typeORMService: TypeORMService, + @InjectDataSource('metadata') + private readonly metadataDataSource: DataSource, private readonly environmentService: EnvironmentService, private readonly foreignDataWrapperQueryFactory: ForeignDataWrapperQueryFactory, ) {} @@ -40,7 +40,6 @@ export class RemoteServerService { validateObject(remoteServerInput.userMappingOptions); } - const mainDatasource = this.typeORMService.getMainDataSource(); const foreignDataWrapperId = v4(); let remoteServerToCreate = { @@ -67,31 +66,37 @@ export class RemoteServerService { }; } - const createdRemoteServer = - await this.remoteServerRepository.create(remoteServerToCreate); - - const foreignDataWrapperQuery = - this.foreignDataWrapperQueryFactory.createForeignDataWrapper( - createdRemoteServer.foreignDataWrapperId, - remoteServerInput.foreignDataWrapperType, - remoteServerInput.foreignDataWrapperOptions, - ); - - await mainDatasource.query(foreignDataWrapperQuery); - - if (remoteServerInput.userMappingOptions) { - const userMappingQuery = - this.foreignDataWrapperQueryFactory.createUserMapping( - createdRemoteServer.foreignDataWrapperId, - remoteServerInput.userMappingOptions, + return this.metadataDataSource.transaction( + async (entityManager: EntityManager) => { + const createdRemoteServer = await entityManager.create( + RemoteServerEntity, + remoteServerToCreate, ); - await mainDatasource.query(userMappingQuery); - } + const foreignDataWrapperQuery = + this.foreignDataWrapperQueryFactory.createForeignDataWrapper( + createdRemoteServer.foreignDataWrapperId, + remoteServerInput.foreignDataWrapperType, + remoteServerInput.foreignDataWrapperOptions, + ); - await this.remoteServerRepository.save(createdRemoteServer); + await entityManager.query(foreignDataWrapperQuery); - return createdRemoteServer; + if (remoteServerInput.userMappingOptions) { + const userMappingQuery = + this.foreignDataWrapperQueryFactory.createUserMapping( + createdRemoteServer.foreignDataWrapperId, + remoteServerInput.userMappingOptions, + ); + + await entityManager.query(userMappingQuery); + } + + await entityManager.save(RemoteServerEntity, createdRemoteServer); + + return createdRemoteServer; + }, + ); } async deleteOneRemoteServer( @@ -111,14 +116,16 @@ export class RemoteServerService { throw new NotFoundException('Object does not exist'); } - const mainDatasource = this.typeORMService.getMainDataSource(); + return this.metadataDataSource.transaction( + async (entityManager: EntityManager) => { + await entityManager.query( + `DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`, + ); + await entityManager.delete(RemoteServerEntity, id); - await mainDatasource.query( - `DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`, + return remoteServer; + }, ); - await this.remoteServerRepository.delete(id); - - return remoteServer; } public async findOneByIdWithinWorkspace(id: string, workspaceId: string) { diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts new file mode 100644 index 000000000..2997f3532 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts @@ -0,0 +1,26 @@ +import { ObjectType, Field, registerEnumType } from '@nestjs/graphql'; + +import { IsEnum } from 'class-validator'; + +export enum RemoteTableStatus { + SYNCED = 'SYNCED', + NOT_SYNCED = 'NOT_SYNCED', +} + +registerEnumType(RemoteTableStatus, { + name: 'RemoteTableStatus', + description: 'Status of the table', +}); + +@ObjectType('RemoteTable') +export class RemoteTableDTO { + @Field(() => String) + name: string; + + @IsEnum(RemoteTableStatus) + @Field(() => RemoteTableStatus) + status: RemoteTableStatus; + + @Field(() => String) + schema: string; +} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts new file mode 100644 index 000000000..ea99ba001 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; +import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver'; +import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'), + WorkspaceDataSourceModule, + ], + providers: [RemoteTableService, RemoteTableResolver], +}) +export class RemoteTableModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts new file mode 100644 index 000000000..631e8e44a --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts @@ -0,0 +1,26 @@ +import { UseGuards } from '@nestjs/common'; +import { Args, Query, Resolver } from '@nestjs/graphql'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; +import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; +import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input'; +import { RemoteTableDTO } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; +import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service'; + +@UseGuards(JwtAuthGuard) +@Resolver(() => RemoteTableDTO) +export class RemoteTableResolver { + constructor(private readonly remoteTableService: RemoteTableService) {} + + @Query(() => [RemoteTableDTO]) + async findAvailableRemoteTablesByServerId( + @Args('input') { id }: RemoteServerIdInput, + @AuthWorkspace() { id: workspaceId }: Workspace, + ) { + return this.remoteTableService.findAvailableRemoteTablesByServerId( + id, + workspaceId, + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts new file mode 100644 index 000000000..3fb76711a --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -0,0 +1,112 @@ +import { NotFoundException } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { DataSource, Repository } from 'typeorm'; + +import { + RemoteServerType, + RemoteServerEntity, +} from 'src/engine/metadata-modules/remote-server/remote-server.entity'; +import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; +import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; +import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; +import { + EXCLUDED_POSTGRES_SCHEMAS, + buildPostgresUrl, +} from 'src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util'; + +export class RemoteTableService { + constructor( + @InjectRepository(RemoteServerEntity, 'metadata') + private readonly remoteServerRepository: Repository< + RemoteServerEntity + >, + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + private readonly environmentService: EnvironmentService, + ) {} + + public async findAvailableRemoteTablesByServerId( + id: string, + workspaceId: string, + ) { + const remoteServer = await this.remoteServerRepository.findOne({ + where: { + id, + workspaceId, + }, + }); + + if (!remoteServer) { + throw new NotFoundException('Remote server does not exist'); + } + + switch (remoteServer.foreignDataWrapperType) { + case RemoteServerType.POSTGRES_FDW: + return this.findAvailableRemotePostgresTables( + workspaceId, + remoteServer, + ); + default: + throw new Error('Unsupported foreign data wrapper type'); + } + } + + // TODO: may be moved into a separated postgres table service once we have more use cases + private async findAvailableRemotePostgresTables( + workspaceId: string, + remoteServer: RemoteServerEntity, + ) { + const remotePostgresTables = + await this.fetchTablesFromRemotePostgresSchema(remoteServer); + + const workspaceDataSource = + await this.workspaceDataSourceService.connectToWorkspaceDataSource( + workspaceId, + ); + + const currentForeignTableNames = ( + await workspaceDataSource.query( + `SELECT foreign_table_name FROM information_schema.foreign_tables`, + ) + ).map((foreignTable) => foreignTable.foreign_table_name); + + return remotePostgresTables.map((remoteTable) => ({ + name: remoteTable.table_name, + schema: remoteTable.table_schema, + status: currentForeignTableNames.includes(remoteTable.table_name) + ? RemoteTableStatus.SYNCED + : RemoteTableStatus.NOT_SYNCED, + })); + } + + private async fetchTablesFromRemotePostgresSchema( + remoteServer: RemoteServerEntity, + ) { + const dataSource = new DataSource({ + url: buildPostgresUrl( + this.environmentService.get('LOGIN_TOKEN_SECRET'), + remoteServer, + ), + type: 'postgres', + logging: true, + }); + + await dataSource.initialize(); + + const schemaNames = await dataSource.query( + `SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map( + (schema) => `'${schema}'`, + ).join(', ')} ) order by schema_name limit 1`, + ); + + const remotePostgresTables = await dataSource.query( + `SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames + .map((schemaName) => `'${schemaName.schema_name}'`) + .join(', ')})`, + ); + + await dataSource.destroy(); + + return remotePostgresTables; + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts new file mode 100644 index 000000000..a044ebe9f --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts @@ -0,0 +1,29 @@ +import { decryptText } from 'src/engine/core-modules/auth/auth.util'; +import { + RemoteServerEntity, + RemoteServerType, +} from 'src/engine/metadata-modules/remote-server/remote-server.entity'; + +export const EXCLUDED_POSTGRES_SCHEMAS = [ + 'information_schema', + 'pg_catalog', + 'pg_toast', +]; + +export const buildPostgresUrl = ( + secretKey: string, + remoteServer: RemoteServerEntity, +): string => { + const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions; + const userMappingOptions = remoteServer.userMappingOptions; + + const password = decryptText( + userMappingOptions.password, + secretKey, + secretKey, + ); + + const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`; + + return url; +};