Add endpoints to create and delete remote server (#4606)

* Build remote server

* Add getters

* Migrate to json inputs

* Use extendable type

* Use regex validation

* Remove acronymes

---------

Co-authored-by: Thomas Trompette <thomast@twenty.com>

packages/twenty-server/.env.test

@@ -9,6 +9,7 @@ FRONT_BASE_URL=http://localhost:3001
 ACCESS_TOKEN_SECRET=secret_jwt
 LOGIN_TOKEN_SECRET=secret_login_tokens
 REFRESH_TOKEN_SECRET=secret_refresh_token
+FILE_TOKEN_SECRET=replace_me_with_a_random_string_refresh
 
 # ———————— Optional ————————
 # DEBUG_MODE=false

packages/twenty-server/src/database/typeorm/metadata/migrations/1711374137222-addRemoteServerTable.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddRemoteServerTable1711374137222 implements MigrationInterface {
+  name = 'AddRemoteServerTable1711374137222';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "metadata"."remoteServer" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "foreignDataWrapperId" uuid NOT NULL DEFAULT uuid_generate_v4(), "foreignDataWrapperType" character varying, "foreignDataWrapperOptions" jsonb, "userMappingOptions" jsonb, "workspaceId" uuid NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_8e5d208498fa2c9710bb934023a" PRIMARY KEY ("id"))`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE "metadata"."remoteServer"`);
+  }
+}

packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts

@@ -5,12 +5,12 @@ import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs';
 
 import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
 import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
+import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
+import { CreateContextFactory } from 'src/engine/api/graphql/graphql-config/factories/create-context.factory';
+import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module';
 import { metadataModuleFactory } from 'src/engine/api/graphql/metadata.module-factory';
 import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
 import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service';
-import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module';
-import { CreateContextFactory } from 'src/engine/api/graphql/graphql-config/factories/create-context.factory';
-import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
 
 @Module({
   imports: [

packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/factories.ts

@@ -1,3 +1,5 @@
+import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
+
 import { ArgsAliasFactory } from './args-alias.factory';
 import { ArgsStringFactory } from './args-string.factory';
 import { RelationFieldAliasFactory } from './relation-field-alias.factory';
@@ -28,4 +30,5 @@ export const workspaceQueryBuilderFactories = [
   UpdateOneQueryFactory,
   UpdateManyQueryFactory,
   DeleteManyQueryFactory,
+  ForeignDataWrapperQueryFactory,
 ];

packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory.ts

@@ -0,0 +1,48 @@
+import { Injectable } from '@nestjs/common';
+
+import {
+  ForeignDataWrapperOptions,
+  RemoteServerType,
+  UserMappingOptions,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+
+@Injectable()
+export class ForeignDataWrapperQueryFactory {
+  createForeignDataWrapper(
+    foreignDataWrapperId: string,
+    foreignDataWrapperType: RemoteServerType,
+    foreignDataWrapperOptions: ForeignDataWrapperOptions<RemoteServerType>,
+  ) {
+    const [name, options] = this.buildNameAndOptionsFromType(
+      foreignDataWrapperType,
+      foreignDataWrapperOptions,
+    );
+
+    return `CREATE SERVER "${foreignDataWrapperId}" FOREIGN DATA WRAPPER ${name} OPTIONS (${options})`;
+  }
+
+  createUserMapping(
+    foreignDataWrapperId: string,
+    userMappingOptions: UserMappingOptions,
+  ) {
+    return `CREATE USER MAPPING IF NOT EXISTS FOR ${userMappingOptions.username} SERVER "${foreignDataWrapperId}" OPTIONS (user '${userMappingOptions.username}', password '${userMappingOptions.password}')`;
+  }
+
+  private buildNameAndOptionsFromType(
+    type: RemoteServerType,
+    options: ForeignDataWrapperOptions<RemoteServerType>,
+  ) {
+    switch (type) {
+      case RemoteServerType.POSTGRES_FDW:
+        return ['postgres_fdw', this.buildPostgresFDWQueryOptions(options)];
+      default:
+        throw new Error('Foreign data wrapper type not supported');
+    }
+  }
+
+  private buildPostgresFDWQueryOptions(
+    foreignDataWrapperOptions: ForeignDataWrapperOptions<RemoteServerType>,
+  ) {
+    return `dbname '${foreignDataWrapperOptions.dbname}', host '${foreignDataWrapperOptions.host}', port '${foreignDataWrapperOptions.port}'`;
+  }
+}

packages/twenty-server/src/engine/core-modules/auth/auth.util.ts

@@ -1,3 +1,5 @@
+import { createCipheriv, createDecipheriv, createHash } from 'crypto';
+
 import * as bcrypt from 'bcrypt';
 
 export const PASSWORD_REGEX = /^.{8,}$/;
@@ -13,3 +15,42 @@ export const hashPassword = async (password: string) => {
 export const compareHash = async (password: string, passwordHash: string) => {
   return bcrypt.compare(password, passwordHash);
 };
+
+export const encryptText = async (
+  textToEncrypt: string,
+  key: string,
+  iv: string,
+): Promise<string> => {
+  const keyHash = createHash('sha512')
+    .update(key)
+    .digest('hex')
+    .substring(0, 32);
+
+  const ivHash = createHash('sha512').update(iv).digest('hex').substring(0, 16);
+
+  const cipher = createCipheriv('aes-256-ctr', keyHash, ivHash);
+
+  return Buffer.concat([cipher.update(textToEncrypt), cipher.final()]).toString(
+    'base64',
+  );
+};
+
+export const decryptText = async (
+  textToDecrypt: string,
+  key: string,
+  iv: string,
+) => {
+  const keyHash = createHash('sha512')
+    .update(key)
+    .digest('hex')
+    .substring(0, 32);
+
+  const ivHash = createHash('sha512').update(iv).digest('hex').substring(0, 16);
+
+  const decipher = createDecipheriv('aes-256-ctr', keyHash, ivHash);
+
+  return Buffer.concat([
+    decipher.update(Buffer.from(textToDecrypt, 'base64')),
+    decipher.final(),
+  ]).toString();
+};

packages/twenty-server/src/engine/core-modules/core-engine.module.ts

@@ -10,6 +10,7 @@ import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timel
 import { TimelineCalendarEventModule } from 'src/engine/core-modules/calendar/timeline-calendar-event.module';
 import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
 import { HealthModule } from 'src/engine/core-modules/health/health.module';
+import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
 
 import { AnalyticsModule } from './analytics/analytics.module';
 import { FileModule } from './file/file.module';
@@ -30,6 +31,7 @@ import { ClientConfigModule } from './client-config/client-config.module';
     TimelineCalendarEventModule,
     UserModule,
     WorkspaceModule,
+    RemoteServerModule,
   ],
   exports: [
     AnalyticsModule,

packages/twenty-server/src/engine/metadata-modules/metadata-engine.module.ts

@@ -4,6 +4,7 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s
 import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
 import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
 import { RelationMetadataModule } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.module';
+import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
 import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
 import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
 
@@ -15,6 +16,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
     RelationMetadataModule,
     WorkspaceCacheVersionModule,
     WorkspaceMigrationModule,
+    RemoteServerModule,
   ],
   providers: [],
   exports: [
@@ -22,6 +24,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
     FieldMetadataModule,
     ObjectMetadataModule,
     RelationMetadataModule,
+    RemoteServerModule,
   ],
 })
 export class MetadataEngineModule {}

packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/create-remote-server.input.ts

@@ -0,0 +1,24 @@
+import { Field, InputType } from '@nestjs/graphql';
+
+import { IsOptional } from 'class-validator';
+import GraphQLJSON from 'graphql-type-json';
+
+import {
+  ForeignDataWrapperOptions,
+  RemoteServerType,
+  UserMappingOptions,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+
+@InputType()
+export class CreateRemoteServerInput<T extends RemoteServerType> {
+  @Field(() => String)
+  foreignDataWrapperType: T;
+
+  @IsOptional()
+  @Field(() => GraphQLJSON)
+  foreignDataWrapperOptions: ForeignDataWrapperOptions<T>;
+
+  @IsOptional()
+  @Field(() => GraphQLJSON, { nullable: true })
+  userMappingOptions?: UserMappingOptions;
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/remote-server-id.input.ts

@@ -0,0 +1,9 @@
+import { InputType, ID } from '@nestjs/graphql';
+
+import { IDField } from '@ptc-org/nestjs-query-graphql';
+
+@InputType()
+export class RemoteServerIdInput {
+  @IDField(() => ID, { description: 'The id of the record.' })
+  id!: string;
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/remote-server-type.input.ts

@@ -0,0 +1,12 @@
+import { InputType, Field } from '@nestjs/graphql';
+
+import { IsString } from 'class-validator';
+
+import { RemoteServerType } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+
+@InputType()
+export class RemoteServerTypeInput<T extends RemoteServerType> {
+  @Field(() => String)
+  @IsString()
+  foreignDataWrapperType!: T;
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/remote-server.dto.ts

@@ -0,0 +1,34 @@
+import { ObjectType, Field, HideField, ID } from '@nestjs/graphql';
+
+import { IsOptional } from 'class-validator';
+import GraphQLJSON from 'graphql-type-json';
+
+import {
+  ForeignDataWrapperOptions,
+  RemoteServerType,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+
+@ObjectType('RemoteServer')
+export class RemoteServerDTO<T extends RemoteServerType> {
+  @Field(() => ID)
+  id: string;
+
+  @Field(() => ID)
+  foreignDataWrapperId: string;
+
+  @Field(() => String)
+  foreignDataWrapperType: T;
+
+  @IsOptional()
+  @Field(() => GraphQLJSON, { nullable: true })
+  foreignDataWrapperOptions?: ForeignDataWrapperOptions<T>;
+
+  @HideField()
+  workspaceId: string;
+
+  @Field()
+  createdAt: Date;
+
+  @Field()
+  updatedAt: Date;
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts

@@ -0,0 +1,59 @@
+import { ObjectType } from '@nestjs/graphql';
+
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  Generated,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm';
+
+export enum RemoteServerType {
+  POSTGRES_FDW = 'postgres_fdw',
+}
+
+type PostgresForeignDataWrapperOptions = {
+  host: string;
+  port: number;
+  dbname: string;
+};
+
+export type ForeignDataWrapperOptions<T extends RemoteServerType> =
+  T extends RemoteServerType.POSTGRES_FDW
+    ? PostgresForeignDataWrapperOptions
+    : never;
+
+export type UserMappingOptions = {
+  username: string;
+  password: string;
+};
+
+@Entity('remoteServer')
+@ObjectType('RemoteServer')
+export class RemoteServerEntity<T extends RemoteServerType> {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column()
+  @Generated('uuid')
+  foreignDataWrapperId: string;
+
+  @Column({ nullable: true })
+  foreignDataWrapperType: T;
+
+  @Column({ nullable: true, type: 'jsonb' })
+  foreignDataWrapperOptions: ForeignDataWrapperOptions<T>;
+
+  @Column({ nullable: true, type: 'jsonb' })
+  userMappingOptions: UserMappingOptions;
+
+  @Column({ nullable: false, type: 'uuid' })
+  workspaceId: string;
+
+  @CreateDateColumn()
+  createdAt: Date;
+
+  @UpdateDateColumn()
+  updatedAt: Date;
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts

@@ -0,0 +1,22 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
+import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
+import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+import { RemoteServerResolver } from 'src/engine/metadata-modules/remote-server/remote-server.resolver';
+import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
+
+@Module({
+  imports: [
+    TypeORMModule,
+    TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
+  ],
+  providers: [
+    RemoteServerService,
+    RemoteServerResolver,
+    ForeignDataWrapperQueryFactory,
+  ],
+  exports: [RemoteServerService],
+})
+export class RemoteServerModule {}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.resolver.ts

@@ -0,0 +1,56 @@
+import { UseGuards } from '@nestjs/common';
+import { Resolver, Args, Mutation, Query } from '@nestjs/graphql';
+
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
+import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
+import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/create-remote-server.input';
+import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
+import { RemoteServerTypeInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-type.input';
+import { RemoteServerDTO } from 'src/engine/metadata-modules/remote-server/dtos/remote-server.dto';
+import { RemoteServerType } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
+
+@UseGuards(JwtAuthGuard)
+@Resolver()
+export class RemoteServerResolver {
+  constructor(
+    private readonly remoteServerService: RemoteServerService<RemoteServerType>,
+  ) {}
+
+  @Mutation(() => RemoteServerDTO)
+  async createOneRemoteServer(
+    @Args('input') input: CreateRemoteServerInput<RemoteServerType>,
+    @AuthWorkspace() { id: workspaceId }: Workspace,
+  ) {
+    return this.remoteServerService.createOneRemoteServer(input, workspaceId);
+  }
+
+  @Mutation(() => RemoteServerDTO)
+  async deleteOneRemoteServer(
+    @Args('input') { id }: RemoteServerIdInput,
+    @AuthWorkspace() { id: workspaceId }: Workspace,
+  ) {
+    return this.remoteServerService.deleteOneRemoteServer(id, workspaceId);
+  }
+
+  @Query(() => RemoteServerDTO)
+  async findOneRemoteServerById(
+    @Args('input') { id }: RemoteServerIdInput,
+    @AuthWorkspace() { id: workspaceId }: Workspace,
+  ) {
+    return this.remoteServerService.findOneByIdWithinWorkspace(id, workspaceId);
+  }
+
+  @Query(() => [RemoteServerDTO])
+  async findManyRemoteServersByType(
+    @Args('input')
+    { foreignDataWrapperType }: RemoteServerTypeInput<RemoteServerType>,
+    @AuthWorkspace() { id: workspaceId }: Workspace,
+  ) {
+    return this.remoteServerService.findManyByTypeWithinWorkspace(
+      foreignDataWrapperType,
+      workspaceId,
+    );
+  }
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts

@@ -0,0 +1,144 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { v4 } from 'uuid';
+import { Repository } from 'typeorm';
+
+import { TypeORMService } from 'src/database/typeorm/typeorm.service';
+import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/create-remote-server.input';
+import {
+  RemoteServerEntity,
+  RemoteServerType,
+} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
+import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
+import { encryptText } from 'src/engine/core-modules/auth/auth.util';
+import {
+  validateObject,
+  validateString,
+} from 'src/engine/metadata-modules/remote-server/utils/validate-remote-server-input';
+import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
+
+@Injectable()
+export class RemoteServerService<T extends RemoteServerType> {
+  constructor(
+    @InjectRepository(RemoteServerEntity, 'metadata')
+    private readonly remoteServerRepository: Repository<
+      RemoteServerEntity<RemoteServerType>
+    >,
+    private readonly typeORMService: TypeORMService,
+    private readonly environmentService: EnvironmentService,
+    private readonly foreignDataWrapperQueryFactory: ForeignDataWrapperQueryFactory,
+  ) {}
+
+  async createOneRemoteServer(
+    remoteServerInput: CreateRemoteServerInput<T>,
+    workspaceId: string,
+  ): Promise<RemoteServerEntity<RemoteServerType>> {
+    validateObject(remoteServerInput.foreignDataWrapperOptions);
+
+    if (remoteServerInput.userMappingOptions) {
+      validateObject(remoteServerInput.userMappingOptions);
+    }
+
+    const mainDatasource = this.typeORMService.getMainDataSource();
+    const foreignDataWrapperId = v4();
+
+    let remoteServerToCreate = {
+      ...remoteServerInput,
+      workspaceId,
+      foreignDataWrapperId,
+    };
+
+    if (remoteServerInput.userMappingOptions) {
+      const key = this.environmentService.get('LOGIN_TOKEN_SECRET');
+      const encryptedPassword = await encryptText(
+        remoteServerInput.userMappingOptions.password,
+        key,
+        // TODO: check if we should use a separated IV
+        key,
+      );
+
+      remoteServerToCreate = {
+        ...remoteServerToCreate,
+        userMappingOptions: {
+          ...remoteServerInput.userMappingOptions,
+          password: encryptedPassword,
+        },
+      };
+    }
+
+    const createdRemoteServer =
+      await this.remoteServerRepository.create(remoteServerToCreate);
+
+    const foreignDataWrapperQuery =
+      this.foreignDataWrapperQueryFactory.createForeignDataWrapper(
+        createdRemoteServer.foreignDataWrapperId,
+        remoteServerInput.foreignDataWrapperType,
+        remoteServerInput.foreignDataWrapperOptions,
+      );
+
+    await mainDatasource.query(foreignDataWrapperQuery);
+
+    if (remoteServerInput.userMappingOptions) {
+      const userMappingQuery =
+        this.foreignDataWrapperQueryFactory.createUserMapping(
+          createdRemoteServer.foreignDataWrapperId,
+          remoteServerInput.userMappingOptions,
+        );
+
+      await mainDatasource.query(userMappingQuery);
+    }
+
+    await this.remoteServerRepository.save(createdRemoteServer);
+
+    return createdRemoteServer;
+  }
+
+  async deleteOneRemoteServer(
+    id: string,
+    workspaceId: string,
+  ): Promise<RemoteServerEntity<RemoteServerType>> {
+    validateString(id);
+
+    const remoteServer = await this.remoteServerRepository.findOne({
+      where: {
+        id,
+        workspaceId,
+      },
+    });
+
+    if (!remoteServer) {
+      throw new NotFoundException('Object does not exist');
+    }
+
+    const mainDatasource = this.typeORMService.getMainDataSource();
+
+    await mainDatasource.query(
+      `DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`,
+    );
+    await this.remoteServerRepository.delete(id);
+
+    return remoteServer;
+  }
+
+  public async findOneByIdWithinWorkspace(id: string, workspaceId: string) {
+    return this.remoteServerRepository.findOne({
+      where: {
+        id,
+        workspaceId,
+      },
+    });
+  }
+
+  public async findManyByTypeWithinWorkspace<T extends RemoteServerType>(
+    foreignDataWrapperType: T,
+    workspaceId: string,
+  ) {
+    return this.remoteServerRepository.find({
+      where: {
+        foreignDataWrapperType,
+        workspaceId,
+      },
+    });
+  }
+}

packages/twenty-server/src/engine/metadata-modules/remote-server/utils/validate-remote-server-input.ts

@@ -0,0 +1,20 @@
+const INPUT_REGEX = /^([A-Za-z0-9\-\_]+)$/;
+
+export const validateObject = (input: object) => {
+  for (const [key, value] of Object.entries(input)) {
+    // Password are encrypted so we don't need to validate them
+    if (key === 'password') {
+      continue;
+    }
+
+    if (!INPUT_REGEX.test(value.toString())) {
+      throw new Error('Invalid remote server input');
+    }
+  }
+};
+
+export const validateString = (input: string) => {
+  if (!INPUT_REGEX.test(input)) {
+    throw new Error('Invalid remote server input');
+  }
+};