admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add endpoints to create and delete remote server (#4606)

Browse Source 
* Build remote server

* Add getters

* Migrate to json inputs

* Use extendable type

* Use regex validation

* Remove acronymes

---------

Co-authored-by: Thomas Trompette <thomast@twenty.com>
This commit is contained in:
Thomas Trompette
2024-03-25 15:21:23 +01:00
committed by GitHub
parent e2af5b8628
commit 9e70f5b650
17 changed files with 496 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/twenty-server/.env.test
View File

@ -9,6 +9,7 @@ FRONT_BASE_URL=http://localhost:3001
ACCESS_TOKEN_SECRET=secret_jwt
LOGIN_TOKEN_SECRET=secret_login_tokens
REFRESH_TOKEN_SECRET=secret_refresh_token
FILE_TOKEN_SECRET=replace_me_with_a_random_string_refresh
# ———————— Optional ————————
# DEBUG_MODE=false

15
packages/twenty-server/src/database/typeorm/metadata/migrations/1711374137222-addRemoteServerTable.ts Normal file
View File

@ -0,0 +1,15 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddRemoteServerTable1711374137222 implements MigrationInterface {
name = 'AddRemoteServerTable1711374137222';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "metadata"."remoteServer" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "foreignDataWrapperId" uuid NOT NULL DEFAULT uuid_generate_v4(), "foreignDataWrapperType" character varying, "foreignDataWrapperOptions" jsonb, "userMappingOptions" jsonb, "workspaceId" uuid NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_8e5d208498fa2c9710bb934023a" PRIMARY KEY ("id"))`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "metadata"."remoteServer"`);
}
}

6
packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts
View File

@ -5,12 +5,12 @@ import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
import { CreateContextFactory } from 'src/engine/api/graphql/graphql-config/factories/create-context.factory';
import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module';
import { metadataModuleFactory } from 'src/engine/api/graphql/metadata.module-factory';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service';
import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module';
import { CreateContextFactory } from 'src/engine/api/graphql/graphql-config/factories/create-context.factory';
import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
@Module({
imports: [

3
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/factories.ts
View File

@ -1,3 +1,5 @@
import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
import { ArgsAliasFactory } from './args-alias.factory';
import { ArgsStringFactory } from './args-string.factory';
import { RelationFieldAliasFactory } from './relation-field-alias.factory';
@ -28,4 +30,5 @@ export const workspaceQueryBuilderFactories = [
UpdateOneQueryFactory,
UpdateManyQueryFactory,
DeleteManyQueryFactory,
ForeignDataWrapperQueryFactory,
];

48
packages/twenty-server/src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory.ts Normal file
View File

@ -0,0 +1,48 @@
import { Injectable } from '@nestjs/common';
import {
ForeignDataWrapperOptions,
RemoteServerType,
UserMappingOptions,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
@Injectable()
export class ForeignDataWrapperQueryFactory {
createForeignDataWrapper(
foreignDataWrapperId: string,
foreignDataWrapperType: RemoteServerType,
foreignDataWrapperOptions: ForeignDataWrapperOptions<RemoteServerType>,
) {
const [name, options] = this.buildNameAndOptionsFromType(
foreignDataWrapperType,
foreignDataWrapperOptions,
);
return `CREATE SERVER "${foreignDataWrapperId}" FOREIGN DATA WRAPPER ${name} OPTIONS (${options})`;
}
createUserMapping(
foreignDataWrapperId: string,
userMappingOptions: UserMappingOptions,
) {
return `CREATE USER MAPPING IF NOT EXISTS FOR ${userMappingOptions.username} SERVER "${foreignDataWrapperId}" OPTIONS (user '${userMappingOptions.username}', password '${userMappingOptions.password}')`;
}
private buildNameAndOptionsFromType(
type: RemoteServerType,
options: ForeignDataWrapperOptions<RemoteServerType>,
) {
switch (type) {
case RemoteServerType.POSTGRES_FDW:
return ['postgres_fdw', this.buildPostgresFDWQueryOptions(options)];
default:
throw new Error('Foreign data wrapper type not supported');
}
}
private buildPostgresFDWQueryOptions(
foreignDataWrapperOptions: ForeignDataWrapperOptions<RemoteServerType>,
) {
return `dbname '${foreignDataWrapperOptions.dbname}', host '${foreignDataWrapperOptions.host}', port '${foreignDataWrapperOptions.port}'`;
}
}

41
packages/twenty-server/src/engine/core-modules/auth/auth.util.ts
View File

@ -1,3 +1,5 @@
import { createCipheriv, createDecipheriv, createHash } from 'crypto';
import * as bcrypt from 'bcrypt';
export const PASSWORD_REGEX = /^.{8,}$/;
@ -13,3 +15,42 @@ export const hashPassword = async (password: string) => {
export const compareHash = async (password: string, passwordHash: string) => {
return bcrypt.compare(password, passwordHash);
};
export const encryptText = async (
textToEncrypt: string,
key: string,
iv: string,
): Promise<string> => {
const keyHash = createHash('sha512')
.update(key)
.digest('hex')
.substring(0, 32);
const ivHash = createHash('sha512').update(iv).digest('hex').substring(0, 16);
const cipher = createCipheriv('aes-256-ctr', keyHash, ivHash);
return Buffer.concat([cipher.update(textToEncrypt), cipher.final()]).toString(
'base64',
);
};
export const decryptText = async (
textToDecrypt: string,
key: string,
iv: string,
) => {
const keyHash = createHash('sha512')
.update(key)
.digest('hex')
.substring(0, 32);
const ivHash = createHash('sha512').update(iv).digest('hex').substring(0, 16);
const decipher = createDecipheriv('aes-256-ctr', keyHash, ivHash);
return Buffer.concat([
decipher.update(Buffer.from(textToDecrypt, 'base64')),
decipher.final(),
]).toString();
};

2
packages/twenty-server/src/engine/core-modules/core-engine.module.ts
View File

@ -10,6 +10,7 @@ import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timel
import { TimelineCalendarEventModule } from 'src/engine/core-modules/calendar/timeline-calendar-event.module';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { HealthModule } from 'src/engine/core-modules/health/health.module';
import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
import { AnalyticsModule } from './analytics/analytics.module';
import { FileModule } from './file/file.module';
@ -30,6 +31,7 @@ import { ClientConfigModule } from './client-config/client-config.module';
TimelineCalendarEventModule,
UserModule,
WorkspaceModule,
RemoteServerModule,
],
exports: [
AnalyticsModule,

3
packages/twenty-server/src/engine/metadata-modules/metadata-engine.module.ts
View File

@ -4,6 +4,7 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { RelationMetadataModule } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.module';
import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
@ -15,6 +16,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
RelationMetadataModule,
WorkspaceCacheVersionModule,
WorkspaceMigrationModule,
RemoteServerModule,
],
providers: [],
exports: [
@ -22,6 +24,7 @@ import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-
FieldMetadataModule,
ObjectMetadataModule,
RelationMetadataModule,
RemoteServerModule,
],
})
export class MetadataEngineModule {}

24
packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/create-remote-server.input.ts Normal file
View File

@ -0,0 +1,24 @@
import { Field, InputType } from '@nestjs/graphql';
import { IsOptional } from 'class-validator';
import GraphQLJSON from 'graphql-type-json';
import {
ForeignDataWrapperOptions,
RemoteServerType,
UserMappingOptions,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
@InputType()
export class CreateRemoteServerInput<T extends RemoteServerType> {
@Field(() => String)
foreignDataWrapperType: T;
@IsOptional()
@Field(() => GraphQLJSON)
foreignDataWrapperOptions: ForeignDataWrapperOptions<T>;
@IsOptional()
@Field(() => GraphQLJSON, { nullable: true })
userMappingOptions?: UserMappingOptions;
}

9
packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/remote-server-id.input.ts Normal file
View File

@ -0,0 +1,9 @@
import { InputType, ID } from '@nestjs/graphql';
import { IDField } from '@ptc-org/nestjs-query-graphql';
@InputType()
export class RemoteServerIdInput {
@IDField(() => ID, { description: 'The id of the record.' })
id!: string;
}

12
packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/remote-server-type.input.ts Normal file
View File

@ -0,0 +1,12 @@
import { InputType, Field } from '@nestjs/graphql';
import { IsString } from 'class-validator';
import { RemoteServerType } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
@InputType()
export class RemoteServerTypeInput<T extends RemoteServerType> {
@Field(() => String)
@IsString()
foreignDataWrapperType!: T;
}

34
packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/remote-server.dto.ts Normal file
View File

@ -0,0 +1,34 @@
import { ObjectType, Field, HideField, ID } from '@nestjs/graphql';
import { IsOptional } from 'class-validator';
import GraphQLJSON from 'graphql-type-json';
import {
ForeignDataWrapperOptions,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
@ObjectType('RemoteServer')
export class RemoteServerDTO<T extends RemoteServerType> {
@Field(() => ID)
id: string;
@Field(() => ID)
foreignDataWrapperId: string;
@Field(() => String)
foreignDataWrapperType: T;
@IsOptional()
@Field(() => GraphQLJSON, { nullable: true })
foreignDataWrapperOptions?: ForeignDataWrapperOptions<T>;
@HideField()
workspaceId: string;
@Field()
createdAt: Date;
@Field()
updatedAt: Date;
}

59
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts Normal file
View File

@ -0,0 +1,59 @@
import { ObjectType } from '@nestjs/graphql';
import {
Column,
CreateDateColumn,
Entity,
Generated,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
export enum RemoteServerType {
POSTGRES_FDW = 'postgres_fdw',
}
type PostgresForeignDataWrapperOptions = {
host: string;
port: number;
dbname: string;
};
export type ForeignDataWrapperOptions<T extends RemoteServerType> =
T extends RemoteServerType.POSTGRES_FDW
? PostgresForeignDataWrapperOptions
: never;
export type UserMappingOptions = {
username: string;
password: string;
};
@Entity('remoteServer')
@ObjectType('RemoteServer')
export class RemoteServerEntity<T extends RemoteServerType> {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column()
@Generated('uuid')
foreignDataWrapperId: string;
@Column({ nullable: true })
foreignDataWrapperType: T;
@Column({ nullable: true, type: 'jsonb' })
foreignDataWrapperOptions: ForeignDataWrapperOptions<T>;
@Column({ nullable: true, type: 'jsonb' })
userMappingOptions: UserMappingOptions;
@Column({ nullable: false, type: 'uuid' })
workspaceId: string;
@CreateDateColumn()
createdAt: Date;
@UpdateDateColumn()
updatedAt: Date;
}

22
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts Normal file
View File

@ -0,0 +1,22 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteServerResolver } from 'src/engine/metadata-modules/remote-server/remote-server.resolver';
import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
@Module({
imports: [
TypeORMModule,
TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
],
providers: [
RemoteServerService,
RemoteServerResolver,
ForeignDataWrapperQueryFactory,
],
exports: [RemoteServerService],
})
export class RemoteServerModule {}

56
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.resolver.ts Normal file
View File

@ -0,0 +1,56 @@
import { UseGuards } from '@nestjs/common';
import { Resolver, Args, Mutation, Query } from '@nestjs/graphql';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/create-remote-server.input';
import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
import { RemoteServerTypeInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-type.input';
import { RemoteServerDTO } from 'src/engine/metadata-modules/remote-server/dtos/remote-server.dto';
import { RemoteServerType } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
@UseGuards(JwtAuthGuard)
@Resolver()
export class RemoteServerResolver {
constructor(
private readonly remoteServerService: RemoteServerService<RemoteServerType>,
) {}
@Mutation(() => RemoteServerDTO)
async createOneRemoteServer(
@Args('input') input: CreateRemoteServerInput<RemoteServerType>,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteServerService.createOneRemoteServer(input, workspaceId);
}
@Mutation(() => RemoteServerDTO)
async deleteOneRemoteServer(
@Args('input') { id }: RemoteServerIdInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteServerService.deleteOneRemoteServer(id, workspaceId);
}
@Query(() => RemoteServerDTO)
async findOneRemoteServerById(
@Args('input') { id }: RemoteServerIdInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteServerService.findOneByIdWithinWorkspace(id, workspaceId);
}
@Query(() => [RemoteServerDTO])
async findManyRemoteServersByType(
@Args('input')
{ foreignDataWrapperType }: RemoteServerTypeInput<RemoteServerType>,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteServerService.findManyByTypeWithinWorkspace(
foreignDataWrapperType,
workspaceId,
);
}
}

144
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts Normal file
View File

@ -0,0 +1,144 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { v4 } from 'uuid';
import { Repository } from 'typeorm';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/create-remote-server.input';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { encryptText } from 'src/engine/core-modules/auth/auth.util';
import {
validateObject,
validateString,
} from 'src/engine/metadata-modules/remote-server/utils/validate-remote-server-input';
import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
@Injectable()
export class RemoteServerService<T extends RemoteServerType> {
constructor(
@InjectRepository(RemoteServerEntity, 'metadata')
private readonly remoteServerRepository: Repository<
RemoteServerEntity<RemoteServerType>
>,
private readonly typeORMService: TypeORMService,
private readonly environmentService: EnvironmentService,
private readonly foreignDataWrapperQueryFactory: ForeignDataWrapperQueryFactory,
) {}
async createOneRemoteServer(
remoteServerInput: CreateRemoteServerInput<T>,
workspaceId: string,
): Promise<RemoteServerEntity<RemoteServerType>> {
validateObject(remoteServerInput.foreignDataWrapperOptions);
if (remoteServerInput.userMappingOptions) {
validateObject(remoteServerInput.userMappingOptions);
}
const mainDatasource = this.typeORMService.getMainDataSource();
const foreignDataWrapperId = v4();
let remoteServerToCreate = {
...remoteServerInput,
workspaceId,
foreignDataWrapperId,
};
if (remoteServerInput.userMappingOptions) {
const key = this.environmentService.get('LOGIN_TOKEN_SECRET');
const encryptedPassword = await encryptText(
remoteServerInput.userMappingOptions.password,
key,
// TODO: check if we should use a separated IV
key,
);
remoteServerToCreate = {
...remoteServerToCreate,
userMappingOptions: {
...remoteServerInput.userMappingOptions,
password: encryptedPassword,
},
};
}
const createdRemoteServer =
await this.remoteServerRepository.create(remoteServerToCreate);
const foreignDataWrapperQuery =
this.foreignDataWrapperQueryFactory.createForeignDataWrapper(
createdRemoteServer.foreignDataWrapperId,
remoteServerInput.foreignDataWrapperType,
remoteServerInput.foreignDataWrapperOptions,
);
await mainDatasource.query(foreignDataWrapperQuery);
if (remoteServerInput.userMappingOptions) {
const userMappingQuery =
this.foreignDataWrapperQueryFactory.createUserMapping(
createdRemoteServer.foreignDataWrapperId,
remoteServerInput.userMappingOptions,
);
await mainDatasource.query(userMappingQuery);
}
await this.remoteServerRepository.save(createdRemoteServer);
return createdRemoteServer;
}
async deleteOneRemoteServer(
id: string,
workspaceId: string,
): Promise<RemoteServerEntity<RemoteServerType>> {
validateString(id);
const remoteServer = await this.remoteServerRepository.findOne({
where: {
id,
workspaceId,
},
});
if (!remoteServer) {
throw new NotFoundException('Object does not exist');
}
const mainDatasource = this.typeORMService.getMainDataSource();
await mainDatasource.query(
`DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`,
);
await this.remoteServerRepository.delete(id);
return remoteServer;
}
public async findOneByIdWithinWorkspace(id: string, workspaceId: string) {
return this.remoteServerRepository.findOne({
where: {
id,
workspaceId,
},
});
}
public async findManyByTypeWithinWorkspace<T extends RemoteServerType>(
foreignDataWrapperType: T,
workspaceId: string,
) {
return this.remoteServerRepository.find({
where: {
foreignDataWrapperType,
workspaceId,
},
});
}
}

20
packages/twenty-server/src/engine/metadata-modules/remote-server/utils/validate-remote-server-input.ts Normal file
View File

@ -0,0 +1,20 @@
const INPUT_REGEX = /^([A-Za-z0-9\-\_]+)$/;
export const validateObject = (input: object) => {
for (const [key, value] of Object.entries(input)) {
// Password are encrypted so we don't need to validate them
if (key === 'password') {
continue;
}
if (!INPUT_REGEX.test(value.toString())) {
throw new Error('Invalid remote server input');
}
}
};
export const validateString = (input: string) => {
if (!INPUT_REGEX.test(input)) {
throw new Error('Invalid remote server input');
}
};