[feat] Add updateRemoteServer endpoint (#5148)

## Context
#4765 

Following investigations
([#5083](https://github.com/twentyhq/twenty/issues/5083)) we decided to
restrict updates of server from which zero tables have been synchronized
only

## How was it tested
Locally with /metadata
1. Updating a database that already has synchronized tables
<img width="1072" alt="Capture d’écran 2024-04-24 à 16 16 05"
src="https://github.com/twentyhq/twenty/assets/51697796/f9a84c34-2dcd-4f3c-b0bc-b710abae5021">

2. Updating a database that has no synchronized tables
<img width="843" alt="Capture d’écran 2024-04-24 à 16 17 28"
src="https://github.com/twentyhq/twenty/assets/51697796/f320fe03-a6bc-4724-bcd0-4e89d3ac31f5">
+ tested that the connection works well
This commit is contained in:
Marie
2024-04-26 18:12:08 +02:00
committed by GitHub
parent b15533e4b3
commit 76d4188ba8
12 changed files with 430 additions and 50 deletions

View File

@ -1,5 +1,7 @@
import { Injectable } from '@nestjs/common';
import { isDefined } from 'class-validator';
import {
ForeignDataWrapperOptions,
RemoteServerType,
@ -21,6 +23,20 @@ export class ForeignDataWrapperQueryFactory {
return `CREATE SERVER "${foreignDataWrapperId}" FOREIGN DATA WRAPPER ${name} OPTIONS (${options})`;
}
updateForeignDataWrapper({
foreignDataWrapperId,
foreignDataWrapperOptions,
}: {
foreignDataWrapperId: string;
foreignDataWrapperOptions: Partial<
ForeignDataWrapperOptions<RemoteServerType>
>;
}) {
const options = this.buildUpdateOptions(foreignDataWrapperOptions);
return `ALTER SERVER "${foreignDataWrapperId}" OPTIONS (${options})`;
}
createUserMapping(
foreignDataWrapperId: string,
userMappingOptions: UserMappingOptions,
@ -29,6 +45,16 @@ export class ForeignDataWrapperQueryFactory {
return `CREATE USER MAPPING IF NOT EXISTS FOR CURRENT_USER SERVER "${foreignDataWrapperId}" OPTIONS (user '${userMappingOptions.username}', password '${userMappingOptions.password}')`;
}
updateUserMapping(
foreignDataWrapperId: string,
userMappingOptions: Partial<UserMappingOptions>,
) {
const options = this.buildUpdateUserMappingOptions(userMappingOptions);
// CURRENT_USER works for now since we are using only one user. But if we switch to a user per workspace, we need to change this.
return `ALTER USER MAPPING FOR CURRENT_USER SERVER "${foreignDataWrapperId}" OPTIONS (${options})`;
}
private buildNameAndOptionsFromType(
type: RemoteServerType,
options: ForeignDataWrapperOptions<RemoteServerType>,
@ -41,6 +67,36 @@ export class ForeignDataWrapperQueryFactory {
}
}
private buildUpdateOptions(
options: Partial<ForeignDataWrapperOptions<RemoteServerType>>,
) {
const rawQuerySetStatements: string[] = [];
Object.entries(options).forEach(([key, value]) => {
if (isDefined(value)) {
rawQuerySetStatements.push(`SET ${key} '${value}'`);
}
});
return rawQuerySetStatements.join(', ');
}
private buildUpdateUserMappingOptions(
userMappingOptions?: Partial<UserMappingOptions>,
) {
const setStatements: string[] = [];
if (isDefined(userMappingOptions?.username)) {
setStatements.push(`SET user '${userMappingOptions?.username}'`);
}
if (isDefined(userMappingOptions?.password)) {
setStatements.push(`SET password '${userMappingOptions?.password}'`);
}
return setStatements.join(', ');
}
private buildPostgresFDWQueryOptions(
foreignDataWrapperOptions: ForeignDataWrapperOptions<RemoteServerType>,
) {

View File

@ -39,8 +39,8 @@ import {
import { DeleteOneFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/delete-field.input';
import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { validateString } from 'src/engine/metadata-modules/remote-server/utils/validate-remote-server-input';
import { InvalidStringException } from 'src/engine/metadata-modules/errors/InvalidStringException';
import { validateMetadataName } from 'src/engine/metadata-modules/utils/validate-metadata-name.utils';
import {
FieldMetadataEntity,
@ -543,7 +543,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
>(fieldMetadataInput: T): T {
if (fieldMetadataInput.name) {
try {
validateString(fieldMetadataInput.name);
validateMetadataName(fieldMetadataInput.name);
} catch (error) {
if (error instanceof InvalidStringException) {
throw new BadRequestException(

View File

@ -8,17 +8,17 @@ import {
RemoteServerType,
UserMappingOptions,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { UserMappingOptionsInput } from 'src/engine/metadata-modules/remote-server/utils/user-mapping-options-input.utils';
@InputType()
export class CreateRemoteServerInput<T extends RemoteServerType> {
@Field(() => String)
foreignDataWrapperType: T;
@IsOptional()
@Field(() => GraphQLJSON)
foreignDataWrapperOptions: ForeignDataWrapperOptions<T>;
@IsOptional()
@Field(() => GraphQLJSON, { nullable: true })
@Field(() => UserMappingOptionsInput, { nullable: true })
userMappingOptions?: UserMappingOptions;
}

View File

@ -0,0 +1,25 @@
import { Field, InputType } from '@nestjs/graphql';
import { IsOptional } from 'class-validator';
import GraphQLJSON from 'graphql-type-json';
import {
ForeignDataWrapperOptions,
RemoteServerType,
UserMappingOptions,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { UserMappingOptionsInput } from 'src/engine/metadata-modules/remote-server/utils/user-mapping-options-input.utils';
@InputType()
export class UpdateRemoteServerInput<T extends RemoteServerType> {
@Field(() => String)
id: string;
@IsOptional()
@Field(() => GraphQLJSON, { nullable: true })
foreignDataWrapperOptions?: Partial<ForeignDataWrapperOptions<T>>;
@IsOptional()
@Field(() => UserMappingOptionsInput, { nullable: true })
userMappingOptions?: Partial<UserMappingOptions>;
}

View File

@ -6,11 +6,13 @@ import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/re
import { RemoteServerResolver } from 'src/engine/metadata-modules/remote-server/remote-server.resolver';
import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
import { RemoteTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
RemoteTableModule,
WorkspaceDataSourceModule,
],
providers: [
RemoteServerService,

View File

@ -8,6 +8,7 @@ import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-serv
import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
import { RemoteServerTypeInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-type.input';
import { RemoteServerDTO } from 'src/engine/metadata-modules/remote-server/dtos/remote-server.dto';
import { UpdateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/update-remote-server.input';
import { RemoteServerType } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
@ -26,6 +27,14 @@ export class RemoteServerResolver {
return this.remoteServerService.createOneRemoteServer(input, workspaceId);
}
@Mutation(() => RemoteServerDTO)
async updateOneRemoteServer(
@Args('input') input: UpdateRemoteServerInput<RemoteServerType>,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteServerService.updateOneRemoteServer(input, workspaceId);
}
@Mutation(() => RemoteServerDTO)
async deleteOneRemoteServer(
@Args('input') { id }: RemoteServerIdInput,

View File

@ -1,4 +1,8 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import {
ForbiddenException,
Injectable,
NotFoundException,
} from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { v4 } from 'uuid';
@ -12,11 +16,14 @@ import {
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { encryptText } from 'src/engine/core-modules/auth/auth.util';
import {
validateObject,
validateString,
} from 'src/engine/metadata-modules/remote-server/utils/validate-remote-server-input';
validateObjectAgainstInjections,
validateStringAgainstInjections,
} from 'src/engine/metadata-modules/remote-server/utils/validate-remote-server-input.utils';
import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
import { UpdateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/update-remote-server.input';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { updateRemoteServerRawQuery } from 'src/engine/metadata-modules/remote-server/utils/build-update-remote-server-raw-query.utils';
@Injectable()
export class RemoteServerService<T extends RemoteServerType> {
@ -30,17 +37,14 @@ export class RemoteServerService<T extends RemoteServerType> {
private readonly environmentService: EnvironmentService,
private readonly foreignDataWrapperQueryFactory: ForeignDataWrapperQueryFactory,
private readonly remoteTableService: RemoteTableService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
async createOneRemoteServer(
remoteServerInput: CreateRemoteServerInput<T>,
workspaceId: string,
): Promise<RemoteServerEntity<RemoteServerType>> {
validateObject(remoteServerInput.foreignDataWrapperOptions);
if (remoteServerInput.userMappingOptions) {
validateObject(remoteServerInput.userMappingOptions);
}
this.validateRemoteServerInputAgainstInjections(remoteServerInput);
const foreignDataWrapperId = v4();
@ -51,24 +55,20 @@ export class RemoteServerService<T extends RemoteServerType> {
};
if (remoteServerInput.userMappingOptions) {
const key = this.environmentService.get('LOGIN_TOKEN_SECRET');
const encryptedPassword = await encryptText(
remoteServerInput.userMappingOptions.password,
key,
);
remoteServerToCreate = {
...remoteServerToCreate,
userMappingOptions: {
...remoteServerInput.userMappingOptions,
password: encryptedPassword,
password: this.encryptPassword(
remoteServerInput.userMappingOptions.password,
),
},
};
}
return this.metadataDataSource.transaction(
async (entityManager: EntityManager) => {
const createdRemoteServer = await entityManager.create(
const createdRemoteServer = entityManager.create(
RemoteServerEntity,
remoteServerToCreate,
);
@ -99,11 +99,104 @@ export class RemoteServerService<T extends RemoteServerType> {
);
}
async updateOneRemoteServer(
remoteServerInput: UpdateRemoteServerInput<T>,
workspaceId: string,
): Promise<RemoteServerEntity<RemoteServerType>> {
this.validateRemoteServerInputAgainstInjections(remoteServerInput);
const remoteServer = await this.findOneByIdWithinWorkspace(
remoteServerInput.id,
workspaceId,
);
if (!remoteServer) {
throw new NotFoundException('Remote server does not exist');
}
const currentRemoteTablesForServer =
await this.remoteTableService.findCurrentRemoteTablesByServerId({
remoteServerId: remoteServer.id,
workspaceId,
});
if (currentRemoteTablesForServer.length > 0) {
throw new ForbiddenException(
'Cannot update remote server with synchronized tables',
);
}
const foreignDataWrapperId = remoteServer.foreignDataWrapperId;
let partialRemoteServerWithUpdates = {
...remoteServerInput,
workspaceId,
foreignDataWrapperId,
};
if (partialRemoteServerWithUpdates?.userMappingOptions?.password) {
partialRemoteServerWithUpdates = {
...partialRemoteServerWithUpdates,
userMappingOptions: {
...partialRemoteServerWithUpdates.userMappingOptions,
password: this.encryptPassword(
partialRemoteServerWithUpdates.userMappingOptions.password,
),
},
};
}
return this.metadataDataSource.transaction(
async (entityManager: EntityManager) => {
const updatedRemoteServer = await this.updateRemoteServer(
partialRemoteServerWithUpdates,
);
if (partialRemoteServerWithUpdates.foreignDataWrapperOptions) {
const foreignDataWrapperQuery =
this.foreignDataWrapperQueryFactory.updateForeignDataWrapper({
foreignDataWrapperId,
foreignDataWrapperOptions:
partialRemoteServerWithUpdates.foreignDataWrapperOptions,
});
await entityManager.query(foreignDataWrapperQuery);
}
if (partialRemoteServerWithUpdates.userMappingOptions) {
const userMappingQuery =
this.foreignDataWrapperQueryFactory.updateUserMapping(
foreignDataWrapperId,
partialRemoteServerWithUpdates.userMappingOptions,
);
await entityManager.query(userMappingQuery);
}
return updatedRemoteServer;
},
);
}
private validateRemoteServerInputAgainstInjections(
remoteServerInput: CreateRemoteServerInput<T> | UpdateRemoteServerInput<T>,
) {
if (remoteServerInput.foreignDataWrapperOptions) {
validateObjectAgainstInjections(
remoteServerInput.foreignDataWrapperOptions,
);
}
if (remoteServerInput.userMappingOptions) {
validateObjectAgainstInjections(remoteServerInput.userMappingOptions);
}
}
async deleteOneRemoteServer(
id: string,
workspaceId: string,
): Promise<RemoteServerEntity<RemoteServerType>> {
validateString(id);
validateStringAgainstInjections(id);
const remoteServer = await this.remoteServerRepository.findOne({
where: {
@ -150,4 +243,26 @@ export class RemoteServerService<T extends RemoteServerType> {
},
});
}
private encryptPassword(password: string) {
const key = this.environmentService.get('LOGIN_TOKEN_SECRET');
return encryptText(password, key);
}
private async updateRemoteServer(
remoteServerToUpdate: DeepPartial<RemoteServerEntity<RemoteServerType>> &
Pick<RemoteServerEntity<RemoteServerType>, 'workspaceId' | 'id'>,
): Promise<RemoteServerEntity<RemoteServerType>> {
const [parameters, rawQuery] =
updateRemoteServerRawQuery(remoteServerToUpdate);
const updateResult = await this.workspaceDataSourceService.executeRawQuery(
rawQuery,
parameters,
remoteServerToUpdate.workspaceId,
);
return updateResult[0][0];
}
}

View File

@ -74,14 +74,14 @@ export class RemoteTableService {
throw new NotFoundException('Remote server does not exist');
}
const currentRemoteTableDistantNames = (
await this.remoteTableRepository.find({
where: {
remoteServerId: id,
workspaceId,
},
})
).map((remoteTable) => remoteTable.distantTableName);
const currentRemoteTables = await this.findCurrentRemoteTablesByServerId({
remoteServerId: id,
workspaceId,
});
const currentRemoteTableDistantNames = currentRemoteTables.map(
(remoteTable) => remoteTable.distantTableName,
);
const tablesInRemoteSchema =
await this.fetchTablesFromRemoteSchema(remoteServer);
@ -95,6 +95,21 @@ export class RemoteTableService {
}));
}
public async findCurrentRemoteTablesByServerId({
remoteServerId,
workspaceId,
}: {
remoteServerId: string;
workspaceId: string;
}) {
return this.remoteTableRepository.find({
where: {
remoteServerId,
workspaceId,
},
});
}
public async syncRemoteTable(input: RemoteTableInput, workspaceId: string) {
if (!input.schema) {
throw new BadRequestException(

View File

@ -0,0 +1,138 @@
import { isDefined } from 'class-validator';
import {
RemoteServerEntity,
RemoteServerType,
UserMappingOptions,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
export type DeepPartial<T> = {
[P in keyof T]?: DeepPartial<T[P]>;
};
const buildUserMappingOptionsQuery = (
parameters: any[],
parametersPositions: object,
userMappingOptions: DeepPartial<UserMappingOptions>,
): string | null => {
const shouldUpdateUserMappingOptionsPassword = isDefined(
userMappingOptions?.password,
);
if (shouldUpdateUserMappingOptionsPassword) {
parameters.push(userMappingOptions?.password);
parametersPositions['password'] = parameters.length;
}
const shouldUpdateUserMappingOptionsUsername = isDefined(
userMappingOptions?.username,
);
if (shouldUpdateUserMappingOptionsUsername) {
parameters.push(userMappingOptions?.username);
parametersPositions['username'] = parameters.length;
}
if (
shouldUpdateUserMappingOptionsPassword ||
shouldUpdateUserMappingOptionsUsername
) {
return `"userMappingOptions" = jsonb_set(${
shouldUpdateUserMappingOptionsPassword &&
shouldUpdateUserMappingOptionsUsername
? `jsonb_set(
"userMappingOptions",
'{username}',
to_jsonb($${parametersPositions['username']}::text)
),
'{password}',
to_jsonb($${parametersPositions['password']}::text)
`
: shouldUpdateUserMappingOptionsPassword
? `"userMappingOptions",
'{password}',
to_jsonb($${parametersPositions['password']}::text)
`
: `"userMappingOptions",
'{username}',
to_jsonb($${parametersPositions['username']}::text)
`
})`;
}
return null;
};
// TO DO This only works for postgres_fdw type for now, lets make it more generic when we have a different type
export const updateRemoteServerRawQuery = (
remoteServerToUpdate: DeepPartial<RemoteServerEntity<RemoteServerType>> &
Pick<RemoteServerEntity<RemoteServerType>, 'workspaceId' | 'id'>,
): [any[], string] => {
const parameters: any[] = [remoteServerToUpdate.id];
const parametersPositions = {};
const options: string[] = [];
if (remoteServerToUpdate.userMappingOptions) {
const userMappingOptionsQuery = buildUserMappingOptionsQuery(
parameters,
parametersPositions,
remoteServerToUpdate.userMappingOptions,
);
if (userMappingOptionsQuery) options.push(userMappingOptionsQuery);
}
const shouldUpdateFdwDbname = isDefined(
remoteServerToUpdate.foreignDataWrapperOptions?.dbname,
);
if (shouldUpdateFdwDbname) {
parameters.push(remoteServerToUpdate?.foreignDataWrapperOptions?.dbname);
parametersPositions['dbname'] = parameters.length;
}
const shouldUpdateFdwHost = isDefined(
remoteServerToUpdate.foreignDataWrapperOptions?.host,
);
if (shouldUpdateFdwHost) {
parameters.push(remoteServerToUpdate?.foreignDataWrapperOptions?.host);
parametersPositions['host'] = parameters.length;
}
const shouldUpdateFdwPort = isDefined(
remoteServerToUpdate.foreignDataWrapperOptions?.port,
);
if (shouldUpdateFdwPort) {
parameters.push(remoteServerToUpdate?.foreignDataWrapperOptions?.port);
parametersPositions['port'] = parameters.length;
}
if (shouldUpdateFdwDbname || shouldUpdateFdwHost || shouldUpdateFdwPort) {
const fwdOptionsQuery = `"foreignDataWrapperOptions" = jsonb_set(${
shouldUpdateFdwDbname && shouldUpdateFdwHost && shouldUpdateFdwPort
? `jsonb_set(jsonb_set("foreignDataWrapperOptions", '{dbname}', to_jsonb($${parametersPositions['dbname']}::text)), '{host}', to_jsonb($${parametersPositions['host']}::text)), '{port}', to_jsonb($${parametersPositions['port']}::text)`
: shouldUpdateFdwDbname && shouldUpdateFdwHost
? `jsonb_set("foreignDataWrapperOptions", '{dbname}', to_jsonb($${parametersPositions['dbname']}::text)), '{host}', to_jsonb($${parametersPositions['host']}::text)`
: shouldUpdateFdwDbname && shouldUpdateFdwPort
? `jsonb_set("foreignDataWrapperOptions", '{dbname}', to_jsonb($${parametersPositions['dbname']}::text)), '{port}', to_jsonb($${parametersPositions['port']}::text)`
: shouldUpdateFdwHost && shouldUpdateFdwPort
? `jsonb_set("foreignDataWrapperOptions", '{host}', to_jsonb($${parametersPositions['host']}::text)), '{port}', to_jsonb($${parametersPositions['port']}::text)`
: shouldUpdateFdwDbname
? `"foreignDataWrapperOptions", '{dbname}', to_jsonb($${parametersPositions['dbname']}::text)`
: shouldUpdateFdwHost
? `"foreignDataWrapperOptions", '{host}', to_jsonb($${parametersPositions['host']}::text)`
: `"foreignDataWrapperOptions", '{port}', to_jsonb($${parametersPositions['port']}::text)`
})`;
options.push(fwdOptionsQuery);
}
const rawQuery = `UPDATE metadata."remoteServer" SET ${options.join(
', ',
)} WHERE "id"= $1 RETURNING *`;
return [parameters, rawQuery];
};

View File

@ -0,0 +1,14 @@
import { InputType, Field } from '@nestjs/graphql';
import { IsOptional } from 'class-validator';
@InputType()
export class UserMappingOptionsInput {
@IsOptional()
@Field(() => String, { nullable: true })
username: string;
@IsOptional()
@Field(() => String, { nullable: true })
password: string;
}

View File

@ -1,20 +0,0 @@
const INPUT_REGEX = /^([A-Za-z0-9\-_.@]+)$/;
export const validateObject = (input: object) => {
for (const [key, value] of Object.entries(input)) {
// Password are encrypted so we don't need to validate them
if (key === 'password') {
continue;
}
if (!INPUT_REGEX.test(value.toString())) {
throw new Error('Invalid remote server input');
}
}
};
export const validateString = (input: string) => {
if (!INPUT_REGEX.test(input)) {
throw new Error('Invalid remote server input');
}
};

View File

@ -0,0 +1,26 @@
import { BadRequestException } from '@nestjs/common';
import { isDefined } from 'class-validator';
const INPUT_REGEX = /^([A-Za-z0-9\-_.@]+)$/;
export const validateObjectAgainstInjections = (input: object) => {
for (const [key, value] of Object.entries(input)) {
// Password are encrypted so we don't need to validate them
if (key === 'password') {
continue;
}
if (!isDefined(value)) {
continue;
}
validateStringAgainstInjections(value.toString());
}
};
export const validateStringAgainstInjections = (input: string) => {
if (!INPUT_REGEX.test(input)) {
throw new BadRequestException('Invalid remote server input');
}
};