admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fetch available remote tables (#4665)

Browse Source 
* Build remote table module

* Use transactions

* Export url builder in util

---------

Co-authored-by: Thomas Trompette <thomast@twenty.com>
This commit is contained in:
Thomas Trompette
2024-03-26 15:50:41 +01:00
committed by GitHub
parent fefa37b300
commit 279d99487c
9 changed files with 254 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
packages/twenty-server/src/engine/core-modules/auth/auth.util.ts
View File

@ -16,11 +16,11 @@ export const compareHash = async (password: string, passwordHash: string) => {
return bcrypt.compare(password, passwordHash);
};
export const encryptText = async (
export const encryptText = (
textToEncrypt: string,
key: string,
iv: string,
): Promise<string> => {
): string => {
const keyHash = createHash('sha512')
.update(key)
.digest('hex')
@ -35,11 +35,11 @@ export const encryptText = async (
);
};
export const decryptText = async (
export const decryptText = (
textToDecrypt: string,
key: string,
iv: string,
) => {
): string => {
const keyHash = createHash('sha512')
.update(key)
.digest('hex')

2
packages/twenty-server/src/engine/core-modules/core-engine.module.ts
View File

@ -10,7 +10,6 @@ import { TimelineMessagingModule } from 'src/engine/core-modules/messaging/timel
import { TimelineCalendarEventModule } from 'src/engine/core-modules/calendar/timeline-calendar-event.module';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { HealthModule } from 'src/engine/core-modules/health/health.module';
import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
import { AnalyticsModule } from './analytics/analytics.module';
import { FileModule } from './file/file.module';
@ -31,7 +30,6 @@ import { ClientConfigModule } from './client-config/client-config.module';
TimelineCalendarEventModule,
UserModule,
WorkspaceModule,
RemoteServerModule,
],
exports: [
AnalyticsModule,

5
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts
View File

@ -1,22 +1,21 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory';
import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteServerResolver } from 'src/engine/metadata-modules/remote-server/remote-server.resolver';
import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service';
import { RemoteTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.module';
@Module({
imports: [
TypeORMModule,
TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
RemoteTableModule,
],
providers: [
RemoteServerService,
RemoteServerResolver,
ForeignDataWrapperQueryFactory,
],
exports: [RemoteServerService],
})
export class RemoteServerModule {}

71
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts
View File

@ -1,10 +1,9 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { v4 } from 'uuid';
import { Repository } from 'typeorm';
import { DataSource, EntityManager, Repository } from 'typeorm';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { CreateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/create-remote-server.input';
import {
RemoteServerEntity,
@ -25,7 +24,8 @@ export class RemoteServerService<T extends RemoteServerType> {
private readonly remoteServerRepository: Repository<
RemoteServerEntity<RemoteServerType>
>,
private readonly typeORMService: TypeORMService,
@InjectDataSource('metadata')
private readonly metadataDataSource: DataSource,
private readonly environmentService: EnvironmentService,
private readonly foreignDataWrapperQueryFactory: ForeignDataWrapperQueryFactory,
) {}
@ -40,7 +40,6 @@ export class RemoteServerService<T extends RemoteServerType> {
validateObject(remoteServerInput.userMappingOptions);
}
const mainDatasource = this.typeORMService.getMainDataSource();
const foreignDataWrapperId = v4();
let remoteServerToCreate = {
@ -67,31 +66,37 @@ export class RemoteServerService<T extends RemoteServerType> {
};
}
const createdRemoteServer =
await this.remoteServerRepository.create(remoteServerToCreate);
const foreignDataWrapperQuery =
this.foreignDataWrapperQueryFactory.createForeignDataWrapper(
createdRemoteServer.foreignDataWrapperId,
remoteServerInput.foreignDataWrapperType,
remoteServerInput.foreignDataWrapperOptions,
);
await mainDatasource.query(foreignDataWrapperQuery);
if (remoteServerInput.userMappingOptions) {
const userMappingQuery =
this.foreignDataWrapperQueryFactory.createUserMapping(
createdRemoteServer.foreignDataWrapperId,
remoteServerInput.userMappingOptions,
return this.metadataDataSource.transaction(
async (entityManager: EntityManager) => {
const createdRemoteServer = await entityManager.create(
RemoteServerEntity,
remoteServerToCreate,
);
await mainDatasource.query(userMappingQuery);
}
const foreignDataWrapperQuery =
this.foreignDataWrapperQueryFactory.createForeignDataWrapper(
createdRemoteServer.foreignDataWrapperId,
remoteServerInput.foreignDataWrapperType,
remoteServerInput.foreignDataWrapperOptions,
);
await this.remoteServerRepository.save(createdRemoteServer);
await entityManager.query(foreignDataWrapperQuery);
return createdRemoteServer;
if (remoteServerInput.userMappingOptions) {
const userMappingQuery =
this.foreignDataWrapperQueryFactory.createUserMapping(
createdRemoteServer.foreignDataWrapperId,
remoteServerInput.userMappingOptions,
);
await entityManager.query(userMappingQuery);
}
await entityManager.save(RemoteServerEntity, createdRemoteServer);
return createdRemoteServer;
},
);
}
async deleteOneRemoteServer(
@ -111,14 +116,16 @@ export class RemoteServerService<T extends RemoteServerType> {
throw new NotFoundException('Object does not exist');
}
const mainDatasource = this.typeORMService.getMainDataSource();
return this.metadataDataSource.transaction(
async (entityManager: EntityManager) => {
await entityManager.query(
`DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`,
);
await entityManager.delete(RemoteServerEntity, id);
await mainDatasource.query(
`DROP SERVER "${remoteServer.foreignDataWrapperId}" CASCADE`,
return remoteServer;
},
);
await this.remoteServerRepository.delete(id);
return remoteServer;
}
public async findOneByIdWithinWorkspace(id: string, workspaceId: string) {

26
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto.ts Normal file
View File

@ -0,0 +1,26 @@
import { ObjectType, Field, registerEnumType } from '@nestjs/graphql';
import { IsEnum } from 'class-validator';
export enum RemoteTableStatus {
SYNCED = 'SYNCED',
NOT_SYNCED = 'NOT_SYNCED',
}
registerEnumType(RemoteTableStatus, {
name: 'RemoteTableStatus',
description: 'Status of the table',
});
@ObjectType('RemoteTable')
export class RemoteTableDTO {
@Field(() => String)
name: string;
@IsEnum(RemoteTableStatus)
@Field(() => RemoteTableStatus)
status: RemoteTableStatus;
@Field(() => String)
schema: string;
}

16
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts Normal file
View File

@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
WorkspaceDataSourceModule,
],
providers: [RemoteTableService, RemoteTableResolver],
})
export class RemoteTableModule {}

26
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts Normal file
View File

@ -0,0 +1,26 @@
import { UseGuards } from '@nestjs/common';
import { Args, Query, Resolver } from '@nestjs/graphql';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
import { RemoteTableDTO } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
@UseGuards(JwtAuthGuard)
@Resolver(() => RemoteTableDTO)
export class RemoteTableResolver {
constructor(private readonly remoteTableService: RemoteTableService) {}
@Query(() => [RemoteTableDTO])
async findAvailableRemoteTablesByServerId(
@Args('input') { id }: RemoteServerIdInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteTableService.findAvailableRemoteTablesByServerId(
id,
workspaceId,
);
}
}

112
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts Normal file
View File

@ -0,0 +1,112 @@
import { NotFoundException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { DataSource, Repository } from 'typeorm';
import {
RemoteServerType,
RemoteServerEntity,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import {
EXCLUDED_POSTGRES_SCHEMAS,
buildPostgresUrl,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util';
export class RemoteTableService {
constructor(
@InjectRepository(RemoteServerEntity, 'metadata')
private readonly remoteServerRepository: Repository<
RemoteServerEntity<RemoteServerType>
>,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly environmentService: EnvironmentService,
) {}
public async findAvailableRemoteTablesByServerId(
id: string,
workspaceId: string,
) {
const remoteServer = await this.remoteServerRepository.findOne({
where: {
id,
workspaceId,
},
});
if (!remoteServer) {
throw new NotFoundException('Remote server does not exist');
}
switch (remoteServer.foreignDataWrapperType) {
case RemoteServerType.POSTGRES_FDW:
return this.findAvailableRemotePostgresTables(
workspaceId,
remoteServer,
);
default:
throw new Error('Unsupported foreign data wrapper type');
}
}
// TODO: may be moved into a separated postgres table service once we have more use cases
private async findAvailableRemotePostgresTables(
workspaceId: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
) {
const remotePostgresTables =
await this.fetchTablesFromRemotePostgresSchema(remoteServer);
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
const currentForeignTableNames = (
await workspaceDataSource.query(
`SELECT foreign_table_name FROM information_schema.foreign_tables`,
)
).map((foreignTable) => foreignTable.foreign_table_name);
return remotePostgresTables.map((remoteTable) => ({
name: remoteTable.table_name,
schema: remoteTable.table_schema,
status: currentForeignTableNames.includes(remoteTable.table_name)
? RemoteTableStatus.SYNCED
: RemoteTableStatus.NOT_SYNCED,
}));
}
private async fetchTablesFromRemotePostgresSchema(
remoteServer: RemoteServerEntity<RemoteServerType>,
) {
const dataSource = new DataSource({
url: buildPostgresUrl(
this.environmentService.get('LOGIN_TOKEN_SECRET'),
remoteServer,
),
type: 'postgres',
logging: true,
});
await dataSource.initialize();
const schemaNames = await dataSource.query(
`SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map(
(schema) => `'${schema}'`,
).join(', ')} ) order by schema_name limit 1`,
);
const remotePostgresTables = await dataSource.query(
`SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames
.map((schemaName) => `'${schemaName.schema_name}'`)
.join(', ')})`,
);
await dataSource.destroy();
return remotePostgresTables;
}
}

29
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts Normal file
View File

@ -0,0 +1,29 @@
import { decryptText } from 'src/engine/core-modules/auth/auth.util';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
export const EXCLUDED_POSTGRES_SCHEMAS = [
'information_schema',
'pg_catalog',
'pg_toast',
];
export const buildPostgresUrl = (
secretKey: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
): string => {
const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions;
const userMappingOptions = remoteServer.userMappingOptions;
const password = decryptText(
userMappingOptions.password,
secretKey,
secretKey,
);
const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`;
return url;
};