diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index b2b1bb22e..d95bc4b8b 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -811,7 +811,6 @@ export type RemoteTable = { export type RemoteTableInput = { name: Scalars['String']['input']; remoteServerId: Scalars['ID']['input']; - schema: Scalars['String']['input']; }; /** Status of the table */ diff --git a/packages/twenty-front/src/modules/object-record/record-show/components/RecordShowContainer.tsx b/packages/twenty-front/src/modules/object-record/record-show/components/RecordShowContainer.tsx index 39c8f6c10..475a3b0a5 100644 --- a/packages/twenty-front/src/modules/object-record/record-show/components/RecordShowContainer.tsx +++ b/packages/twenty-front/src/modules/object-record/record-show/components/RecordShowContainer.tsx @@ -124,6 +124,8 @@ export const RecordShowContainer = ({ : 'inlineFieldMetadataItems', ); + const isReadOnly = objectMetadataItem.isRemote; + return ( @@ -162,7 +164,7 @@ export const RecordShowContainer = ({ hotkeyScope: InlineCellHotkeyScope.InlineCell, }} > - + } avatarType={recordIdentifier?.avatarType ?? 'rounded'} @@ -191,7 +193,7 @@ export const RecordShowContainer = ({ hotkeyScope: InlineCellHotkeyScope.InlineCell, }} > - + ))} diff --git a/packages/twenty-front/src/modules/settings/integrations/components/SettingsIntegrationDatabaseConnectionForm.tsx b/packages/twenty-front/src/modules/settings/integrations/components/SettingsIntegrationDatabaseConnectionForm.tsx index 22f389e14..8bde7c851 100644 --- a/packages/twenty-front/src/modules/settings/integrations/components/SettingsIntegrationDatabaseConnectionForm.tsx +++ b/packages/twenty-front/src/modules/settings/integrations/components/SettingsIntegrationDatabaseConnectionForm.tsx @@ -10,6 +10,7 @@ export const settingsIntegrationPostgreSQLConnectionFormSchema = z.object({ port: z.preprocess((val) => parseInt(val as string), z.number().positive()), username: z.string().min(1), password: z.string().min(1), + schema: z.string().min(1), }); type SettingsIntegrationPostgreSQLConnectionFormValues = z.infer< @@ -42,6 +43,7 @@ export const SettingsIntegrationPostgreSQLConnectionForm = () => { { name: 'port' as const, label: 'Port' }, { name: 'username' as const, label: 'Username' }, { name: 'password' as const, label: 'Password', type: 'password' }, + { name: 'schema' as const, label: 'Schema' }, ].map(({ name, label, type }) => ( { + await queryRunner.query( + `ALTER TABLE "metadata"."remoteServer" ADD "schema" text`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."remoteServer" ADD "availableTables" jsonb`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."remoteServer" DROP COLUMN "availableTables"`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."remoteServer" DROP COLUMN "schema"`, + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index b3c3534bc..9938a7eae 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -59,7 +59,7 @@ import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/ut import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity'; import { validateObjectMetadataInput } from 'src/engine/metadata-modules/object-metadata/utils/validate-object-metadata-input.util'; -import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util'; +import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; import { ObjectMetadataEntity } from './object-metadata.entity'; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/create-remote-server.input.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/create-remote-server.input.ts index 64dcccaf9..eaf2a8d14 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/create-remote-server.input.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/dtos/create-remote-server.input.ts @@ -21,4 +21,8 @@ export class CreateRemoteServerInput { @IsOptional() @Field(() => UserMappingOptionsInput, { nullable: true }) userMappingOptions?: UserMappingOptions; + + @IsOptional() + @Field(() => String, { nullable: true }) + schema?: string; } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts index 1ce95023a..116c479df 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.entity.ts @@ -10,6 +10,7 @@ import { } from 'typeorm'; import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; +import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; export enum RemoteServerType { POSTGRES_FDW = 'postgres_fdw', @@ -49,13 +50,19 @@ export class RemoteServerEntity { @Column({ nullable: true, type: 'jsonb' }) userMappingOptions: UserMappingOptions; + @Column({ type: 'text', nullable: true }) + schema: string; + @Column({ nullable: false, type: 'uuid' }) workspaceId: string; + @Column({ type: 'jsonb', nullable: true }) + availableTables: DistantTables; + @OneToMany(() => RemoteTableEntity, (table) => table.server, { cascade: true, }) - tables: Relation; + syncedTables: Relation; @CreateDateColumn({ type: 'timestamptz' }) createdAt: Date; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts index 1fb0dc74c..bee0089f3 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { ForeignDataWrapperQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/foreign-data-wrapper-query.factory'; +import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; import { RemoteServerResolver } from 'src/engine/metadata-modules/remote-server/remote-server.resolver'; import { RemoteServerService } from 'src/engine/metadata-modules/remote-server/remote-server.service'; @@ -13,6 +14,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'), RemoteTableModule, WorkspaceDataSourceModule, + TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), ], providers: [ RemoteServerService, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts index fa9d10204..b3cbbd8dc 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-server.service.ts @@ -24,6 +24,8 @@ import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/re import { UpdateRemoteServerInput } from 'src/engine/metadata-modules/remote-server/dtos/update-remote-server.input'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { updateRemoteServerRawQuery } from 'src/engine/metadata-modules/remote-server/utils/build-update-remote-server-raw-query.utils'; +import { validateRemoteServerType } from 'src/engine/metadata-modules/remote-server/utils/validate-remote-server-type.util'; +import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; @Injectable() export class RemoteServerService { @@ -38,14 +40,22 @@ export class RemoteServerService { private readonly foreignDataWrapperQueryFactory: ForeignDataWrapperQueryFactory, private readonly remoteTableService: RemoteTableService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, + @InjectRepository(FeatureFlagEntity, 'core') + private readonly featureFlagRepository: Repository, ) {} - async createOneRemoteServer( + public async createOneRemoteServer( remoteServerInput: CreateRemoteServerInput, workspaceId: string, ): Promise> { this.validateRemoteServerInputAgainstInjections(remoteServerInput); + validateRemoteServerType( + remoteServerInput.foreignDataWrapperType, + this.featureFlagRepository, + workspaceId, + ); + const foreignDataWrapperId = v4(); let remoteServerToCreate = { @@ -99,7 +109,7 @@ export class RemoteServerService { ); } - async updateOneRemoteServer( + public async updateOneRemoteServer( remoteServerInput: UpdateRemoteServerInput, workspaceId: string, ): Promise> { @@ -178,21 +188,7 @@ export class RemoteServerService { ); } - private validateRemoteServerInputAgainstInjections( - remoteServerInput: CreateRemoteServerInput | UpdateRemoteServerInput, - ) { - if (remoteServerInput.foreignDataWrapperOptions) { - validateObjectAgainstInjections( - remoteServerInput.foreignDataWrapperOptions, - ); - } - - if (remoteServerInput.userMappingOptions) { - validateObjectAgainstInjections(remoteServerInput.userMappingOptions); - } - } - - async deleteOneRemoteServer( + public async deleteOneRemoteServer( id: string, workspaceId: string, ): Promise> { @@ -265,4 +261,18 @@ export class RemoteServerService { return updateResult[0][0]; } + + private validateRemoteServerInputAgainstInjections( + remoteServerInput: CreateRemoteServerInput | UpdateRemoteServerInput, + ) { + if (remoteServerInput.foreignDataWrapperOptions) { + validateObjectAgainstInjections( + remoteServerInput.foreignDataWrapperOptions, + ); + } + + if (remoteServerInput.userMappingOptions) { + validateObjectAgainstInjections(remoteServerInput.userMappingOptions); + } + } } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.module.ts new file mode 100644 index 000000000..43e299a30 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; +import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; + +@Module({ + imports: [ + WorkspaceDataSourceModule, + TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'), + ], + providers: [DistantTableService], + exports: [DistantTableService], +}) +export class DistantTableModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts new file mode 100644 index 000000000..4be865409 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service.ts @@ -0,0 +1,102 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { EntityManager, Repository } from 'typeorm'; +import { v4 } from 'uuid'; + +import { + RemoteServerEntity, + RemoteServerType, +} from 'src/engine/metadata-modules/remote-server/remote-server.entity'; +import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; +import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column'; +import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table'; + +@Injectable() +export class DistantTableService { + constructor( + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + @InjectRepository(RemoteServerEntity, 'metadata') + private readonly remoteServerRepository: Repository< + RemoteServerEntity + >, + ) {} + + public async fetchDistantTableColumns( + remoteServer: RemoteServerEntity, + tableName: string, + ): Promise { + if (!remoteServer.availableTables) { + throw new Error('Remote server available tables are not defined'); + } + + return remoteServer.availableTables[tableName]; + } + + public async fetchDistantTableNames( + remoteServer: RemoteServerEntity, + workspaceId: string, + ): Promise { + const availableTables = + remoteServer.availableTables ?? + (await this.createAvailableTables(remoteServer, workspaceId)); + + return Object.keys(availableTables); + } + + private async createAvailableTables( + remoteServer: RemoteServerEntity, + workspaceId: string, + ): Promise { + if (!remoteServer.schema) { + throw new BadRequestException('Remote server schema is not defined'); + } + + const tmpSchemaId = v4(); + const tmpSchemaName = `${workspaceId}_${remoteServer.id}_${tmpSchemaId}`; + + const workspaceDataSource = + await this.workspaceDataSourceService.connectToWorkspaceDataSource( + workspaceId, + ); + + const availableTables = await workspaceDataSource.transaction( + async (entityManager: EntityManager) => { + await entityManager.query(`CREATE SCHEMA "${tmpSchemaName}"`); + + await entityManager.query( + `IMPORT FOREIGN SCHEMA "${remoteServer.schema}" FROM SERVER "${remoteServer.foreignDataWrapperId}" INTO "${tmpSchemaName}"`, + ); + + const createdForeignTableNames = await entityManager.query( + `SELECT table_name, column_name, data_type, udt_name FROM information_schema.columns WHERE table_schema = '${tmpSchemaName}'`, + ); + + await entityManager.query(`DROP SCHEMA "${tmpSchemaName}" CASCADE`); + + return createdForeignTableNames.reduce( + (acc, { table_name, column_name, data_type, udt_name }) => { + if (!acc[table_name]) { + acc[table_name] = []; + } + + acc[table_name].push({ + columnName: column_name, + dataType: data_type, + udtName: udt_name, + }); + + return acc; + }, + {}, + ); + }, + ); + + await this.remoteServerRepository.update(remoteServer.id, { + availableTables, + }); + + return availableTables; + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table-column.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column.ts similarity index 77% rename from packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table-column.ts rename to packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column.ts index d3c8012be..847a6ca2c 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table-column.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column.ts @@ -1,5 +1,5 @@ // Type will evolve as we add more remote table types -export type RemoteTableColumn = { +export type DistantTableColumn = { columnName: string; dataType: string; udtName: string; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts new file mode 100644 index 000000000..93a64b97a --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table.ts @@ -0,0 +1,5 @@ +import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column'; + +export type DistantTables = { + [tableName: string]: DistantTableColumn[]; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input.ts index 178e5ad80..47e8b4196 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input.ts @@ -7,7 +7,4 @@ export class RemoteTableInput { @Field(() => String) name: string; - - @Field(() => String) - schema?: string; } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module.ts deleted file mode 100644 index f12aba42a..000000000 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { RemotePostgresTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service'; - -@Module({ - providers: [RemotePostgresTableService], - exports: [RemotePostgresTableService], -}) -export class RemotePostgresTableModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service.ts deleted file mode 100644 index 46249a202..000000000 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { DataSource } from 'typeorm'; - -import { - RemoteServerEntity, - RemoteServerType, -} from 'src/engine/metadata-modules/remote-server/remote-server.entity'; -import { - buildPostgresUrl, - EXCLUDED_POSTGRES_SCHEMAS, -} from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util'; -import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { RemoteTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/types/remote-table-column'; -import { RemoteTable } from 'src/engine/metadata-modules/remote-server/remote-table/types/remote-table'; - -@Injectable() -export class RemotePostgresTableService { - constructor(private readonly environmentService: EnvironmentService) {} - - public async fetchPostgresTableColumnsSchema( - remoteServer: RemoteServerEntity, - tableName: string, - tableSchema: string, - ): Promise { - const dataSource = new DataSource({ - url: buildPostgresUrl( - this.environmentService.get('LOGIN_TOKEN_SECRET'), - remoteServer, - ), - type: 'postgres', - logging: true, - }); - - await dataSource.initialize(); - - const columns = await dataSource.query( - `SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_name = '${tableName}' AND table_schema = '${tableSchema}'`, - ); - - await dataSource.destroy(); - - return columns.map((column) => ({ - columnName: column.column_name, - dataType: column.data_type, - udtName: column.udt_name, - })); - } - - public async fetchTablesFromRemotePostgresSchema( - remoteServer: RemoteServerEntity, - ): Promise { - const dataSource = new DataSource({ - url: buildPostgresUrl( - this.environmentService.get('LOGIN_TOKEN_SECRET'), - remoteServer, - ), - type: 'postgres', - logging: true, - }); - - await dataSource.initialize(); - - const schemaNames = await dataSource.query( - `SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map( - (schema) => `'${schema}'`, - ).join(', ')} ) order by schema_name limit 1`, - ); - - const remotePostgresTables = await dataSource.query( - `SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames - .map((schemaName) => `'${schemaName.schema_name}'`) - .join(', ')})`, - ); - - await dataSource.destroy(); - - return remotePostgresTables.map((table) => ({ - tableName: table.table_name, - tableSchema: table.table_schema, - })); - } -} diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util.ts deleted file mode 100644 index b66e4fb64..000000000 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Repository } from 'typeorm/repository/Repository'; - -import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; - -import { decryptText } from 'src/engine/core-modules/auth/auth.util'; -import { - FeatureFlagEntity, - FeatureFlagKeys, -} from 'src/engine/core-modules/feature-flag/feature-flag.entity'; -import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { - RemoteServerEntity, - RemoteServerType, -} from 'src/engine/metadata-modules/remote-server/remote-server.entity'; - -export const EXCLUDED_POSTGRES_SCHEMAS = [ - 'information_schema', - 'pg_catalog', - 'pg_toast', -]; - -export const buildPostgresUrl = ( - secretKey: string, - remoteServer: RemoteServerEntity, -): string => { - const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions; - const userMappingOptions = remoteServer.userMappingOptions; - - const password = decryptText(userMappingOptions.password, secretKey); - - const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`; - - return url; -}; - -export const mapUdtNameToFieldType = (udtName: string): FieldMetadataType => { - switch (udtName) { - case 'uuid': - return FieldMetadataType.UUID; - case 'varchar': - return FieldMetadataType.TEXT; - case 'bool': - return FieldMetadataType.BOOLEAN; - case 'timestamp': - case 'timestamptz': - return FieldMetadataType.DATE_TIME; - case 'integer': - case 'int2': - case 'int4': - case 'int8': - return FieldMetadataType.NUMBER; - default: - return FieldMetadataType.TEXT; - } -}; - -export const mapUdtNameToSettings = ( - udtName: string, -): FieldMetadataSettings | undefined => { - switch (udtName) { - case 'integer': - case 'int2': - case 'int4': - case 'int8': - return { - precision: 0, - } satisfies FieldMetadataSettings; - default: - return undefined; - } -}; - -export const isPostgreSQLIntegrationEnabled = async ( - featureFlagRepository: Repository, - workspaceId: string, -) => { - const featureFlag = await featureFlagRepository.findOneBy({ - workspaceId, - key: FeatureFlagKeys.IsPostgreSQLIntegrationEnabled, - value: true, - }); - - const featureFlagEnabled = featureFlag && featureFlag.value; - - if (!featureFlagEnabled) { - throw new Error('PostgreSQL integration is not enabled'); - } -}; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.entity.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.entity.ts index 1ab4514a3..182e2e84c 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.entity.ts @@ -31,7 +31,7 @@ export class RemoteTableEntity { @Column({ nullable: false, type: 'uuid' }) remoteServerId: string; - @ManyToOne(() => RemoteServerEntity, (server) => server.tables, { + @ManyToOne(() => RemoteServerEntity, (server) => server.syncedTables, { onDelete: 'CASCADE', }) @JoinColumn({ name: 'remoteServerId' }) diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts index 72f323e87..56640927f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts @@ -1,12 +1,11 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; -import { RemotePostgresTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module'; +import { DistantTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.module'; import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver'; import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service'; @@ -17,15 +16,14 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor @Module({ imports: [ + DistantTableModule, TypeOrmModule.forFeature( [RemoteServerEntity, RemoteTableEntity], 'metadata', ), - TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), DataSourceModule, ObjectMetadataModule, FieldMetadataModule, - RemotePostgresTableModule, WorkspaceCacheVersionModule, WorkspaceMigrationModule, WorkspaceMigrationRunnerModule, diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts index 24e0728b3..ca098290a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { BadRequestException, Logger, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; @@ -10,18 +10,15 @@ import { } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; import { - isPostgreSQLIntegrationEnabled, mapUdtNameToFieldType, - mapUdtNameToSettings, -} from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util'; + mapUdtNameToFieldSettings, +} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input'; import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service'; import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input'; -import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; -import { RemotePostgresTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service'; import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { camelCase } from 'src/utils/camel-case'; import { camelToTitleCase } from 'src/utils/camel-to-title-case'; @@ -29,17 +26,19 @@ import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { - WorkspaceMigrationColumnDefinition, + WorkspaceMigrationForeignColumnDefinition, WorkspaceMigrationForeignTable, WorkspaceMigrationTableActionType, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; -import { RemoteTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/types/remote-table-column'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; -import { RemoteTable } from 'src/engine/metadata-modules/remote-server/remote-table/types/remote-table'; import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity'; import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util'; +import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service'; +import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column'; export class RemoteTableService { + private readonly logger = new Logger(RemoteTableService.name); + constructor( @InjectRepository(RemoteTableEntity, 'metadata') private readonly remoteTableRepository: Repository, @@ -47,13 +46,11 @@ export class RemoteTableService { private readonly remoteServerRepository: Repository< RemoteServerEntity >, - @InjectRepository(FeatureFlagEntity, 'core') - private readonly featureFlagRepository: Repository, private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, private readonly dataSourceService: DataSourceService, private readonly objectMetadataService: ObjectMetadataService, private readonly fieldMetadataService: FieldMetadataService, - private readonly remotePostgresTableService: RemotePostgresTableService, + private readonly distantTableService: DistantTableService, private readonly workspaceMigrationService: WorkspaceMigrationService, private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, @@ -83,13 +80,16 @@ export class RemoteTableService { (remoteTable) => remoteTable.distantTableName, ); - const tablesInRemoteSchema = - await this.fetchTablesFromRemoteSchema(remoteServer); + const distantTableNames = + await this.distantTableService.fetchDistantTableNames( + remoteServer, + workspaceId, + ); - return tablesInRemoteSchema.map((remoteTable) => ({ - name: remoteTable.tableName, - schema: remoteTable.tableSchema, - status: currentRemoteTableDistantNames.includes(remoteTable.tableName) + return distantTableNames.map((tableName) => ({ + name: tableName, + schema: remoteServer.schema, + status: currentRemoteTableDistantNames.includes(tableName) ? RemoteTableStatus.SYNCED : RemoteTableStatus.NOT_SYNCED, })); @@ -111,12 +111,6 @@ export class RemoteTableService { } public async syncRemoteTable(input: RemoteTableInput, workspaceId: string) { - if (!input.schema) { - throw new BadRequestException( - 'Schema is required for syncing remote table', - ); - } - const remoteServer = await this.remoteServerRepository.findOne({ where: { id: input.remoteServerId, @@ -161,18 +155,18 @@ export class RemoteTableService { remoteServerId: remoteServer.id, }); - const remoteTableColumns = await this.fetchTableColumnsSchema( - remoteServer, - input.name, - input.schema, - ); + const distantTableColumns = + await this.distantTableService.fetchDistantTableColumns( + remoteServer, + input.name, + ); // We only support remote tables with an id column for now. - const remoteTableIdColumn = remoteTableColumns.find( + const distantTableIdColumn = distantTableColumns.find( (column) => column.columnName === 'id', ); - if (!remoteTableIdColumn) { + if (!distantTableIdColumn) { throw new BadRequestException('Remote table must have an id column'); } @@ -181,14 +175,14 @@ export class RemoteTableService { localTableName, input, remoteServer, - remoteTableColumns, + distantTableColumns, ); await this.createRemoteTableMetadata( workspaceId, localTableName, - remoteTableColumns, - remoteTableIdColumn, + distantTableColumns, + distantTableIdColumn, dataSourceMetatada.id, ); @@ -199,7 +193,7 @@ export class RemoteTableService { return { id: remoteTableEntity.id, name: input.name, - schema: input.schema, + schema: remoteServer.schema, status: RemoteTableStatus.SYNCED, }; } @@ -232,7 +226,7 @@ export class RemoteTableService { return { name: input.name, - schema: input.schema, + schema: remoteServer.schema, status: RemoteTableStatus.NOT_SYNCED, }; } @@ -300,46 +294,6 @@ export class RemoteTableService { await this.workspaceCacheVersionService.incrementVersion(workspaceId); } - private async fetchTableColumnsSchema( - remoteServer: RemoteServerEntity, - tableName: string, - tableSchema: string, - ): Promise { - switch (remoteServer.foreignDataWrapperType) { - case RemoteServerType.POSTGRES_FDW: - await isPostgreSQLIntegrationEnabled( - this.featureFlagRepository, - remoteServer.workspaceId, - ); - - return this.remotePostgresTableService.fetchPostgresTableColumnsSchema( - remoteServer, - tableName, - tableSchema, - ); - default: - throw new BadRequestException('Unsupported foreign data wrapper type'); - } - } - - private async fetchTablesFromRemoteSchema( - remoteServer: RemoteServerEntity, - ): Promise { - switch (remoteServer.foreignDataWrapperType) { - case RemoteServerType.POSTGRES_FDW: - await isPostgreSQLIntegrationEnabled( - this.featureFlagRepository, - remoteServer.workspaceId, - ); - - return this.remotePostgresTableService.fetchTablesFromRemotePostgresSchema( - remoteServer, - ); - default: - throw new BadRequestException('Unsupported foreign data wrapper type'); - } - } - private async validateTableNameDoesNotExists( tableName: string, workspaceId: string, @@ -382,14 +336,8 @@ export class RemoteTableService { localTableName: string, remoteTableInput: RemoteTableInput, remoteServer: RemoteServerEntity, - remoteTableColumns: RemoteTableColumn[], + distantTableColumns: DistantTableColumn[], ) { - if (!remoteTableInput.schema) { - throw new BadRequestException( - 'Schema is required for creating foreign table', - ); - } - const workspaceMigration = await this.workspaceMigrationService.createCustomMigration( generateMigrationName(`create-foreign-table-${localTableName}`), @@ -399,15 +347,16 @@ export class RemoteTableService { name: localTableName, action: WorkspaceMigrationTableActionType.CREATE_FOREIGN_TABLE, foreignTable: { - columns: remoteTableColumns.map( + columns: distantTableColumns.map( (column) => ({ - columnName: column.columnName, + columnName: camelCase(column.columnName), columnType: column.dataType, - }) satisfies WorkspaceMigrationColumnDefinition, + distantColumnName: column.columnName, + }) satisfies WorkspaceMigrationForeignColumnDefinition, ), referencedTableName: remoteTableInput.name, - referencedTableSchema: remoteTableInput.schema, + referencedTableSchema: remoteServer.schema, foreignDataWrapperId: remoteServer.foreignDataWrapperId, } satisfies WorkspaceMigrationForeignTable, }, @@ -431,8 +380,8 @@ export class RemoteTableService { private async createRemoteTableMetadata( workspaceId: string, localTableName: string, - remoteTableColumns: RemoteTableColumn[], - remoteTableIdColumn: RemoteTableColumn, + distantTableColumns: DistantTableColumn[], + distantTableIdColumn: DistantTableColumn, dataSourceMetadataId: string, ) { const objectMetadata = await this.objectMetadataService.createOne({ @@ -445,33 +394,39 @@ export class RemoteTableService { workspaceId: workspaceId, icon: 'IconPlug', isRemote: true, - primaryKeyColumnType: remoteTableIdColumn.udtName, - // TODO: function should work for other types than Postgres - primaryKeyFieldMetadataSettings: mapUdtNameToSettings( - remoteTableIdColumn.udtName, + primaryKeyColumnType: distantTableIdColumn.udtName, + primaryKeyFieldMetadataSettings: mapUdtNameToFieldSettings( + distantTableIdColumn.udtName, ), } satisfies CreateObjectInput); - for (const column of remoteTableColumns) { - const field = await this.fieldMetadataService.createOne({ - name: column.columnName, - label: camelToTitleCase(camelCase(column.columnName)), - description: 'Field of remote', - // TODO: function should work for other types than Postgres - type: mapUdtNameToFieldType(column.udtName), - workspaceId: workspaceId, - objectMetadataId: objectMetadata.id, - isRemoteCreation: true, - isNullable: true, - icon: 'IconPlug', - // TODO: function should work for other types than Postgres - settings: mapUdtNameToSettings(column.udtName), - } satisfies CreateFieldInput); + for (const column of distantTableColumns) { + const columnName = camelCase(column.columnName); - if (column.columnName === 'id') { - await this.objectMetadataService.updateOne(objectMetadata.id, { - labelIdentifierFieldMetadataId: field.id, - }); + // TODO: return error to the user when a column cannot be managed + try { + const field = await this.fieldMetadataService.createOne({ + name: columnName, + label: camelToTitleCase(columnName), + description: 'Field of remote', + type: mapUdtNameToFieldType(column.udtName), + workspaceId: workspaceId, + objectMetadataId: objectMetadata.id, + isRemoteCreation: true, + isNullable: true, + icon: 'IconPlug', + settings: mapUdtNameToFieldSettings(column.udtName), + } satisfies CreateFieldInput); + + if (columnName === 'id') { + await this.objectMetadataService.updateOne(objectMetadata.id, { + labelIdentifierFieldMetadataId: field.id, + }); + } + } catch (error) { + this.logger.error( + `Could not create field ${columnName} for remote table ${localTableName}: ${error}`, + ); } } } diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table.ts deleted file mode 100644 index 443f1245d..000000000 --- a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/types/remote-table.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Type will evolve as we add more remote table types -export type RemoteTable = { - tableName: string; - tableSchema: string; -}; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util.ts new file mode 100644 index 000000000..4b18e072a --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util.ts @@ -0,0 +1,38 @@ +import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; + +import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; + +export const mapUdtNameToFieldType = (udtName: string): FieldMetadataType => { + switch (udtName) { + case 'uuid': + return FieldMetadataType.UUID; + case 'varchar': + return FieldMetadataType.TEXT; + case 'bool': + return FieldMetadataType.BOOLEAN; + case 'timestamp': + case 'timestamptz': + return FieldMetadataType.DATE_TIME; + case 'integer': + case 'int2': + case 'int4': + return FieldMetadataType.NUMBER; + default: + return FieldMetadataType.TEXT; + } +}; + +export const mapUdtNameToFieldSettings = ( + udtName: string, +): FieldMetadataSettings | undefined => { + switch (udtName) { + case 'integer': + case 'int2': + case 'int4': + return { + precision: 0, + } satisfies FieldMetadataSettings; + default: + return undefined; + } +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/remote-server/utils/validate-remote-server-type.util.ts b/packages/twenty-server/src/engine/metadata-modules/remote-server/utils/validate-remote-server-type.util.ts new file mode 100644 index 000000000..36921e2d1 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/remote-server/utils/validate-remote-server-type.util.ts @@ -0,0 +1,40 @@ +import { BadRequestException } from '@nestjs/common'; + +import { Repository } from 'typeorm'; + +import { + FeatureFlagEntity, + FeatureFlagKeys, +} from 'src/engine/core-modules/feature-flag/feature-flag.entity'; +import { RemoteServerType } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; + +export const validateRemoteServerType = async ( + remoteServerType: RemoteServerType, + featureFlagRepository: Repository, + workspaceId: string, +) => { + const featureFlagKey = getFeatureFlagKey(remoteServerType); + + const featureFlag = await featureFlagRepository.findOneBy({ + workspaceId, + key: featureFlagKey, + value: true, + }); + + const featureFlagEnabled = featureFlag && featureFlag.value; + + if (!featureFlagEnabled) { + throw new BadRequestException(`Type ${remoteServerType} is not supported.`); + } +}; + +const getFeatureFlagKey = (remoteServerType: RemoteServerType) => { + switch (remoteServerType) { + case RemoteServerType.POSTGRES_FDW: + return FeatureFlagKeys.IsPostgreSQLIntegrationEnabled; + default: + throw new BadRequestException( + `Type ${remoteServerType} is not supported.`, + ); + } +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts index 224eaf4e3..ad8068883 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts @@ -62,8 +62,13 @@ export type WorkspaceMigrationCreateComment = { comment: string; }; +export type WorkspaceMigrationForeignColumnDefinition = + WorkspaceMigrationColumnDefinition & { + distantColumnName: string; + }; + export type WorkspaceMigrationForeignTable = { - columns: WorkspaceMigrationColumnDefinition[]; + columns: WorkspaceMigrationForeignColumnDefinition[]; referencedTableName: string; referencedTableSchema: string; foreignDataWrapperId: string; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index 96fbc52eb..2f0991625 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -489,7 +489,10 @@ export class WorkspaceMigrationRunnerService { } const foreignTableColumns = foreignTable.columns - .map((column) => `"${column.columnName}" ${column.columnType}`) + .map( + (column) => + `"${column.columnName}" ${column.columnType} OPTIONS (column_name '${column.distantColumnName}')`, + ) .join(', '); await queryRunner.query(