diff --git a/server/package.json b/server/package.json index c84166df4..6ab6f7edc 100644 --- a/server/package.json +++ b/server/package.json @@ -44,6 +44,7 @@ "@nestjs/platform-express": "^9.0.0", "@nestjs/serve-static": "^3.0.0", "@nestjs/terminus": "^9.2.2", + "@nestjs/typeorm": "^10.0.0", "@paljs/plugins": "^5.3.3", "@prisma/client": "4.13.0", "@sentry/node": "^7.66.0", diff --git a/server/src/core/tenant/datasource/datasource.module.ts b/server/src/core/tenant/datasource/datasource.module.ts deleted file mode 100644 index 387969d26..000000000 --- a/server/src/core/tenant/datasource/datasource.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { DataSourceService } from './services/datasource.service'; - -@Module({ - exports: [DataSourceService], - providers: [DataSourceService], -}) -export class DataSourceModule {} diff --git a/server/src/core/tenant/datasource/entities/data-source-metadata.entity.ts b/server/src/core/tenant/datasource/entities/data-source-metadata.entity.ts deleted file mode 100644 index b0fba4ed0..000000000 --- a/server/src/core/tenant/datasource/entities/data-source-metadata.entity.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm'; - -// export const dataSourceEntity = new EntitySchema({ -// name: 'data_sources', -// columns: { -// id: { -// primary: true, -// type: 'uuid', -// generated: 'uuid', -// }, -// url: { -// type: 'text', -// nullable: true, -// }, -// schema: { -// type: 'text', -// nullable: true, -// }, -// type: { -// type: 'text', -// nullable: true, -// }, -// name: { -// type: 'text', -// nullable: true, -// }, -// is_remote: { -// type: 'boolean', -// nullable: true, -// default: false, -// }, -// workspace_id: { -// type: 'uuid', -// nullable: true, -// }, -// }, -// }); - -@Entity('data_source_metadata') -export class DataSourceMetadata { - @PrimaryGeneratedColumn('uuid') - id: string; - - @CreateDateColumn() - created_at: Date; - - @UpdateDateColumn() - updated_at: Date; - - @Column({ nullable: true }) - url: string; - - @Column({ nullable: true }) - schema: string; - - @Column({ default: 'postgres' }) - type: string; - - @Column({ nullable: true }) - name: string; - - @Column({ default: false }) - is_remote: boolean; - - @Column({ nullable: false }) - workspace_id: string; -} diff --git a/server/src/core/tenant/datasource/entities/field-metadata.entity.ts b/server/src/core/tenant/datasource/entities/field-metadata.entity.ts deleted file mode 100644 index 0412f0328..000000000 --- a/server/src/core/tenant/datasource/entities/field-metadata.entity.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm'; - -import { ObjectMetadata } from './object-metadata'; - -// export const fieldMetadataEntity = new EntitySchema({ -// name: 'field_metadata', -// columns: { -// id: { -// primary: true, -// type: 'uuid', -// generated: 'uuid', -// }, -// object_id: { -// type: 'uuid', -// nullable: true, -// }, -// type: { -// type: 'text', -// nullable: true, -// }, -// name: { -// type: 'text', -// nullable: true, -// }, -// is_custom: { -// type: 'boolean', -// nullable: true, -// default: false, -// }, -// workspace_id: { -// type: 'uuid', -// nullable: true, -// }, -// }, -// relations: { -// object: { -// type: 'many-to-one', -// target: 'object_metadata', -// joinColumn: { -// name: 'object_id', -// referencedColumnName: 'id', -// }, -// inverseSide: 'fields', -// }, -// } as any, -// }); - -@Entity('field_metadata') -export class FieldMetadata { - @PrimaryGeneratedColumn('uuid') - id: string; - - @CreateDateColumn() - created_at: Date; - - @UpdateDateColumn() - updated_at: Date; - - @Column({ nullable: false }) - object_id: string; - - @Column({ nullable: false }) - type: string; - - @Column({ nullable: false }) - name: string; - - @Column({ default: false }) - is_custom: boolean; - - @Column({ nullable: false }) - workspace_id: string; - - @ManyToOne(() => ObjectMetadata, (object) => object.fields) - @JoinColumn({ name: 'object_id' }) - object: ObjectMetadata; -} diff --git a/server/src/core/tenant/datasource/entities/object-metadata.ts b/server/src/core/tenant/datasource/entities/object-metadata.ts deleted file mode 100644 index 1364e33eb..000000000 --- a/server/src/core/tenant/datasource/entities/object-metadata.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - OneToMany, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm'; - -import { FieldMetadata } from './field-metadata.entity'; - -// export const objectMetadataEntity = new EntitySchema({ -// name: 'object_metadata', -// columns: { -// id: { -// primary: true, -// type: 'uuid', -// generated: 'uuid', -// }, -// data_source_id: { -// type: 'uuid', -// nullable: true, -// }, -// name: { -// type: 'text', -// nullable: true, -// }, -// is_custom: { -// type: 'boolean', -// nullable: true, -// default: false, -// }, -// workspace_id: { -// type: 'uuid', -// nullable: true, -// }, -// }, -// relations: { -// fields: { -// type: 'one-to-many', -// target: 'field_metadata', -// inverseSide: 'object', -// }, -// } as any, -// }); - -@Entity('object_metadata') -export class ObjectMetadata { - @PrimaryGeneratedColumn('uuid') - id: string; - - @CreateDateColumn() - created_at: Date; - - @UpdateDateColumn() - updated_at: Date; - - @Column({ nullable: false }) - data_source_id: string; - - @Column({ nullable: false }) - name: string; - - @Column({ default: false }) - is_custom: boolean; - - @Column({ nullable: false }) - workspace_id: string; - - @OneToMany(() => FieldMetadata, (field) => field.object) - fields: FieldMetadata[]; -} diff --git a/server/src/core/tenant/datasource/entities/relation-medata.ts b/server/src/core/tenant/datasource/entities/relation-medata.ts deleted file mode 100644 index 593a01c5e..000000000 --- a/server/src/core/tenant/datasource/entities/relation-medata.ts +++ /dev/null @@ -1,30 +0,0 @@ -// export const relationMetadataEntity = new EntitySchema({ -// name: 'relation_metadata', -// columns: { -// id: { -// primary: true, -// type: 'uuid', -// generated: 'uuid', -// }, -// source_field_id: { -// type: 'uuid', -// nullable: true, -// }, -// target_object_id: { -// type: 'uuid', -// nullable: true, -// }, -// target_foreign_key: { -// type: 'uuid', -// nullable: true, -// }, -// type: { -// type: 'text', -// nullable: true, -// }, -// workspace_id: { -// type: 'uuid', -// nullable: true, -// }, -// }, -// }); diff --git a/server/src/core/tenant/datasource/services/datasource.service.ts b/server/src/core/tenant/datasource/services/datasource.service.ts deleted file mode 100644 index d3ca254ce..000000000 --- a/server/src/core/tenant/datasource/services/datasource.service.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; - -import { DataSource, EntitySchema } from 'typeorm'; -import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; - -import { EnvironmentService } from 'src/integrations/environment/environment.service'; -import { DataSourceMetadata } from 'src/core/tenant/datasource/entities/data-source-metadata.entity'; -import { ObjectMetadata } from 'src/core/tenant/datasource/entities/object-metadata'; -import { FieldMetadata } from 'src/core/tenant/datasource/entities/field-metadata.entity'; -import { baseColumns } from 'src/core/tenant/datasource/entities/base.entity'; -import { - convertFieldTypeToPostgresType, - sanitizeColumnName, - uuidToBase36, -} from 'src/core/tenant/datasource/utils/datasource.util'; - -@Injectable() -export class DataSourceService implements OnModuleInit, OnModuleDestroy { - private mainDataSource: DataSource; - private metadataDataSource: DataSource; - private connectionOptions: PostgresConnectionOptions; - private dataSources = new Map(); - - constructor(environmentService: EnvironmentService) { - this.connectionOptions = { - url: environmentService.getPGDatabaseUrl(), - type: 'postgres', - logging: false, - schema: 'public', - }; - this.mainDataSource = new DataSource(this.connectionOptions); - this.metadataDataSource = new DataSource({ - ...this.connectionOptions, - schema: 'metadata', - synchronize: true, // TODO: remove this in production - entities: [DataSourceMetadata, ObjectMetadata, FieldMetadata], - }); - } - - /** - * - * Returns the schema name for a given workspaceId - * @param workspaceId - * @returns string - */ - public getSchemaName(workspaceId: string): string { - return `workspace_${uuidToBase36(workspaceId)}`; - } - - /** - * Creates a new schema for a given workspaceId - * @param workspaceId - * @returns Promise - */ - public async createWorkspaceSchema(workspaceId: string): Promise { - const schemaName = this.getSchemaName(workspaceId); - - const queryRunner = this.mainDataSource.createQueryRunner(); - await queryRunner.createSchema(schemaName, true); - await queryRunner.release(); - - await this.insertNewDataSourceMetadata(workspaceId, schemaName); - } - - /** - * Inserts a new data source - * @param workspaceId - * @param workspaceSchema this can be computed from the workspaceId but it won't be the case for remote data sources - * @returns Promise - */ - private async insertNewDataSourceMetadata( - workspaceId: string, - workspaceSchema: string, - ): Promise { - await this.metadataDataSource - ?.createQueryBuilder() - .insert() - .into(DataSourceMetadata) - .values({ - workspace_id: workspaceId, - schema: workspaceSchema, - }) - .execute(); - } - - /** - * Connects to a workspace data source using the workspace metadata. Returns a cached connection if it exists. - * @param workspaceId - * @returns Promise - */ - public async connectToWorkspaceDataSource( - workspaceId: string, - ): Promise { - // if (this.dataSources.has(workspaceId)) { - // const cachedDataSource = this.dataSources.get(workspaceId); - // return cachedDataSource; - // } - - const dataSourceMetadata = - await this.getLastDataSourceMetadataFromWorkspaceIdOrFail(workspaceId); - - const schema = dataSourceMetadata.schema; - - // Probably not needed as we will ask for the schema name OR store public by default if it's remote - if (!schema && !dataSourceMetadata.is_remote) { - throw Error( - "No schema found for this non-remote data source, we don't want to fallback to public for workspace data sources.", - ); - } - - const metadata = await this.fetchObjectsAndFieldsFromMetadata(workspaceId); - - const entities = this.convertMetadataToEntities(metadata); - - const workspaceDataSource = new DataSource({ - ...this.connectionOptions, - schema, - entities: entities, - synchronize: true, // TODO: remove this in production - }); - - await workspaceDataSource.initialize(); - - return workspaceDataSource; - // this.dataSources.set(workspaceId, workspaceDataSource); - - // return this.dataSources.get(workspaceId); - } - - /** - * Returns the last data source metadata for a given workspaceId - * In the future we should handle multiple data sources. - * Most likely the twenty workspace connection and n remote connections should be fetched. - * - * @param workspaceId - * @returns Promise - */ - private async getLastDataSourceMetadataFromWorkspaceIdOrFail( - workspaceId: string, - ): Promise { - return this.metadataDataSource - ?.createQueryBuilder() - .select('data_source') - .from(DataSourceMetadata, 'data_source') - .where('data_source.workspace_id = :workspaceId', { workspaceId }) - .orderBy('data_source.created_at', 'DESC') - .getOneOrFail(); - } - - /** - * Returns all the objects and fields for a given workspaceId and the first associated data source metadata registered. - * - * @param workspaceId - * @returns - */ - public async fetchObjectsAndFieldsFromMetadata(workspaceId: string) { - const dataSource = - await this.getLastDataSourceMetadataFromWorkspaceIdOrFail(workspaceId); - - const objectRepository = - this.metadataDataSource.getRepository(ObjectMetadata); - - return await objectRepository - .createQueryBuilder('object_metadata') - .select(['object_metadata.id', 'object_metadata.name']) - .leftJoinAndSelect('object_metadata.fields', 'field') - .where('object_metadata.data_source_id = :dataSourceId', { - dataSourceId: dataSource?.id, - }) - .getMany(); - } - - /** - * Converts the metadata to entities that can be interpreted by typeorm. - * @param metadata - * @returns EntitySchema[] - * - */ - public convertMetadataToEntities(metadata): EntitySchema[] { - const entities = metadata.map((object) => { - return new EntitySchema({ - name: object.name, - columns: { - ...baseColumns, - ...object.fields.reduce((columns, field) => { - return { - ...columns, - [sanitizeColumnName(field.name)]: { - type: convertFieldTypeToPostgresType(field.type), - nullable: true, - }, - }; - }, {}), - }, - }); - }); - - return entities; - } - - async onModuleInit() { - await this.mainDataSource.initialize(); - await this.metadataDataSource.initialize(); - } - - async onModuleDestroy(): Promise { - await this.mainDataSource.destroy(); - await this.metadataDataSource.destroy(); - } -} diff --git a/server/src/core/tenant/metadata/metadata.controller.ts b/server/src/core/tenant/metadata/metadata.controller.ts deleted file mode 100644 index e5332d27f..000000000 --- a/server/src/core/tenant/metadata/metadata.controller.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Controller, Get, UseGuards } from '@nestjs/common'; - -import { Workspace } from '@prisma/client'; - -import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator'; -import { JwtAuthGuard } from 'src/guards/jwt.auth.guard'; - -import { MetadataService } from './metadata.service'; - -@UseGuards(JwtAuthGuard) -@Controller('metadata') -export class MetadataController { - constructor(private readonly metadataService: MetadataService) {} - - @Get() - async getMetadata(@AuthWorkspace() workspace: Workspace) { - return this.metadataService.fetchMetadataFromWorkspaceId(workspace.id); - } -} diff --git a/server/src/core/tenant/metadata/metadata.module.ts b/server/src/core/tenant/metadata/metadata.module.ts deleted file mode 100644 index 3c9a6bee5..000000000 --- a/server/src/core/tenant/metadata/metadata.module.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Module } from '@nestjs/common'; - -import { DataSourceModule } from 'src/core/tenant/datasource/datasource.module'; - -import { MetadataController } from './metadata.controller'; -import { MetadataService } from './metadata.service'; - -@Module({ - imports: [DataSourceModule], - controllers: [MetadataController], - providers: [MetadataService], -}) -export class MetadataModule {} diff --git a/server/src/core/tenant/metadata/metadata.service.ts b/server/src/core/tenant/metadata/metadata.service.ts deleted file mode 100644 index 696ad4bd3..000000000 --- a/server/src/core/tenant/metadata/metadata.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { DataSourceService } from 'src/core/tenant/datasource/services/datasource.service'; - -@Injectable() -export class MetadataService { - constructor(private readonly dataSourceService: DataSourceService) {} - - public async fetchMetadataFromWorkspaceId(workspaceId: string) { - await this.dataSourceService.connectToWorkspaceDataSource(workspaceId); - - return await this.dataSourceService.fetchObjectsAndFieldsFromMetadata( - workspaceId, - ); - } -} diff --git a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts new file mode 100644 index 000000000..49c9ad7b4 --- /dev/null +++ b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts @@ -0,0 +1,40 @@ +import { + Column, + CreateDateColumn, + Entity, + PrimaryGeneratedColumn, + UpdateDateColumn, + DataSourceOptions, +} from 'typeorm'; + +type DataSourceType = DataSourceOptions['type']; + +@Entity('data_source_metadata') +export class DataSourceMetadata { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: true }) + url: string; + + @Column({ nullable: true }) + schema: string; + + @Column({ type: 'enum', enum: ['postgres'], default: 'postgres' }) + type: DataSourceType; + + @Column({ nullable: true }) + name: string; + + @Column({ default: false, name: 'is_remote' }) + isRemote: boolean; + + @Column({ nullable: false, name: 'workspace_id' }) + workspaceId: string; + + @CreateDateColumn({ name: 'created_at' }) + createdAt: Date; + + @UpdateDateColumn({ name: 'updated_at' }) + updatedAt: Date; +} diff --git a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.module.ts b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.module.ts new file mode 100644 index 000000000..b4fdc2f38 --- /dev/null +++ b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { DataSourceMetadataService } from './data-source-metadata.service'; +import { DataSourceMetadata } from './data-source-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([DataSourceMetadata], 'metadata')], + providers: [DataSourceMetadataService], + exports: [DataSourceMetadataService], +}) +export class DataSourceMetadataModule {} diff --git a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.spec.ts b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.spec.ts new file mode 100644 index 000000000..50403ed8b --- /dev/null +++ b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.spec.ts @@ -0,0 +1,27 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; + +import { DataSourceMetadataService } from './data-source-metadata.service'; +import { DataSourceMetadata } from './data-source-metadata.entity'; + +describe('DataSourceMetadataService', () => { + let service: DataSourceMetadataService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DataSourceMetadataService, + { + provide: getRepositoryToken(DataSourceMetadata, 'metadata'), + useValue: {}, + }, + ], + }).compile(); + + service = module.get(DataSourceMetadataService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts new file mode 100644 index 000000000..6c094656e --- /dev/null +++ b/server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts @@ -0,0 +1,37 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { DataSourceMetadata } from './data-source-metadata.entity'; + +@Injectable() +export class DataSourceMetadataService { + constructor( + @InjectRepository(DataSourceMetadata, 'metadata') + private readonly dataSourceMetadataRepository: Repository, + ) {} + + async createDataSourceMetadata(workspaceId: string, workspaceSchema: string) { + // TODO: Double check if this is the correct way to do this + const dataSource = await this.dataSourceMetadataRepository.findOne({ + where: { workspaceId }, + }); + + if (dataSource) { + return dataSource; + } + + return this.dataSourceMetadataRepository.save({ + workspaceId, + schema: workspaceSchema, + }); + } + + getDataSourcesMedataFromWorkspaceId(workspaceId: string) { + return this.dataSourceMetadataRepository.find({ + where: { workspaceId }, + order: { createdAt: 'DESC' }, + }); + } +} diff --git a/server/src/tenant/metadata/data-source/data-source.module.ts b/server/src/tenant/metadata/data-source/data-source.module.ts new file mode 100644 index 000000000..9a108643c --- /dev/null +++ b/server/src/tenant/metadata/data-source/data-source.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; + +import { DataSourceMetadataModule } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.module'; +import { EntitySchemaGeneratorModule } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.module'; + +import { DataSourceService } from './data-source.service'; + +@Module({ + imports: [DataSourceMetadataModule, EntitySchemaGeneratorModule], + providers: [DataSourceService], + exports: [DataSourceService], +}) +export class DataSourceModule {} diff --git a/server/src/tenant/metadata/data-source/data-source.service.spec.ts b/server/src/tenant/metadata/data-source/data-source.service.spec.ts new file mode 100644 index 000000000..61a798b48 --- /dev/null +++ b/server/src/tenant/metadata/data-source/data-source.service.spec.ts @@ -0,0 +1,39 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { EnvironmentService } from 'src/integrations/environment/environment.service'; +import { DataSourceMetadataService } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.service'; +import { EntitySchemaGeneratorService } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.service'; + +import { DataSourceService } from './data-source.service'; + +describe('DataSourceService', () => { + let service: DataSourceService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DataSourceService, + { + provide: EnvironmentService, + useValue: { + getPGDatabaseUrl: () => '', + }, + }, + { + provide: DataSourceMetadataService, + useValue: {}, + }, + { + provide: EntitySchemaGeneratorService, + useValue: {}, + }, + ], + }).compile(); + + service = module.get(DataSourceService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/data-source/data-source.service.ts b/server/src/tenant/metadata/data-source/data-source.service.ts new file mode 100644 index 000000000..f89f77ea3 --- /dev/null +++ b/server/src/tenant/metadata/data-source/data-source.service.ts @@ -0,0 +1,127 @@ +import { + Injectable, + NotFoundException, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; + +import { DataSource } from 'typeorm'; + +import { EnvironmentService } from 'src/integrations/environment/environment.service'; +import { DataSourceMetadataService } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.service'; +import { EntitySchemaGeneratorService } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.service'; + +import { uuidToBase36 } from './data-source.util'; + +@Injectable() +export class DataSourceService implements OnModuleInit, OnModuleDestroy { + private mainDataSource: DataSource; + + constructor( + private readonly environmentService: EnvironmentService, + private readonly dataSourceMetadataService: DataSourceMetadataService, + private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService, + ) { + this.mainDataSource = new DataSource({ + url: environmentService.getPGDatabaseUrl(), + type: 'postgres', + logging: false, + schema: 'public', + }); + } + + /** + * Creates a new schema for a given workspaceId + * @param workspaceId + * @returns Promise + */ + public async createWorkspaceSchema(workspaceId: string): Promise { + const schemaName = this.getSchemaName(workspaceId); + + const queryRunner = this.mainDataSource.createQueryRunner(); + await queryRunner.createSchema(schemaName, true); + await queryRunner.release(); + + await this.dataSourceMetadataService.createDataSourceMetadata( + workspaceId, + schemaName, + ); + } + + /** + * Connects to a workspace data source using the workspace metadata. Returns a cached connection if it exists. + * @param workspaceId + * @returns Promise + */ + public async connectToWorkspaceDataSource( + workspaceId: string, + ): Promise { + // if (this.dataSources.has(workspaceId)) { + // const cachedDataSource = this.dataSources.get(workspaceId); + // return cachedDataSource; + // } + + const dataSourcesMetadata = + await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId( + workspaceId, + ); + + if (dataSourcesMetadata.length === 0) { + throw new NotFoundException( + `We can't find any data source for this workspace id (${workspaceId}).`, + ); + } + + const dataSourceMetadata = dataSourcesMetadata[0]; + const schema = dataSourceMetadata.schema; + + // Probably not needed as we will ask for the schema name OR store public by default if it's remote + if (!schema && !dataSourceMetadata.isRemote) { + throw Error( + "No schema found for this non-remote data source, we don't want to fallback to public for workspace data sources.", + ); + } + + const entities = + await this.entitySchemaGeneratorService.getTypeORMEntitiesByDataSourceId( + dataSourceMetadata.id, + ); + + const workspaceDataSource = new DataSource({ + // TODO: We should use later dataSourceMetadata.type and use a switch case condition to create the right data source + url: dataSourceMetadata.url ?? this.environmentService.getPGDatabaseUrl(), + type: 'postgres', + logging: ['query'], + schema, + entities: entities, + synchronize: true, // TODO: remove this in production + }); + + await workspaceDataSource.initialize(); + + return workspaceDataSource; + // this.dataSources.set(workspaceId, workspaceDataSource); + + // return this.dataSources.get(workspaceId); + } + + /** + * + * Returns the schema name for a given workspaceId + * @param workspaceId + * @returns string + */ + private getSchemaName(workspaceId: string): string { + return `workspace_${uuidToBase36(workspaceId)}`; + } + + async onModuleInit() { + // Init main data source "default" schema + await this.mainDataSource.initialize(); + } + + async onModuleDestroy() { + // Destroy main data source "default" schema + await this.mainDataSource.destroy(); + } +} diff --git a/server/src/tenant/metadata/data-source/data-source.util.ts b/server/src/tenant/metadata/data-source/data-source.util.ts new file mode 100644 index 000000000..5cc23c7e1 --- /dev/null +++ b/server/src/tenant/metadata/data-source/data-source.util.ts @@ -0,0 +1,14 @@ +/** + * Converts a UUID to a base 36 string. + * This is used to generate the schema name since hyphens from workspace uuid are not allowed in postgres schema names. + * + * @param uuid + * @returns + */ +export function uuidToBase36(uuid: string): string { + const hexString = uuid.replace(/-/g, ''); + const base10Number = BigInt('0x' + hexString); + const base36String = base10Number.toString(36); + + return base36String; +} diff --git a/server/src/core/tenant/datasource/entities/base.entity.ts b/server/src/tenant/metadata/entity-schema-generator/base.entity.ts similarity index 89% rename from server/src/core/tenant/datasource/entities/base.entity.ts rename to server/src/tenant/metadata/entity-schema-generator/base.entity.ts index cceb3ee47..f26bb08ef 100644 --- a/server/src/core/tenant/datasource/entities/base.entity.ts +++ b/server/src/tenant/metadata/entity-schema-generator/base.entity.ts @@ -4,4 +4,4 @@ export const baseColumns = { type: 'uuid', generated: 'uuid', }, -}; +} as const; diff --git a/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.module.ts b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.module.ts new file mode 100644 index 000000000..96f080744 --- /dev/null +++ b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; + +import { ObjectMetadataModule } from 'src/tenant/metadata/object-metadata/object-metadata.module'; + +import { EntitySchemaGeneratorService } from './entity-schema-generator.service'; + +@Module({ + imports: [ObjectMetadataModule], + providers: [EntitySchemaGeneratorService], + exports: [EntitySchemaGeneratorService], +}) +export class EntitySchemaGeneratorModule {} diff --git a/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.spec.ts b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.spec.ts new file mode 100644 index 000000000..5ca4c0ddc --- /dev/null +++ b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.spec.ts @@ -0,0 +1,29 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { ObjectMetadataService } from 'src/tenant/metadata/object-metadata/object-metadata.service'; + +import { EntitySchemaGeneratorService } from './entity-schema-generator.service'; + +describe('EntitySchemaGeneratorService', () => { + let service: EntitySchemaGeneratorService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + EntitySchemaGeneratorService, + { + provide: ObjectMetadataService, + useValue: {}, + }, + ], + }).compile(); + + service = module.get( + EntitySchemaGeneratorService, + ); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts new file mode 100644 index 000000000..f7e629b29 --- /dev/null +++ b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts @@ -0,0 +1,43 @@ +import { Injectable } from '@nestjs/common'; + +import { EntitySchema } from 'typeorm'; + +import { ObjectMetadataService } from 'src/tenant/metadata/object-metadata/object-metadata.service'; + +import { baseColumns } from './base.entity'; +import { + convertFieldTypeToPostgresType, + sanitizeColumnName, +} from './entity-schema-generator.util'; + +@Injectable() +export class EntitySchemaGeneratorService { + constructor(private readonly objectMetadataService: ObjectMetadataService) {} + + async getTypeORMEntitiesByDataSourceId(dataSourceId: string) { + const objectMetadata = + await this.objectMetadataService.getObjectMetadataFromDataSourceId( + dataSourceId, + ); + + const entities = objectMetadata.map((object) => { + return new EntitySchema({ + name: object.name, + columns: { + ...baseColumns, + ...object.fields.reduce((columns, field) => { + return { + ...columns, + [sanitizeColumnName(field.name)]: { + type: convertFieldTypeToPostgresType(field.type), + nullable: true, + }, + }; + }, {}), + }, + }); + }); + + return entities; + } +} diff --git a/server/src/core/tenant/datasource/utils/datasource.util.ts b/server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.util.ts similarity index 100% rename from server/src/core/tenant/datasource/utils/datasource.util.ts rename to server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.util.ts diff --git a/server/src/tenant/metadata/field-metadata/field-metadata.entity.ts b/server/src/tenant/metadata/field-metadata/field-metadata.entity.ts new file mode 100644 index 000000000..3dacaa896 --- /dev/null +++ b/server/src/tenant/metadata/field-metadata/field-metadata.entity.ts @@ -0,0 +1,42 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm'; + +import { ObjectMetadata } from 'src/tenant/metadata/object-metadata/object-metadata.entity'; + +@Entity('field_metadata') +export class FieldMetadata { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: false, name: 'object_id' }) + objectId: string; + + @Column({ nullable: false }) + type: string; + + @Column({ nullable: false }) + name: string; + + @Column({ default: false, name: 'is_custom' }) + isCustom: boolean; + + @Column({ nullable: false, name: 'workspace_id' }) + workspaceId: string; + + @ManyToOne(() => ObjectMetadata, (object) => object.fields) + @JoinColumn({ name: 'object_id' }) + object: ObjectMetadata; + + @CreateDateColumn({ name: 'created_at' }) + createdAt: Date; + + @UpdateDateColumn({ name: 'updated_at' }) + updatedAt: Date; +} diff --git a/server/src/tenant/metadata/field-metadata/field-metadata.module.ts b/server/src/tenant/metadata/field-metadata/field-metadata.module.ts new file mode 100644 index 000000000..f7692c037 --- /dev/null +++ b/server/src/tenant/metadata/field-metadata/field-metadata.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { FieldMetadataService } from './field-metadata.service'; +import { FieldMetadata } from './field-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([FieldMetadata], 'metadata')], + providers: [FieldMetadataService], + exports: [FieldMetadataService], +}) +export class FieldMetadataModule {} diff --git a/server/src/tenant/metadata/field-metadata/field-metadata.service.spec.ts b/server/src/tenant/metadata/field-metadata/field-metadata.service.spec.ts new file mode 100644 index 000000000..4cc103d8c --- /dev/null +++ b/server/src/tenant/metadata/field-metadata/field-metadata.service.spec.ts @@ -0,0 +1,27 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; + +import { FieldMetadataService } from './field-metadata.service'; +import { FieldMetadata } from './field-metadata.entity'; + +describe('FieldMetadataService', () => { + let service: FieldMetadataService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + FieldMetadataService, + { + provide: getRepositoryToken(FieldMetadata, 'metadata'), + useValue: {}, + }, + ], + }).compile(); + + service = module.get(FieldMetadataService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/field-metadata/field-metadata.service.ts b/server/src/tenant/metadata/field-metadata/field-metadata.service.ts new file mode 100644 index 000000000..12b1d9b0f --- /dev/null +++ b/server/src/tenant/metadata/field-metadata/field-metadata.service.ts @@ -0,0 +1,14 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { FieldMetadata } from './field-metadata.entity'; + +@Injectable() +export class FieldMetadataService { + constructor( + @InjectRepository(FieldMetadata, 'metadata') + private readonly fieldMetadataRepository: Repository, + ) {} +} diff --git a/server/src/tenant/metadata/metadata.controller.spec.ts b/server/src/tenant/metadata/metadata.controller.spec.ts new file mode 100644 index 000000000..5b0503d76 --- /dev/null +++ b/server/src/tenant/metadata/metadata.controller.spec.ts @@ -0,0 +1,37 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { MetadataController } from './metadata.controller'; + +import { DataSourceService } from './data-source/data-source.service'; +import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service'; +import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service'; + +describe('MetadataController', () => { + let controller: MetadataController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [MetadataController], + providers: [ + { + provide: DataSourceService, + useValue: {}, + }, + { + provide: DataSourceMetadataService, + useValue: {}, + }, + { + provide: EntitySchemaGeneratorService, + useValue: {}, + }, + ], + }).compile(); + + controller = module.get(MetadataController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/metadata.controller.ts b/server/src/tenant/metadata/metadata.controller.ts new file mode 100644 index 000000000..514f9de50 --- /dev/null +++ b/server/src/tenant/metadata/metadata.controller.ts @@ -0,0 +1,46 @@ +import { Controller, Get, UseGuards } from '@nestjs/common'; + +import { Workspace } from '@prisma/client'; +import { EntitySchema } from 'typeorm'; + +import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator'; +import { JwtAuthGuard } from 'src/guards/jwt.auth.guard'; + +import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service'; +import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service'; +import { DataSourceService } from './data-source/data-source.service'; + +@UseGuards(JwtAuthGuard) +@Controller('metadata') +export class MetadataController { + constructor( + private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService, + private readonly dataSourceMetadataService: DataSourceMetadataService, + private readonly dataSourceService: DataSourceService, + ) {} + + @Get() + async getMetadata(@AuthWorkspace() workspace: Workspace) { + const dataSourcesMetadata = + await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId( + workspace.id, + ); + + const entities: EntitySchema<{ + id: unknown; + }>[] = []; + + for (const dataSource of dataSourcesMetadata) { + const dataSourceEntities = + await this.entitySchemaGeneratorService.getTypeORMEntitiesByDataSourceId( + dataSource.id, + ); + + entities.push(...dataSourceEntities); + } + + this.dataSourceService.connectToWorkspaceDataSource(workspace.id); + + return entities; + } +} diff --git a/server/src/tenant/metadata/metadata.module.ts b/server/src/tenant/metadata/metadata.module.ts new file mode 100644 index 000000000..3bf9540b5 --- /dev/null +++ b/server/src/tenant/metadata/metadata.module.ts @@ -0,0 +1,46 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule, TypeOrmModuleOptions } from '@nestjs/typeorm'; + +import { EnvironmentService } from 'src/integrations/environment/environment.service'; + +import { MetadataService } from './metadata.service'; +import { MetadataController } from './metadata.controller'; + +import { DataSourceModule } from './data-source/data-source.module'; +import { DataSourceMetadataModule } from './data-source-metadata/data-source-metadata.module'; +import { FieldMetadataModule } from './field-metadata/field-metadata.module'; +import { ObjectMetadataModule } from './object-metadata/object-metadata.module'; +import { EntitySchemaGeneratorModule } from './entity-schema-generator/entity-schema-generator.module'; +import { DataSourceMetadata } from './data-source-metadata/data-source-metadata.entity'; +import { FieldMetadata } from './field-metadata/field-metadata.entity'; +import { ObjectMetadata } from './object-metadata/object-metadata.entity'; + +const typeORMFactory = async ( + environmentService: EnvironmentService, +): Promise => ({ + url: environmentService.getPGDatabaseUrl(), + type: 'postgres', + logging: false, + schema: 'metadata', + entities: [DataSourceMetadata, FieldMetadata, ObjectMetadata], + synchronize: true, +}); + +@Module({ + imports: [ + TypeOrmModule.forRootAsync({ + useFactory: typeORMFactory, + inject: [EnvironmentService], + name: 'metadata', + }), + DataSourceModule, + DataSourceMetadataModule, + FieldMetadataModule, + ObjectMetadataModule, + EntitySchemaGeneratorModule, + ], + providers: [MetadataService], + exports: [MetadataService], + controllers: [MetadataController], +}) +export class MetadataModule {} diff --git a/server/src/tenant/metadata/metadata.service.spec.ts b/server/src/tenant/metadata/metadata.service.spec.ts new file mode 100644 index 000000000..458996d71 --- /dev/null +++ b/server/src/tenant/metadata/metadata.service.spec.ts @@ -0,0 +1,19 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { MetadataService } from './metadata.service'; + +describe('MetadataService', () => { + let service: MetadataService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [MetadataService], + }).compile(); + + service = module.get(MetadataService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/metadata.service.ts b/server/src/tenant/metadata/metadata.service.ts new file mode 100644 index 000000000..97c026450 --- /dev/null +++ b/server/src/tenant/metadata/metadata.service.ts @@ -0,0 +1,4 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class MetadataService {} diff --git a/server/src/tenant/metadata/object-metadata/object-metadata.entity.ts b/server/src/tenant/metadata/object-metadata/object-metadata.entity.ts new file mode 100644 index 000000000..22351630f --- /dev/null +++ b/server/src/tenant/metadata/object-metadata/object-metadata.entity.ts @@ -0,0 +1,37 @@ +import { + Column, + CreateDateColumn, + Entity, + OneToMany, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm'; + +import { FieldMetadata } from 'src/tenant/metadata/field-metadata/field-metadata.entity'; + +@Entity('object_metadata') +export class ObjectMetadata { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: false, name: 'data_source_id' }) + dataSourceId: string; + + @Column({ nullable: false }) + name: string; + + @Column({ default: false, name: 'is_custom' }) + isCustom: boolean; + + @Column({ nullable: false, name: 'workspace_id' }) + workspaceId: string; + + @OneToMany(() => FieldMetadata, (field) => field.object) + fields: FieldMetadata[]; + + @CreateDateColumn({ name: 'created_at' }) + createdAt: Date; + + @UpdateDateColumn({ name: 'updated_at' }) + updatedAt: Date; +} diff --git a/server/src/tenant/metadata/object-metadata/object-metadata.module.ts b/server/src/tenant/metadata/object-metadata/object-metadata.module.ts new file mode 100644 index 000000000..35bcf3187 --- /dev/null +++ b/server/src/tenant/metadata/object-metadata/object-metadata.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { ObjectMetadataService } from './object-metadata.service'; +import { ObjectMetadata } from './object-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([ObjectMetadata], 'metadata')], + providers: [ObjectMetadataService], + exports: [ObjectMetadataService], +}) +export class ObjectMetadataModule {} diff --git a/server/src/tenant/metadata/object-metadata/object-metadata.service.spec.ts b/server/src/tenant/metadata/object-metadata/object-metadata.service.spec.ts new file mode 100644 index 000000000..1f563e2c9 --- /dev/null +++ b/server/src/tenant/metadata/object-metadata/object-metadata.service.spec.ts @@ -0,0 +1,27 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; + +import { ObjectMetadataService } from './object-metadata.service'; +import { ObjectMetadata } from './object-metadata.entity'; + +describe('ObjectMetadataService', () => { + let service: ObjectMetadataService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ObjectMetadataService, + { + provide: getRepositoryToken(ObjectMetadata, 'metadata'), + useValue: {}, + }, + ], + }).compile(); + + service = module.get(ObjectMetadataService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/tenant/metadata/object-metadata/object-metadata.service.ts b/server/src/tenant/metadata/object-metadata/object-metadata.service.ts new file mode 100644 index 000000000..0ae2b03cf --- /dev/null +++ b/server/src/tenant/metadata/object-metadata/object-metadata.service.ts @@ -0,0 +1,21 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { ObjectMetadata } from './object-metadata.entity'; + +@Injectable() +export class ObjectMetadataService { + constructor( + @InjectRepository(ObjectMetadata, 'metadata') + private readonly fieldMetadataRepository: Repository, + ) {} + + public async getObjectMetadataFromDataSourceId(dataSourceId: string) { + return this.fieldMetadataRepository.find({ + where: { dataSourceId }, + relations: ['fields'], + }); + } +} diff --git a/server/src/core/tenant/tenant.module.ts b/server/src/tenant/tenant.module.ts similarity index 50% rename from server/src/core/tenant/tenant.module.ts rename to server/src/tenant/tenant.module.ts index ed960a79b..8da06930d 100644 --- a/server/src/core/tenant/tenant.module.ts +++ b/server/src/tenant/tenant.module.ts @@ -1,10 +1,8 @@ import { Module } from '@nestjs/common'; -import { DataSourceModule } from './datasource/datasource.module'; import { MetadataModule } from './metadata/metadata.module'; @Module({ - imports: [DataSourceModule, MetadataModule], - exports: [DataSourceModule], + imports: [MetadataModule], }) export class TenantModule {} diff --git a/server/yarn.lock b/server/yarn.lock index a14ce170f..0a320716b 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -1875,6 +1875,13 @@ dependencies: tslib "2.5.3" +"@nestjs/typeorm@^10.0.0": + version "10.0.0" + resolved "https://registry.yarnpkg.com/@nestjs/typeorm/-/typeorm-10.0.0.tgz#78e20d3413d59dd3dfee03260c904f0f4040b4e1" + integrity sha512-WQU4HCDTz4UavsFzvGUKDHqi0MO5K47yFoPXdmh+Z/hCNO7SHCMmV9jLiLukM8n5nKUqJ3jDqiljkWBcZPdCtA== + dependencies: + uuid "9.0.0" + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz"