admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor tenant ORM integration (#1650)

Browse Source 
* Refactor tenant ORM integration

* fix tests
This commit is contained in:
Weiko
2023-09-19 17:58:28 +02:00
committed by GitHub
parent 07684c4f08
commit ec90c77ec1
38 changed files with 747 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
server/src/core/tenant/datasource/datasource.module.ts
View File

@ -1,9 +0,0 @@
import { Module } from '@nestjs/common';
import { DataSourceService } from './services/datasource.service';
@Module({
exports: [DataSourceService],
providers: [DataSourceService],
})
export class DataSourceModule {}

73
server/src/core/tenant/datasource/entities/data-source-metadata.entity.ts
View File

@ -1,73 +0,0 @@
import {
Column,
CreateDateColumn,
Entity,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
// export const dataSourceEntity = new EntitySchema({
// name: 'data_sources',
// columns: {
// id: {
// primary: true,
// type: 'uuid',
// generated: 'uuid',
// },
// url: {
// type: 'text',
// nullable: true,
// },
// schema: {
// type: 'text',
// nullable: true,
// },
// type: {
// type: 'text',
// nullable: true,
// },
// name: {
// type: 'text',
// nullable: true,
// },
// is_remote: {
// type: 'boolean',
// nullable: true,
// default: false,
// },
// workspace_id: {
// type: 'uuid',
// nullable: true,
// },
// },
// });
@Entity('data_source_metadata')
export class DataSourceMetadata {
@PrimaryGeneratedColumn('uuid')
id: string;
@CreateDateColumn()
created_at: Date;
@UpdateDateColumn()
updated_at: Date;
@Column({ nullable: true })
url: string;
@Column({ nullable: true })
schema: string;
@Column({ default: 'postgres' })
type: string;
@Column({ nullable: true })
name: string;
@Column({ default: false })
is_remote: boolean;
@Column({ nullable: false })
workspace_id: string;
}

85
server/src/core/tenant/datasource/entities/field-metadata.entity.ts
View File

@ -1,85 +0,0 @@
import {
Column,
CreateDateColumn,
Entity,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
import { ObjectMetadata } from './object-metadata';
// export const fieldMetadataEntity = new EntitySchema({
// name: 'field_metadata',
// columns: {
// id: {
// primary: true,
// type: 'uuid',
// generated: 'uuid',
// },
// object_id: {
// type: 'uuid',
// nullable: true,
// },
// type: {
// type: 'text',
// nullable: true,
// },
// name: {
// type: 'text',
// nullable: true,
// },
// is_custom: {
// type: 'boolean',
// nullable: true,
// default: false,
// },
// workspace_id: {
// type: 'uuid',
// nullable: true,
// },
// },
// relations: {
// object: {
// type: 'many-to-one',
// target: 'object_metadata',
// joinColumn: {
// name: 'object_id',
// referencedColumnName: 'id',
// },
// inverseSide: 'fields',
// },
// } as any,
// });
@Entity('field_metadata')
export class FieldMetadata {
@PrimaryGeneratedColumn('uuid')
id: string;
@CreateDateColumn()
created_at: Date;
@UpdateDateColumn()
updated_at: Date;
@Column({ nullable: false })
object_id: string;
@Column({ nullable: false })
type: string;
@Column({ nullable: false })
name: string;
@Column({ default: false })
is_custom: boolean;
@Column({ nullable: false })
workspace_id: string;
@ManyToOne(() => ObjectMetadata, (object) => object.fields)
@JoinColumn({ name: 'object_id' })
object: ObjectMetadata;
}

72
server/src/core/tenant/datasource/entities/object-metadata.ts
View File

@ -1,72 +0,0 @@
import {
Column,
CreateDateColumn,
Entity,
OneToMany,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
import { FieldMetadata } from './field-metadata.entity';
// export const objectMetadataEntity = new EntitySchema({
// name: 'object_metadata',
// columns: {
// id: {
// primary: true,
// type: 'uuid',
// generated: 'uuid',
// },
// data_source_id: {
// type: 'uuid',
// nullable: true,
// },
// name: {
// type: 'text',
// nullable: true,
// },
// is_custom: {
// type: 'boolean',
// nullable: true,
// default: false,
// },
// workspace_id: {
// type: 'uuid',
// nullable: true,
// },
// },
// relations: {
// fields: {
// type: 'one-to-many',
// target: 'field_metadata',
// inverseSide: 'object',
// },
// } as any,
// });
@Entity('object_metadata')
export class ObjectMetadata {
@PrimaryGeneratedColumn('uuid')
id: string;
@CreateDateColumn()
created_at: Date;
@UpdateDateColumn()
updated_at: Date;
@Column({ nullable: false })
data_source_id: string;
@Column({ nullable: false })
name: string;
@Column({ default: false })
is_custom: boolean;
@Column({ nullable: false })
workspace_id: string;
@OneToMany(() => FieldMetadata, (field) => field.object)
fields: FieldMetadata[];
}

30
server/src/core/tenant/datasource/entities/relation-medata.ts
View File

@ -1,30 +0,0 @@
// export const relationMetadataEntity = new EntitySchema({
// name: 'relation_metadata',
// columns: {
// id: {
// primary: true,
// type: 'uuid',
// generated: 'uuid',
// },
// source_field_id: {
// type: 'uuid',
// nullable: true,
// },
// target_object_id: {
// type: 'uuid',
// nullable: true,
// },
// target_foreign_key: {
// type: 'uuid',
// nullable: true,
// },
// type: {
// type: 'text',
// nullable: true,
// },
// workspace_id: {
// type: 'uuid',
// nullable: true,
// },
// },
// });

210
server/src/core/tenant/datasource/services/datasource.service.ts
View File

@ -1,210 +0,0 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { DataSource, EntitySchema } from 'typeorm';
import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { DataSourceMetadata } from 'src/core/tenant/datasource/entities/data-source-metadata.entity';
import { ObjectMetadata } from 'src/core/tenant/datasource/entities/object-metadata';
import { FieldMetadata } from 'src/core/tenant/datasource/entities/field-metadata.entity';
import { baseColumns } from 'src/core/tenant/datasource/entities/base.entity';
import {
convertFieldTypeToPostgresType,
sanitizeColumnName,
uuidToBase36,
} from 'src/core/tenant/datasource/utils/datasource.util';
@Injectable()
export class DataSourceService implements OnModuleInit, OnModuleDestroy {
private mainDataSource: DataSource;
private metadataDataSource: DataSource;
private connectionOptions: PostgresConnectionOptions;
private dataSources = new Map<string, DataSource>();
constructor(environmentService: EnvironmentService) {
this.connectionOptions = {
url: environmentService.getPGDatabaseUrl(),
type: 'postgres',
logging: false,
schema: 'public',
};
this.mainDataSource = new DataSource(this.connectionOptions);
this.metadataDataSource = new DataSource({
...this.connectionOptions,
schema: 'metadata',
synchronize: true, // TODO: remove this in production
entities: [DataSourceMetadata, ObjectMetadata, FieldMetadata],
});
}
/**
*
* Returns the schema name for a given workspaceId
* @param workspaceId
* @returns string
*/
public getSchemaName(workspaceId: string): string {
return `workspace_${uuidToBase36(workspaceId)}`;
}
/**
* Creates a new schema for a given workspaceId
* @param workspaceId
* @returns Promise<void>
*/
public async createWorkspaceSchema(workspaceId: string): Promise<void> {
const schemaName = this.getSchemaName(workspaceId);
const queryRunner = this.mainDataSource.createQueryRunner();
await queryRunner.createSchema(schemaName, true);
await queryRunner.release();
await this.insertNewDataSourceMetadata(workspaceId, schemaName);
}
/**
* Inserts a new data source
* @param workspaceId
* @param workspaceSchema this can be computed from the workspaceId but it won't be the case for remote data sources
* @returns Promise<void>
*/
private async insertNewDataSourceMetadata(
workspaceId: string,
workspaceSchema: string,
): Promise<void> {
await this.metadataDataSource
?.createQueryBuilder()
.insert()
.into(DataSourceMetadata)
.values({
workspace_id: workspaceId,
schema: workspaceSchema,
})
.execute();
}
/**
* Connects to a workspace data source using the workspace metadata. Returns a cached connection if it exists.
* @param workspaceId
* @returns Promise<DataSource | undefined>
*/
public async connectToWorkspaceDataSource(
workspaceId: string,
): Promise<DataSource | undefined> {
// if (this.dataSources.has(workspaceId)) {
// const cachedDataSource = this.dataSources.get(workspaceId);
// return cachedDataSource;
// }
const dataSourceMetadata =
await this.getLastDataSourceMetadataFromWorkspaceIdOrFail(workspaceId);
const schema = dataSourceMetadata.schema;
// Probably not needed as we will ask for the schema name OR store public by default if it's remote
if (!schema && !dataSourceMetadata.is_remote) {
throw Error(
"No schema found for this non-remote data source, we don't want to fallback to public for workspace data sources.",
);
}
const metadata = await this.fetchObjectsAndFieldsFromMetadata(workspaceId);
const entities = this.convertMetadataToEntities(metadata);
const workspaceDataSource = new DataSource({
...this.connectionOptions,
schema,
entities: entities,
synchronize: true, // TODO: remove this in production
});
await workspaceDataSource.initialize();
return workspaceDataSource;
// this.dataSources.set(workspaceId, workspaceDataSource);
// return this.dataSources.get(workspaceId);
}
/**
* Returns the last data source metadata for a given workspaceId
* In the future we should handle multiple data sources.
* Most likely the twenty workspace connection and n remote connections should be fetched.
*
* @param workspaceId
* @returns Promise<DataSourceMetadata>
*/
private async getLastDataSourceMetadataFromWorkspaceIdOrFail(
workspaceId: string,
): Promise<DataSourceMetadata> {
return this.metadataDataSource
?.createQueryBuilder()
.select('data_source')
.from(DataSourceMetadata, 'data_source')
.where('data_source.workspace_id = :workspaceId', { workspaceId })
.orderBy('data_source.created_at', 'DESC')
.getOneOrFail();
}
/**
* Returns all the objects and fields for a given workspaceId and the first associated data source metadata registered.
*
* @param workspaceId
* @returns
*/
public async fetchObjectsAndFieldsFromMetadata(workspaceId: string) {
const dataSource =
await this.getLastDataSourceMetadataFromWorkspaceIdOrFail(workspaceId);
const objectRepository =
this.metadataDataSource.getRepository(ObjectMetadata);
return await objectRepository
.createQueryBuilder('object_metadata')
.select(['object_metadata.id', 'object_metadata.name'])
.leftJoinAndSelect('object_metadata.fields', 'field')
.where('object_metadata.data_source_id = :dataSourceId', {
dataSourceId: dataSource?.id,
})
.getMany();
}
/**
* Converts the metadata to entities that can be interpreted by typeorm.
* @param metadata
* @returns EntitySchema[]
*
*/
public convertMetadataToEntities(metadata): EntitySchema[] {
const entities = metadata.map((object) => {
return new EntitySchema({
name: object.name,
columns: {
...baseColumns,
...object.fields.reduce((columns, field) => {
return {
...columns,
[sanitizeColumnName(field.name)]: {
type: convertFieldTypeToPostgresType(field.type),
nullable: true,
},
};
}, {}),
},
});
});
return entities;
}
async onModuleInit() {
await this.mainDataSource.initialize();
await this.metadataDataSource.initialize();
}
async onModuleDestroy(): Promise<void> {
await this.mainDataSource.destroy();
await this.metadataDataSource.destroy();
}
}

19
server/src/core/tenant/metadata/metadata.controller.ts
View File

@ -1,19 +0,0 @@
import { Controller, Get, UseGuards } from '@nestjs/common';
import { Workspace } from '@prisma/client';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { MetadataService } from './metadata.service';
@UseGuards(JwtAuthGuard)
@Controller('metadata')
export class MetadataController {
constructor(private readonly metadataService: MetadataService) {}
@Get()
async getMetadata(@AuthWorkspace() workspace: Workspace) {
return this.metadataService.fetchMetadataFromWorkspaceId(workspace.id);
}
}

13
server/src/core/tenant/metadata/metadata.module.ts
View File

@ -1,13 +0,0 @@
import { Module } from '@nestjs/common';
import { DataSourceModule } from 'src/core/tenant/datasource/datasource.module';
import { MetadataController } from './metadata.controller';
import { MetadataService } from './metadata.service';
@Module({
imports: [DataSourceModule],
controllers: [MetadataController],
providers: [MetadataService],
})
export class MetadataModule {}

16
server/src/core/tenant/metadata/metadata.service.ts
View File

@ -1,16 +0,0 @@
import { Injectable } from '@nestjs/common';
import { DataSourceService } from 'src/core/tenant/datasource/services/datasource.service';
@Injectable()
export class MetadataService {
constructor(private readonly dataSourceService: DataSourceService) {}
public async fetchMetadataFromWorkspaceId(workspaceId: string) {
await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
return await this.dataSourceService.fetchObjectsAndFieldsFromMetadata(
workspaceId,
);
}
}

40
server/src/tenant/metadata/data-source-metadata/data-source-metadata.entity.ts Normal file
View File

@ -0,0 +1,40 @@
import {
Column,
CreateDateColumn,
Entity,
PrimaryGeneratedColumn,
UpdateDateColumn,
DataSourceOptions,
} from 'typeorm';
type DataSourceType = DataSourceOptions['type'];
@Entity('data_source_metadata')
export class DataSourceMetadata {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: true })
url: string;
@Column({ nullable: true })
schema: string;
@Column({ type: 'enum', enum: ['postgres'], default: 'postgres' })
type: DataSourceType;
@Column({ nullable: true })
name: string;
@Column({ default: false, name: 'is_remote' })
isRemote: boolean;
@Column({ nullable: false, name: 'workspace_id' })
workspaceId: string;
@CreateDateColumn({ name: 'created_at' })
createdAt: Date;
@UpdateDateColumn({ name: 'updated_at' })
updatedAt: Date;
}

12
server/src/tenant/metadata/data-source-metadata/data-source-metadata.module.ts Normal file
View File

@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { DataSourceMetadataService } from './data-source-metadata.service';
import { DataSourceMetadata } from './data-source-metadata.entity';
@Module({
imports: [TypeOrmModule.forFeature([DataSourceMetadata], 'metadata')],
providers: [DataSourceMetadataService],
exports: [DataSourceMetadataService],
})
export class DataSourceMetadataModule {}

27
server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.spec.ts Normal file
View File

@ -0,0 +1,27 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { DataSourceMetadataService } from './data-source-metadata.service';
import { DataSourceMetadata } from './data-source-metadata.entity';
describe('DataSourceMetadataService', () => {
let service: DataSourceMetadataService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
DataSourceMetadataService,
{
provide: getRepositoryToken(DataSourceMetadata, 'metadata'),
useValue: {},
},
],
}).compile();
service = module.get<DataSourceMetadataService>(DataSourceMetadataService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

37
server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts Normal file
View File

@ -0,0 +1,37 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { DataSourceMetadata } from './data-source-metadata.entity';
@Injectable()
export class DataSourceMetadataService {
constructor(
@InjectRepository(DataSourceMetadata, 'metadata')
private readonly dataSourceMetadataRepository: Repository<DataSourceMetadata>,
) {}
async createDataSourceMetadata(workspaceId: string, workspaceSchema: string) {
// TODO: Double check if this is the correct way to do this
const dataSource = await this.dataSourceMetadataRepository.findOne({
where: { workspaceId },
});
if (dataSource) {
return dataSource;
}
return this.dataSourceMetadataRepository.save({
workspaceId,
schema: workspaceSchema,
});
}
getDataSourcesMedataFromWorkspaceId(workspaceId: string) {
return this.dataSourceMetadataRepository.find({
where: { workspaceId },
order: { createdAt: 'DESC' },
});
}
}

13
server/src/tenant/metadata/data-source/data-source.module.ts Normal file
View File

@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { DataSourceMetadataModule } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.module';
import { EntitySchemaGeneratorModule } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.module';
import { DataSourceService } from './data-source.service';
@Module({
imports: [DataSourceMetadataModule, EntitySchemaGeneratorModule],
providers: [DataSourceService],
exports: [DataSourceService],
})
export class DataSourceModule {}

39
server/src/tenant/metadata/data-source/data-source.service.spec.ts Normal file
View File

@ -0,0 +1,39 @@
import { Test, TestingModule } from '@nestjs/testing';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { DataSourceMetadataService } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.service';
import { DataSourceService } from './data-source.service';
describe('DataSourceService', () => {
let service: DataSourceService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
DataSourceService,
{
provide: EnvironmentService,
useValue: {
getPGDatabaseUrl: () => '',
},
},
{
provide: DataSourceMetadataService,
useValue: {},
},
{
provide: EntitySchemaGeneratorService,
useValue: {},
},
],
}).compile();
service = module.get<DataSourceService>(DataSourceService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

127
server/src/tenant/metadata/data-source/data-source.service.ts Normal file
View File

@ -0,0 +1,127 @@
import {
Injectable,
NotFoundException,
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { DataSource } from 'typeorm';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { DataSourceMetadataService } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.service';
import { uuidToBase36 } from './data-source.util';
@Injectable()
export class DataSourceService implements OnModuleInit, OnModuleDestroy {
private mainDataSource: DataSource;
constructor(
private readonly environmentService: EnvironmentService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService,
) {
this.mainDataSource = new DataSource({
url: environmentService.getPGDatabaseUrl(),
type: 'postgres',
logging: false,
schema: 'public',
});
}
/**
* Creates a new schema for a given workspaceId
* @param workspaceId
* @returns Promise<void>
*/
public async createWorkspaceSchema(workspaceId: string): Promise<void> {
const schemaName = this.getSchemaName(workspaceId);
const queryRunner = this.mainDataSource.createQueryRunner();
await queryRunner.createSchema(schemaName, true);
await queryRunner.release();
await this.dataSourceMetadataService.createDataSourceMetadata(
workspaceId,
schemaName,
);
}
/**
* Connects to a workspace data source using the workspace metadata. Returns a cached connection if it exists.
* @param workspaceId
* @returns Promise<DataSource | undefined>
*/
public async connectToWorkspaceDataSource(
workspaceId: string,
): Promise<DataSource | undefined> {
// if (this.dataSources.has(workspaceId)) {
// const cachedDataSource = this.dataSources.get(workspaceId);
// return cachedDataSource;
// }
const dataSourcesMetadata =
await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId(
workspaceId,
);
if (dataSourcesMetadata.length === 0) {
throw new NotFoundException(
`We can't find any data source for this workspace id (${workspaceId}).`,
);
}
const dataSourceMetadata = dataSourcesMetadata[0];
const schema = dataSourceMetadata.schema;
// Probably not needed as we will ask for the schema name OR store public by default if it's remote
if (!schema && !dataSourceMetadata.isRemote) {
throw Error(
"No schema found for this non-remote data source, we don't want to fallback to public for workspace data sources.",
);
}
const entities =
await this.entitySchemaGeneratorService.getTypeORMEntitiesByDataSourceId(
dataSourceMetadata.id,
);
const workspaceDataSource = new DataSource({
// TODO: We should use later dataSourceMetadata.type and use a switch case condition to create the right data source
url: dataSourceMetadata.url ?? this.environmentService.getPGDatabaseUrl(),
type: 'postgres',
logging: ['query'],
schema,
entities: entities,
synchronize: true, // TODO: remove this in production
});
await workspaceDataSource.initialize();
return workspaceDataSource;
// this.dataSources.set(workspaceId, workspaceDataSource);
// return this.dataSources.get(workspaceId);
}
/**
*
* Returns the schema name for a given workspaceId
* @param workspaceId
* @returns string
*/
private getSchemaName(workspaceId: string): string {
return `workspace_${uuidToBase36(workspaceId)}`;
}
async onModuleInit() {
// Init main data source "default" schema
await this.mainDataSource.initialize();
}
async onModuleDestroy() {
// Destroy main data source "default" schema
await this.mainDataSource.destroy();
}
}

14
server/src/tenant/metadata/data-source/data-source.util.ts Normal file
View File

@ -0,0 +1,14 @@
/**
* Converts a UUID to a base 36 string.
* This is used to generate the schema name since hyphens from workspace uuid are not allowed in postgres schema names.
*
* @param uuid
* @returns
*/
export function uuidToBase36(uuid: string): string {
const hexString = uuid.replace(/-/g, '');
const base10Number = BigInt('0x' + hexString);
const base36String = base10Number.toString(36);
return base36String;
}

2
server/src/core/tenant/datasource/entities/base.entity.ts → server/src/tenant/metadata/entity-schema-generator/base.entity.ts
View File

@ -4,4 +4,4 @@ export const baseColumns = {
type: 'uuid',
generated: 'uuid',
},
};
} as const;

12
server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.module.ts Normal file
View File

@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { ObjectMetadataModule } from 'src/tenant/metadata/object-metadata/object-metadata.module';
import { EntitySchemaGeneratorService } from './entity-schema-generator.service';
@Module({
imports: [ObjectMetadataModule],
providers: [EntitySchemaGeneratorService],
exports: [EntitySchemaGeneratorService],
})
export class EntitySchemaGeneratorModule {}

29
server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.spec.ts Normal file
View File

@ -0,0 +1,29 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ObjectMetadataService } from 'src/tenant/metadata/object-metadata/object-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator.service';
describe('EntitySchemaGeneratorService', () => {
let service: EntitySchemaGeneratorService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
EntitySchemaGeneratorService,
{
provide: ObjectMetadataService,
useValue: {},
},
],
}).compile();
service = module.get<EntitySchemaGeneratorService>(
EntitySchemaGeneratorService,
);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

43
server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.service.ts Normal file
View File

@ -0,0 +1,43 @@
import { Injectable } from '@nestjs/common';
import { EntitySchema } from 'typeorm';
import { ObjectMetadataService } from 'src/tenant/metadata/object-metadata/object-metadata.service';
import { baseColumns } from './base.entity';
import {
convertFieldTypeToPostgresType,
sanitizeColumnName,
} from './entity-schema-generator.util';
@Injectable()
export class EntitySchemaGeneratorService {
constructor(private readonly objectMetadataService: ObjectMetadataService) {}
async getTypeORMEntitiesByDataSourceId(dataSourceId: string) {
const objectMetadata =
await this.objectMetadataService.getObjectMetadataFromDataSourceId(
dataSourceId,
);
const entities = objectMetadata.map((object) => {
return new EntitySchema({
name: object.name,
columns: {
...baseColumns,
...object.fields.reduce((columns, field) => {
return {
...columns,
[sanitizeColumnName(field.name)]: {
type: convertFieldTypeToPostgresType(field.type),
nullable: true,
},
};
}, {}),
},
});
});
return entities;
}
}

0
server/src/core/tenant/datasource/utils/datasource.util.ts → server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.util.ts
View File

42
server/src/tenant/metadata/field-metadata/field-metadata.entity.ts Normal file
View File

@ -0,0 +1,42 @@
import {
Column,
CreateDateColumn,
Entity,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
import { ObjectMetadata } from 'src/tenant/metadata/object-metadata/object-metadata.entity';
@Entity('field_metadata')
export class FieldMetadata {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: false, name: 'object_id' })
objectId: string;
@Column({ nullable: false })
type: string;
@Column({ nullable: false })
name: string;
@Column({ default: false, name: 'is_custom' })
isCustom: boolean;
@Column({ nullable: false, name: 'workspace_id' })
workspaceId: string;
@ManyToOne(() => ObjectMetadata, (object) => object.fields)
@JoinColumn({ name: 'object_id' })
object: ObjectMetadata;
@CreateDateColumn({ name: 'created_at' })
createdAt: Date;
@UpdateDateColumn({ name: 'updated_at' })
updatedAt: Date;
}

12
server/src/tenant/metadata/field-metadata/field-metadata.module.ts Normal file
View File

@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FieldMetadataService } from './field-metadata.service';
import { FieldMetadata } from './field-metadata.entity';
@Module({
imports: [TypeOrmModule.forFeature([FieldMetadata], 'metadata')],
providers: [FieldMetadataService],
exports: [FieldMetadataService],
})
export class FieldMetadataModule {}

27
server/src/tenant/metadata/field-metadata/field-metadata.service.spec.ts Normal file
View File

@ -0,0 +1,27 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { FieldMetadataService } from './field-metadata.service';
import { FieldMetadata } from './field-metadata.entity';
describe('FieldMetadataService', () => {
let service: FieldMetadataService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
FieldMetadataService,
{
provide: getRepositoryToken(FieldMetadata, 'metadata'),
useValue: {},
},
],
}).compile();
service = module.get<FieldMetadataService>(FieldMetadataService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

14
server/src/tenant/metadata/field-metadata/field-metadata.service.ts Normal file
View File

@ -0,0 +1,14 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { FieldMetadata } from './field-metadata.entity';
@Injectable()
export class FieldMetadataService {
constructor(
@InjectRepository(FieldMetadata, 'metadata')
private readonly fieldMetadataRepository: Repository<FieldMetadata>,
) {}
}

37
server/src/tenant/metadata/metadata.controller.spec.ts Normal file
View File

@ -0,0 +1,37 @@
import { Test, TestingModule } from '@nestjs/testing';
import { MetadataController } from './metadata.controller';
import { DataSourceService } from './data-source/data-source.service';
import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service';
describe('MetadataController', () => {
let controller: MetadataController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [MetadataController],
providers: [
{
provide: DataSourceService,
useValue: {},
},
{
provide: DataSourceMetadataService,
useValue: {},
},
{
provide: EntitySchemaGeneratorService,
useValue: {},
},
],
}).compile();
controller = module.get<MetadataController>(MetadataController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

46
server/src/tenant/metadata/metadata.controller.ts Normal file
View File

@ -0,0 +1,46 @@
import { Controller, Get, UseGuards } from '@nestjs/common';
import { Workspace } from '@prisma/client';
import { EntitySchema } from 'typeorm';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service';
import { DataSourceService } from './data-source/data-source.service';
@UseGuards(JwtAuthGuard)
@Controller('metadata')
export class MetadataController {
constructor(
private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
private readonly dataSourceService: DataSourceService,
) {}
@Get()
async getMetadata(@AuthWorkspace() workspace: Workspace) {
const dataSourcesMetadata =
await this.dataSourceMetadataService.getDataSourcesMedataFromWorkspaceId(
workspace.id,
);
const entities: EntitySchema<{
id: unknown;
}>[] = [];
for (const dataSource of dataSourcesMetadata) {
const dataSourceEntities =
await this.entitySchemaGeneratorService.getTypeORMEntitiesByDataSourceId(
dataSource.id,
);
entities.push(...dataSourceEntities);
}
this.dataSourceService.connectToWorkspaceDataSource(workspace.id);
return entities;
}
}

46
server/src/tenant/metadata/metadata.module.ts Normal file
View File

@ -0,0 +1,46 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule, TypeOrmModuleOptions } from '@nestjs/typeorm';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { MetadataService } from './metadata.service';
import { MetadataController } from './metadata.controller';
import { DataSourceModule } from './data-source/data-source.module';
import { DataSourceMetadataModule } from './data-source-metadata/data-source-metadata.module';
import { FieldMetadataModule } from './field-metadata/field-metadata.module';
import { ObjectMetadataModule } from './object-metadata/object-metadata.module';
import { EntitySchemaGeneratorModule } from './entity-schema-generator/entity-schema-generator.module';
import { DataSourceMetadata } from './data-source-metadata/data-source-metadata.entity';
import { FieldMetadata } from './field-metadata/field-metadata.entity';
import { ObjectMetadata } from './object-metadata/object-metadata.entity';
const typeORMFactory = async (
environmentService: EnvironmentService,
): Promise<TypeOrmModuleOptions> => ({
url: environmentService.getPGDatabaseUrl(),
type: 'postgres',
logging: false,
schema: 'metadata',
entities: [DataSourceMetadata, FieldMetadata, ObjectMetadata],
synchronize: true,
});
@Module({
imports: [
TypeOrmModule.forRootAsync({
useFactory: typeORMFactory,
inject: [EnvironmentService],
name: 'metadata',
}),
DataSourceModule,
DataSourceMetadataModule,
FieldMetadataModule,
ObjectMetadataModule,
EntitySchemaGeneratorModule,
],
providers: [MetadataService],
exports: [MetadataService],
controllers: [MetadataController],
})
export class MetadataModule {}

19
server/src/tenant/metadata/metadata.service.spec.ts Normal file
View File

@ -0,0 +1,19 @@
import { Test, TestingModule } from '@nestjs/testing';
import { MetadataService } from './metadata.service';
describe('MetadataService', () => {
let service: MetadataService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [MetadataService],
}).compile();
service = module.get<MetadataService>(MetadataService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

4
server/src/tenant/metadata/metadata.service.ts Normal file
View File

@ -0,0 +1,4 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class MetadataService {}

37
server/src/tenant/metadata/object-metadata/object-metadata.entity.ts Normal file
View File

@ -0,0 +1,37 @@
import {
Column,
CreateDateColumn,
Entity,
OneToMany,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
import { FieldMetadata } from 'src/tenant/metadata/field-metadata/field-metadata.entity';
@Entity('object_metadata')
export class ObjectMetadata {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ nullable: false, name: 'data_source_id' })
dataSourceId: string;
@Column({ nullable: false })
name: string;
@Column({ default: false, name: 'is_custom' })
isCustom: boolean;
@Column({ nullable: false, name: 'workspace_id' })
workspaceId: string;
@OneToMany(() => FieldMetadata, (field) => field.object)
fields: FieldMetadata[];
@CreateDateColumn({ name: 'created_at' })
createdAt: Date;
@UpdateDateColumn({ name: 'updated_at' })
updatedAt: Date;
}

12
server/src/tenant/metadata/object-metadata/object-metadata.module.ts Normal file
View File

@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ObjectMetadataService } from './object-metadata.service';
import { ObjectMetadata } from './object-metadata.entity';
@Module({
imports: [TypeOrmModule.forFeature([ObjectMetadata], 'metadata')],
providers: [ObjectMetadataService],
exports: [ObjectMetadataService],
})
export class ObjectMetadataModule {}

27
server/src/tenant/metadata/object-metadata/object-metadata.service.spec.ts Normal file
View File

@ -0,0 +1,27 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { ObjectMetadataService } from './object-metadata.service';
import { ObjectMetadata } from './object-metadata.entity';
describe('ObjectMetadataService', () => {
let service: ObjectMetadataService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
ObjectMetadataService,
{
provide: getRepositoryToken(ObjectMetadata, 'metadata'),
useValue: {},
},
],
}).compile();
service = module.get<ObjectMetadataService>(ObjectMetadataService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

21
server/src/tenant/metadata/object-metadata/object-metadata.service.ts Normal file
View File

@ -0,0 +1,21 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ObjectMetadata } from './object-metadata.entity';
@Injectable()
export class ObjectMetadataService {
constructor(
@InjectRepository(ObjectMetadata, 'metadata')
private readonly fieldMetadataRepository: Repository<ObjectMetadata>,
) {}
public async getObjectMetadataFromDataSourceId(dataSourceId: string) {
return this.fieldMetadataRepository.find({
where: { dataSourceId },
relations: ['fields'],
});
}
}

4
server/src/core/tenant/tenant.module.ts → server/src/tenant/tenant.module.ts
View File

@ -1,10 +1,8 @@
import { Module } from '@nestjs/common';
import { DataSourceModule } from './datasource/datasource.module';
import { MetadataModule } from './metadata/metadata.module';
@Module({
imports: [DataSourceModule, MetadataModule],
exports: [DataSourceModule],
imports: [MetadataModule],
})
export class TenantModule {}