Simplify ORM (#6373)

This PR refactors the ORM-Manager to simplify and unify the datasource
creation. I'm deprecating all usages if InjectWorkspaceDatasource and
InjectWorkspaceRepository as we can't be sure they are up-to-date
This commit is contained in:
Charles Bochet
2024-07-23 14:13:16 +02:00
committed by GitHub
parent 794e73eeb6
commit 6c34ef9a14
23 changed files with 205 additions and 1672 deletions

View File

@ -1,7 +0,0 @@
import { Inject } from '@nestjs/common';
import { TWENTY_ORM_WORKSPACE_DATASOURCE } from 'src/engine/twenty-orm/twenty-orm.constants';
// nit: The datasource can be null if it's used outside of an authenticated request context
export const InjectWorkspaceDatasource = () =>
Inject(TWENTY_ORM_WORKSPACE_DATASOURCE);

View File

@ -1,9 +0,0 @@
import { Inject } from '@nestjs/common';
import { EntityClassOrSchema } from '@nestjs/typeorm/dist/interfaces/entity-class-or-schema.type';
import { getWorkspaceRepositoryToken } from 'src/engine/twenty-orm/utils/get-workspace-repository-token.util';
// nit: The repository can be null if it's used outside of an authenticated request context
export const InjectWorkspaceRepository = (
entity: EntityClassOrSchema,
): ReturnType<typeof Inject> => Inject(getWorkspaceRepositoryToken(entity));

View File

@ -1,10 +1,15 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { EntitySchema } from 'typeorm';
import { Repository } from 'typeorm';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { workspaceDataSourceCacheInstance } from 'src/engine/twenty-orm/twenty-orm-core.module';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
@ -13,45 +18,111 @@ export class WorkspaceDatasourceFactory {
private readonly dataSourceService: DataSourceService,
private readonly environmentService: EnvironmentService,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly entitySchemaFactory: EntitySchemaFactory,
) {}
public async create(
entities: EntitySchema[],
workspaceId: string,
): Promise<WorkspaceDataSource | null> {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
workspaceId,
);
cacheVersion: string | null,
): Promise<WorkspaceDataSource> {
let dataSourceCacheVersion: string;
if (!dataSourceMetadata) {
return null;
if (cacheVersion) {
dataSourceCacheVersion = cacheVersion;
} else {
const cacheVersionFromDatabase =
await this.workspaceCacheVersionService.getVersion(workspaceId);
if (!cacheVersionFromDatabase) {
throw new Error('Cache version not found');
}
dataSourceCacheVersion = cacheVersionFromDatabase;
}
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
workspaceCacheStorage: this.workspaceCacheStorageService,
},
{
url:
dataSourceMetadata.url ??
this.environmentService.get('PG_DATABASE_URL'),
type: 'postgres',
logging: this.environmentService.get('DEBUG_MODE')
? ['query', 'error']
: ['error'],
schema: dataSourceMetadata.schema,
entities,
ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED')
? {
rejectUnauthorized: false,
}
: undefined,
const workspaceDataSource = await workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${dataSourceCacheVersion}`,
async () => {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
workspaceId,
);
if (!dataSourceMetadata) {
throw new Error('Data source metadata not found');
}
const latestCacheVersion =
await this.workspaceCacheVersionService.getVersion(workspaceId);
if (latestCacheVersion !== dataSourceCacheVersion) {
throw new Error('Cache version mismatch');
}
let objectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!objectMetadataCollection) {
objectMetadataCollection = await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
objectMetadataCollection,
);
}
const entities = await Promise.all(
objectMetadataCollection.map((objectMetadata) =>
this.entitySchemaFactory.create(workspaceId, objectMetadata),
),
);
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
workspaceCacheStorage: this.workspaceCacheStorageService,
},
{
url:
dataSourceMetadata.url ??
this.environmentService.get('PG_DATABASE_URL'),
type: 'postgres',
logging: this.environmentService.get('DEBUG_MODE')
? ['query', 'error']
: ['error'],
schema: dataSourceMetadata.schema,
entities,
ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED')
? {
rejectUnauthorized: false,
}
: undefined,
},
);
await workspaceDataSource.initialize();
return workspaceDataSource;
},
(dataSource) => dataSource.destroy(),
);
await workspaceDataSource.initialize();
if (!workspaceDataSource) {
throw new Error('Workspace data source not found');
}
return workspaceDataSource;
}

View File

@ -4,11 +4,8 @@ import {
Logger,
Module,
OnApplicationShutdown,
Provider,
} from '@nestjs/common';
import { TypeOrmModule, getRepositoryToken } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { TypeOrmModule } from '@nestjs/typeorm';
import {
TwentyORMModuleAsyncOptions,
@ -22,18 +19,11 @@ import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/l
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { entitySchemaFactories } from 'src/engine/twenty-orm/factories';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
import { CacheManager } from 'src/engine/twenty-orm/storage/cache-manager.storage';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { TWENTY_ORM_WORKSPACE_DATASOURCE } from 'src/engine/twenty-orm/twenty-orm.constants';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {
ConfigurableModuleClass,
MODULE_OPTIONS_TOKEN,
} from 'src/engine/twenty-orm/twenty-orm.module-definition';
import { ConfigurableModuleClass } from 'src/engine/twenty-orm/twenty-orm.module-definition';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
export const workspaceDataSourceCacheInstance =
new CacheManager<WorkspaceDataSource>();
@ -68,28 +58,10 @@ export class TwentyORMCoreModule
static register(options: TwentyORMOptions): DynamicModule {
const dynamicModule = super.register(options);
// TODO: Avoid code duplication here
const providers: Provider[] = [
{
provide: TWENTY_ORM_WORKSPACE_DATASOURCE,
useFactory: this.createWorkspaceDataSource,
inject: [
WorkspaceCacheStorageService,
getRepositoryToken(ObjectMetadataEntity, 'metadata'),
EntitySchemaFactory,
ScopedWorkspaceContextFactory,
WorkspaceDatasourceFactory,
],
},
];
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), ...providers],
exports: [
...(dynamicModule.exports ?? []),
TWENTY_ORM_WORKSPACE_DATASOURCE,
],
providers: [...(dynamicModule.providers ?? [])],
exports: [...(dynamicModule.exports ?? [])],
};
}
@ -97,89 +69,14 @@ export class TwentyORMCoreModule
asyncOptions: TwentyORMModuleAsyncOptions,
): DynamicModule {
const dynamicModule = super.registerAsync(asyncOptions);
const providers: Provider[] = [
{
provide: TWENTY_ORM_WORKSPACE_DATASOURCE,
useFactory: this.createWorkspaceDataSource,
inject: [
WorkspaceCacheStorageService,
getRepositoryToken(ObjectMetadataEntity, 'metadata'),
EntitySchemaFactory,
ScopedWorkspaceContextFactory,
WorkspaceDatasourceFactory,
MODULE_OPTIONS_TOKEN,
],
},
];
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), ...providers],
exports: [
...(dynamicModule.exports ?? []),
TWENTY_ORM_WORKSPACE_DATASOURCE,
],
providers: [...(dynamicModule.providers ?? [])],
exports: [...(dynamicModule.exports ?? [])],
};
}
static async createWorkspaceDataSource(
workspaceCacheStorageService: WorkspaceCacheStorageService,
objectMetadataRepository: Repository<ObjectMetadataEntity>,
entitySchemaFactory: EntitySchemaFactory,
scopedWorkspaceContextFactory: ScopedWorkspaceContextFactory,
workspaceDataSourceFactory: WorkspaceDatasourceFactory,
_options?: TwentyORMOptions,
) {
const { workspaceId, cacheVersion } =
scopedWorkspaceContextFactory.create();
if (!workspaceId) {
return null;
}
return workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${cacheVersion}`,
async () => {
let objectMetadataCollection =
await workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!objectMetadataCollection) {
objectMetadataCollection = await objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
objectMetadataCollection,
);
}
const entities = await Promise.all(
objectMetadataCollection.map((objectMetadata) =>
entitySchemaFactory.create(workspaceId, objectMetadata),
),
);
const workspaceDataSource = await workspaceDataSourceFactory.create(
entities,
workspaceId,
);
return workspaceDataSource;
},
(dataSource) => dataSource.destroy(),
);
}
/**
* Destroys all data sources on application shutdown
*/

View File

@ -1,114 +1,29 @@
import { Injectable, Type } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Injectable } from '@nestjs/common';
import { ObjectLiteral, Repository } from 'typeorm';
import { ObjectLiteral } from 'typeorm';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { CustomWorkspaceEntity } from 'src/engine/twenty-orm/custom.workspace-entity';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { workspaceDataSourceCacheInstance } from 'src/engine/twenty-orm/twenty-orm-core.module';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util';
@Injectable()
export class TwentyORMGlobalManager {
constructor(
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceDataSourceFactory: WorkspaceDatasourceFactory,
private readonly entitySchemaFactory: EntitySchemaFactory,
) {}
async getRepositoryForWorkspace<T extends ObjectLiteral>(
workspaceId: string,
entityClass: Type<T>,
): Promise<WorkspaceRepository<T>>;
async getRepositoryForWorkspace(
workspaceId: string,
objectMetadataName: string,
): Promise<WorkspaceRepository<CustomWorkspaceEntity>>;
async getRepositoryForWorkspace<T extends ObjectLiteral>(
workspaceId: string,
entityClassOrobjectMetadataName: Type<T> | string,
): Promise<
WorkspaceRepository<T> | WorkspaceRepository<CustomWorkspaceEntity>
> {
let objectMetadataName: string;
if (typeof entityClassOrobjectMetadataName === 'string') {
objectMetadataName = entityClassOrobjectMetadataName;
} else {
objectMetadataName = convertClassNameToObjectMetadataName(
entityClassOrobjectMetadataName.name,
);
}
return this.buildRepositoryForWorkspace<T>(workspaceId, objectMetadataName);
}
async buildDatasourceForWorkspace(workspaceId: string) {
const cacheVersion =
await this.workspaceCacheVersionService.getVersion(workspaceId);
let objectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!objectMetadataCollection) {
objectMetadataCollection = await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
objectMetadataCollection,
);
}
const entities = await Promise.all(
objectMetadataCollection.map((objectMetadata) =>
this.entitySchemaFactory.create(workspaceId, objectMetadata),
),
): Promise<WorkspaceRepository<T>> {
const workspaceDataSource = await this.workspaceDataSourceFactory.create(
workspaceId,
null,
);
return await workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${cacheVersion}`,
async () => {
const workspaceDataSource =
await this.workspaceDataSourceFactory.create(entities, workspaceId);
return workspaceDataSource;
},
(dataSource) => dataSource.destroy(),
);
}
async buildRepositoryForWorkspace<T extends ObjectLiteral>(
workspaceId: string,
objectMetadataName: string,
) {
const workspaceDataSource =
await this.buildDatasourceForWorkspace(workspaceId);
if (!workspaceDataSource) {
throw new Error('Workspace data source not found');
}
return workspaceDataSource.getRepository<T>(objectMetadataName);
}
async getDataSourceForWorkspace(workspaceId: string) {
return this.workspaceDataSourceFactory.create(workspaceId, null);
}
}

View File

@ -1,2 +0,0 @@
export const TWENTY_ORM_WORKSPACE_DATASOURCE =
'TWENTY_ORM_WORKSPACE_DATASOURCE';

View File

@ -1,159 +1,44 @@
import { Injectable, Optional, Type } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Injectable } from '@nestjs/common';
import { ObjectLiteral, Repository } from 'typeorm';
import { ObjectLiteral } from 'typeorm';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { CustomWorkspaceEntity } from 'src/engine/twenty-orm/custom.workspace-entity';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { InjectWorkspaceDatasource } from 'src/engine/twenty-orm/decorators/inject-workspace-datasource.decorator';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { workspaceDataSourceCacheInstance } from 'src/engine/twenty-orm/twenty-orm-core.module';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util';
@Injectable()
export class TwentyORMManager {
constructor(
@Optional()
@InjectWorkspaceDatasource()
private readonly workspaceDataSource: WorkspaceDataSource | null,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceDataSourceFactory: WorkspaceDatasourceFactory,
private readonly entitySchemaFactory: EntitySchemaFactory,
private readonly scopedWorkspaceContextFactory: ScopedWorkspaceContextFactory,
) {}
async getRepository<T extends ObjectLiteral>(
objectMetadataName: string,
): Promise<WorkspaceRepository<T>>;
async getRepository<T extends ObjectLiteral>(
entityClass: Type<T>,
): Promise<WorkspaceRepository<T>>;
async getRepository<T extends ObjectLiteral>(
entityClassOrobjectMetadataName: Type<T> | string,
): Promise<WorkspaceRepository<T>> {
let objectMetadataName: string;
const { workspaceId, cacheVersion } =
this.scopedWorkspaceContextFactory.create();
if (typeof entityClassOrobjectMetadataName === 'string') {
objectMetadataName = entityClassOrobjectMetadataName;
} else {
objectMetadataName = convertClassNameToObjectMetadataName(
entityClassOrobjectMetadataName.name,
);
if (!workspaceId) {
throw new Error('Workspace not found');
}
if (!this.workspaceDataSource) {
throw new Error('Workspace data source not found');
}
const workspaceId = this.workspaceDataSource.getWorkspaceId();
return this.buildRepositoryForWorkspace<T>(workspaceId, objectMetadataName);
}
async getRepositoryForWorkspace<T extends ObjectLiteral>(
workspaceId: string,
entityClass: Type<T>,
): Promise<WorkspaceRepository<T>>;
async getRepositoryForWorkspace(
workspaceId: string,
objectMetadataName: string,
): Promise<WorkspaceRepository<CustomWorkspaceEntity>>;
async getRepositoryForWorkspace<T extends ObjectLiteral>(
workspaceId: string,
entityClassOrobjectMetadataName: Type<T> | string,
): Promise<
WorkspaceRepository<T> | WorkspaceRepository<CustomWorkspaceEntity>
> {
let objectMetadataName: string;
if (typeof entityClassOrobjectMetadataName === 'string') {
objectMetadataName = entityClassOrobjectMetadataName;
} else {
objectMetadataName = convertClassNameToObjectMetadataName(
entityClassOrobjectMetadataName.name,
);
}
return this.buildRepositoryForWorkspace<T>(workspaceId, objectMetadataName);
}
async getWorkspaceDatasource() {
if (!this.workspaceDataSource) {
throw new Error('Workspace data source not found');
}
const workspaceId = this.workspaceDataSource.getWorkspaceId();
return this.buildDatasourceForWorkspace(workspaceId);
}
async buildDatasourceForWorkspace(workspaceId: string) {
const cacheVersion =
await this.workspaceCacheVersionService.getVersion(workspaceId);
let objectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!objectMetadataCollection) {
objectMetadataCollection = await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
objectMetadataCollection,
);
}
const entities = await Promise.all(
objectMetadataCollection.map((objectMetadata) =>
this.entitySchemaFactory.create(workspaceId, objectMetadata),
),
const workspaceDataSource = await this.workspaceDataSourceFactory.create(
workspaceId,
cacheVersion,
);
return await workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${cacheVersion}`,
async () => {
const workspaceDataSource =
await this.workspaceDataSourceFactory.create(entities, workspaceId);
return workspaceDataSource;
},
(dataSource) => dataSource.destroy(),
);
}
async buildRepositoryForWorkspace<T extends ObjectLiteral>(
workspaceId: string,
objectMetadataName: string,
) {
const workspaceDataSource =
await this.buildDatasourceForWorkspace(workspaceId);
if (!workspaceDataSource) {
throw new Error('Workspace data source not found');
}
return workspaceDataSource.getRepository<T>(objectMetadataName);
}
async getDatasource() {
const { workspaceId, cacheVersion } =
this.scopedWorkspaceContextFactory.create();
if (!workspaceId) {
throw new Error('Workspace not found');
}
return this.workspaceDataSourceFactory.create(workspaceId, cacheVersion);
}
}

View File

@ -1,24 +0,0 @@
import { EntityClassOrSchema } from '@nestjs/typeorm/dist/interfaces/entity-class-or-schema.type';
import { EntitySchema, Repository } from 'typeorm';
export function getWorkspaceRepositoryToken(
entity: EntityClassOrSchema,
// eslint-disable-next-line @typescript-eslint/ban-types
): Function | string {
if (entity === null || entity === undefined) {
throw new Error('Circular dependency @InjectWorkspaceRepository()');
}
if (entity instanceof Function && entity.prototype instanceof Repository) {
return entity;
}
if (entity instanceof EntitySchema) {
return `${
entity.options.target ? entity.options.target.name : entity.options.name
}WorkspaceRepository`;
}
return `${entity.name}WorkspaceRepository`;
}