admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor metadata caching (#7011)

Browse Source 
This PR introduces the following changes:
- add the metadataVersion to all our metadata cache keys to ease
troubleshooting:
<img width="1146" alt="image"
src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12">
- introduce a cache recompute lock to avoid overloading the database to
recompute the cache many time
This commit is contained in:
Charles Bochet
2024-09-12 15:57:30 +02:00
committed by Charles Bochet
parent 9b46e8c663
commit 3c4168759a
32 changed files with 420 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts Normal file
View File

@ -0,0 +1,15 @@
import { CustomException } from 'src/utils/custom-exception';
export class TwentyORMException extends CustomException {
code: TwentyORMExceptionCode;
constructor(message: string, code: TwentyORMExceptionCode) {
super(message, code);
}
}
export enum TwentyORMExceptionCode {
METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
METADATA_VERSION_MISMATCH = 'METADATA_VERSION_MISMATCH',
METADATA_COLLECTION_NOT_FOUND = 'METADATA_COLLECTION_NOT_FOUND',
WORKSPACE_SCHEMA_NOT_FOUND = 'WORKSPACE_SCHEMA_NOT_FOUND',
}

2
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts
View File

@ -19,6 +19,7 @@ export class EntitySchemaRelationFactory {
async create(
workspaceId: string,
metadataVersion: number,
fieldMetadataCollection: FieldMetadataEntity[],
): Promise<EntitySchemaRelationMap> {
const entitySchemaRelationMap: EntitySchemaRelationMap = {};
@ -40,6 +41,7 @@ export class EntitySchemaRelationFactory {
const objectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
metadataVersion,
);
if (!objectMetadataCollection) {

2
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts
View File

@ -17,6 +17,7 @@ export class EntitySchemaFactory {
async create(
workspaceId: string,
metadataVersion: number,
objectMetadata: ObjectMetadataEntity,
): Promise<EntitySchema> {
const columns = this.entitySchemaColumnFactory.create(
@ -26,6 +27,7 @@ export class EntitySchemaFactory {
const relations = await this.entitySchemaRelationFactory.create(
workspaceId,
metadataVersion,
objectMetadata.fields,
);

4
packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts
View File

@ -11,11 +11,11 @@ export class ScopedWorkspaceContextFactory {
public create(): {
workspaceId: string | null;
workspaceMetadataVersion: string | null;
workspaceMetadataVersion: number | null;
} {
const workspaceId: string | undefined =
this.request?.['req']?.['workspaceId'];
const workspaceMetadataVersion: string | undefined =
const workspaceMetadataVersion: number | undefined =
this.request?.['req']?.['workspaceMetadataVersion'];
return {

80
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -1,13 +1,15 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { EntitySchema, Repository } from 'typeorm';
import { EntitySchema } from 'typeorm';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import {
TwentyORMException,
TwentyORMExceptionCode,
} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { CacheManager } from 'src/engine/twenty-orm/storage/cache-manager.storage';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@ -20,63 +22,56 @@ export class WorkspaceDatasourceFactory {
private readonly dataSourceService: DataSourceService,
private readonly environmentService: EnvironmentService,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
private readonly entitySchemaFactory: EntitySchemaFactory,
) {}
public async create(
workspaceId: string,
workspaceMetadataVersion: string | null,
workspaceMetadataVersion: number | null,
): Promise<WorkspaceDataSource> {
const latestWorkspaceMetadataVersion =
await this.workspaceMetadataVersionService.getMetadataVersion(
await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
if (latestWorkspaceMetadataVersion === undefined) {
await this.workspaceMetadataCacheService.recomputeMetadataCache(
workspaceId,
);
throw new TwentyORMException(
`Metadata version not found for workspace ${workspaceId}`,
TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND,
);
}
const desiredWorkspaceMetadataVersion =
workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;
if (!desiredWorkspaceMetadataVersion) {
throw new Error(
`Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`,
);
}
if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) {
throw new Error(
throw new TwentyORMException(
`Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`,
TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
);
}
const workspaceDataSource = await this.cacheManager.execute(
`${workspaceId}-${latestWorkspaceMetadataVersion}`,
`${workspaceId}-${desiredWorkspaceMetadataVersion}`,
async () => {
let cachedObjectMetadataCollection =
const cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
desiredWorkspaceMetadataVersion,
);
if (!cachedObjectMetadataCollection) {
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
await this.workspaceMetadataCacheService.recomputeMetadataCache(
workspaceId,
freshObjectMetadataCollection,
true,
);
cachedObjectMetadataCollection = freshObjectMetadataCollection;
throw new TwentyORMException(
`Object metadata collection not found for workspace ${workspaceId}`,
TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND,
);
}
const dataSourceMetadata =
@ -85,20 +80,16 @@ export class WorkspaceDatasourceFactory {
);
if (!dataSourceMetadata) {
throw new Error(
`Data source metadata not found for workspace ${workspaceId}`,
);
}
if (!cachedObjectMetadataCollection) {
throw new Error(
`Object metadata collection not found for workspace ${workspaceId}`,
throw new TwentyORMException(
`Workspace Schema not found for workspace ${workspaceId}`,
TwentyORMExceptionCode.WORKSPACE_SCHEMA_NOT_FOUND,
);
}
const cachedEntitySchemaOptions =
await this.workspaceCacheStorageService.getORMEntitySchema(
workspaceId,
desiredWorkspaceMetadataVersion,
);
let cachedEntitySchemas: EntitySchema[];
@ -110,12 +101,17 @@ export class WorkspaceDatasourceFactory {
} else {
const entitySchemas = await Promise.all(
cachedObjectMetadataCollection.map((objectMetadata) =>
this.entitySchemaFactory.create(workspaceId, objectMetadata),
this.entitySchemaFactory.create(
workspaceId,
desiredWorkspaceMetadataVersion,
objectMetadata,
),
),
);
await this.workspaceCacheStorageService.setORMEntitySchema(
workspaceId,
desiredWorkspaceMetadataVersion,
entitySchemas.map((entitySchema) => entitySchema.options),
);

6
packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts
View File

@ -47,6 +47,10 @@ export class TwentyORMGlobalManager {
}
async getDataSourceForWorkspace(workspaceId: string) {
return this.workspaceDataSourceFactory.create(workspaceId, null);
return await this.workspaceDataSourceFactory.create(workspaceId, null);
}
async loadDataSourceForWorkspace(workspaceId: string) {
await this.workspaceDataSourceFactory.create(workspaceId, null);
}
}

4
packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts
View File

@ -3,7 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
import { entitySchemaFactories } from 'src/engine/twenty-orm/factories';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
@ -16,7 +16,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/
TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
DataSourceModule,
WorkspaceCacheStorageModule,
WorkspaceMetadataVersionModule,
WorkspaceMetadataCacheModule,
],
providers: [
...entitySchemaFactories,