From 40bbee8d9fe35094335040bfe5093b7ec911adb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Bosi?= <71827178+bosiraphael@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:54:55 +0200 Subject: [PATCH] 5x Fix cache performance issues (#6616) Calling `getObjectMetadata` from `WorkspaceCacheStorageService` in every query was causing big performance issues. The `objectMetadataCollection` is now part of the `WorkspaceInternalContext` so we only instance it once in the `WorkspaceDatasourceFactory`. Queries are now much faster, for instance for TimelineCalendar, it went from ~450ms to 80ms. --- .../entity-schema-relation.factory.ts | 14 +++++++--- .../factories/workspace-datasource.factory.ts | 2 +- .../workspace-internal-context.interface.ts | 4 +-- .../repository/workspace.repository.ts | 26 +++++++------------ .../utils/determine-relation-details.util.ts | 8 +++--- .../workspace-cache-storage.service.ts | 15 ----------- 6 files changed, 27 insertions(+), 42 deletions(-) diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts index cdd1a5585..2a90e827a 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts @@ -3,8 +3,8 @@ import { Injectable } from '@nestjs/common'; import { EntitySchemaRelationOptions } from 'typeorm'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util'; import { determineRelationDetails } from 'src/engine/twenty-orm/utils/determine-relation-details.util'; +import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; type EntitySchemaRelationMap = { @@ -37,11 +37,19 @@ export class EntitySchemaRelationFactory { ); } + const objectMetadataCollection = + await this.workspaceCacheStorageService.getObjectMetadataCollection( + workspaceId, + ); + + if (!objectMetadataCollection) { + throw new Error('Object metadata collection not found'); + } + const relationDetails = await determineRelationDetails( - workspaceId, fieldMetadata, relationMetadata, - this.workspaceCacheStorageService, + objectMetadataCollection, ); entitySchemaRelationMap[fieldMetadata.name] = { diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 45c4d2a81..1ee50a0b4 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -93,7 +93,7 @@ export class WorkspaceDatasourceFactory { const workspaceDataSource = new WorkspaceDataSource( { workspaceId, - workspaceCacheStorage: this.workspaceCacheStorageService, + objectMetadataCollection: cachedObjectMetadataCollection, }, { url: diff --git a/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts b/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts index 07bfeabd1..266c60324 100644 --- a/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts +++ b/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts @@ -1,6 +1,6 @@ -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; export interface WorkspaceInternalContext { workspaceId: string; - workspaceCacheStorage: WorkspaceCacheStorageService; + objectMetadataCollection: ObjectMetadataEntity[]; } diff --git a/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts b/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts index d0dc9a5cb..3adb6e21d 100644 --- a/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts +++ b/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts @@ -18,9 +18,9 @@ import { SaveOptions, UpdateResult, } from 'typeorm'; +import { PickKeysByType } from 'typeorm/common/PickKeysByType'; import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'; import { UpsertOptions } from 'typeorm/repository/UpsertOptions'; -import { PickKeysByType } from 'typeorm/common/PickKeysByType'; import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface'; @@ -235,6 +235,7 @@ export class WorkspaceRepository< formattedEntityOrEntities, options, ); + const formattedResult = await this.formatResult(result); return formattedResult; @@ -423,6 +424,7 @@ export class WorkspaceRepository< entityManager?: EntityManager, ): Promise { const manager = entityManager || this.manager; + const formatedEntity = await this.formatData(entity); const result = await manager.insert(this.target, formatedEntity); const formattedResult = await this.formatResult(result); @@ -621,22 +623,15 @@ export class WorkspaceRepository< throw new Error('Object metadata name is missing'); } - const objectMetadata = - await this.internalContext.workspaceCacheStorage.getObjectMetadata( - this.internalContext.workspaceId, - (objectMetadata) => objectMetadata.nameSingular === objectMetadataName, - ); + const objectMetadata = this.internalContext.objectMetadataCollection.find( + (objectMetadata) => objectMetadata.nameSingular === objectMetadataName, + ); if (!objectMetadata) { - const objectMetadataCollection = - await this.internalContext.workspaceCacheStorage.getObjectMetadataCollection( - this.internalContext.workspaceId, - ); - throw new Error( `Object metadata for object "${objectMetadataName}" is missing ` + `in workspace "${this.internalContext.workspaceId}" ` + - `with object metadata collection length: ${objectMetadataCollection?.length}`, + `with object metadata collection length: ${this.internalContext.objectMetadataCollection.length}`, ); } @@ -806,15 +801,13 @@ export class WorkspaceRepository< if (relationMetadata) { const toObjectMetadata = - await this.internalContext.workspaceCacheStorage.getObjectMetadata( - relationMetadata.workspaceId, + this.internalContext.objectMetadataCollection.find( (objectMetadata) => objectMetadata.id === relationMetadata.toObjectMetadataId, ); const fromObjectMetadata = - await this.internalContext.workspaceCacheStorage.getObjectMetadata( - relationMetadata.workspaceId, + this.internalContext.objectMetadataCollection.find( (objectMetadata) => objectMetadata.id === relationMetadata.fromObjectMetadataId, ); @@ -833,6 +826,7 @@ export class WorkspaceRepository< newData[key] = await this.formatResult( value, + relationType === 'one-to-many' ? toObjectMetadata : fromObjectMetadata, diff --git a/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts b/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts index bcd8cb6b1..4a1cdb8e8 100644 --- a/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts +++ b/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts @@ -4,7 +4,6 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { computeRelationType } from 'src/engine/twenty-orm/utils/compute-relation-type.util'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; interface RelationDetails { relationType: RelationType; @@ -14,10 +13,9 @@ interface RelationDetails { } export async function determineRelationDetails( - workspaceId: string, fieldMetadata: FieldMetadataEntity, relationMetadata: RelationMetadataEntity, - workspaceCacheStorageService: WorkspaceCacheStorageService, + objectMetadataCollection: ObjectMetadataEntity[], ): Promise { const relationType = computeRelationType(fieldMetadata, relationMetadata); let fromObjectMetadata: ObjectMetadataEntity | undefined = @@ -28,8 +26,8 @@ export async function determineRelationDetails( // RelationMetadata always store the relation from the perspective of the `from` object, MANY_TO_ONE relations are not stored yet if (relationType === 'many-to-one') { fromObjectMetadata = fieldMetadata.object; - toObjectMetadata = await workspaceCacheStorageService.getObjectMetadata( - workspaceId, + + toObjectMetadata = objectMetadataCollection.find( (objectMetadata) => objectMetadata.id === relationMetadata.fromObjectMetadataId, ); diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts index 6dfcc5f3a..616084702 100644 --- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts +++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts @@ -63,21 +63,6 @@ export class WorkspaceCacheStorageService { ); } - async getObjectMetadata( - workspaceId: string, - predicate: (objectMetadata: ObjectMetadataEntity) => boolean, - ): Promise { - const objectMetadataCollection = await this.workspaceSchemaCache.get< - ObjectMetadataEntity[] - >(`objectMetadataCollection:${workspaceId}`); - - if (!objectMetadataCollection) { - return; - } - - return objectMetadataCollection.find(predicate); - } - setTypeDefs(workspaceId: string, typeDefs: string): Promise { return this.workspaceSchemaCache.set( `typeDefs:${workspaceId}`,