From 612a875c7bce85634ae7c08bd434508c456c75e7 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 21 Aug 2024 14:17:11 +0200 Subject: [PATCH] Remove performance logs (#6709) We have found the root cause of the issue: - when using a datasource (including the cached ones), we are fetching ObjectMetadataCollection from cache (700kB). Datasource usage is happening any time we are using twentyORM, which is everywhere in the jobs and in some resolvers (including the GetCurrentUser one). This is leading to a high load on redis and leading to the performance issues we are seeing. - we actually don't need to fetch this objectMetadataCollection while using a cached datasource, only when we instantiate a new one --- .../user/services/user.service.ts | 13 ---- .../engine/core-modules/user/user.resolver.ts | 5 -- .../factories/workspace-datasource.factory.ts | 67 +++++++------------ .../storage/cache-manager.storage.ts | 2 - .../twenty-orm/twenty-orm-global.manager.ts | 6 -- 5 files changed, 25 insertions(+), 68 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts index 6c42f93b2..aec142d26 100644 --- a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts +++ b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts @@ -40,24 +40,18 @@ export class UserService extends TypeOrmQueryService { return null; } - console.time('loadWorkspaceMember repo'); const workspaceMemberRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( user.defaultWorkspaceId, 'workspaceMember', ); - console.timeEnd('loadWorkspaceMember repo'); - - console.time('loadWorkspaceMember find'); const workspaceMember = await workspaceMemberRepository.findOne({ where: { userId: user.id, }, }); - console.timeEnd('loadWorkspaceMember find'); - return workspaceMember; } @@ -66,21 +60,14 @@ export class UserService extends TypeOrmQueryService { return []; } - console.time('loadWorkspaceMembers repo'); const workspaceMemberRepository = await this.twentyORMGlobalManager.getRepositoryForWorkspace( workspace.id, 'workspaceMember', ); - console.timeEnd('loadWorkspaceMembers repo'); - - console.time('loadWorkspaceMembers find'); - const workspaceMembers = workspaceMemberRepository.find(); - console.timeEnd('loadWorkspaceMembers find'); - return workspaceMembers; } diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts index b91140621..c77bb690b 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts @@ -98,7 +98,6 @@ export class UserResolver { nullable: true, }) async workspaceMember(@Parent() user: User): Promise { - console.time('resolver workspaceMember'); const workspaceMember = await this.userService.loadWorkspaceMember(user); if (workspaceMember && workspaceMember.avatarUrl) { @@ -109,7 +108,6 @@ export class UserResolver { workspaceMember.avatarUrl = `${workspaceMember.avatarUrl}?token=${avatarUrlToken}`; } - console.timeEnd('resolver workspaceMember'); // TODO: Fix typing disrepency between Entity and DTO return workspaceMember as WorkspaceMember | null; @@ -119,7 +117,6 @@ export class UserResolver { nullable: true, }) async workspaceMembers(@Parent() user: User): Promise { - console.time('resolver workspaceMembers'); const workspaceMembers = await this.userService.loadWorkspaceMembers( user.defaultWorkspace, ); @@ -135,8 +132,6 @@ export class UserResolver { } } - console.timeEnd('resolver workspaceMembers'); - // TODO: Fix typing disrepency between Entity and DTO return workspaceMembers as WorkspaceMember[]; } diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 1a1b92a28..c13301020 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { EntitySchema, Repository } from 'typeorm'; -import { v4 } from 'uuid'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; @@ -29,17 +28,11 @@ export class WorkspaceDatasourceFactory { workspaceId: string, workspaceMetadataVersion: string | null, ): Promise { - const logId = v4(); - - console.time(`fetch in datasource factory ${logId}`); - const latestWorkspaceMetadataVersion = await this.workspaceMetadataVersionService.getMetadataVersion( workspaceId, ); - console.timeEnd(`fetch in datasource factory ${logId}`); - const desiredWorkspaceMetadataVersion = workspaceMetadataVersion ?? latestWorkspaceMetadataVersion; @@ -55,38 +48,35 @@ export class WorkspaceDatasourceFactory { ); } - let cachedObjectMetadataCollection = - await this.workspaceCacheStorageService.getObjectMetadataCollection( - workspaceId, - ); - - if (!cachedObjectMetadataCollection) { - const freshObjectMetadataCollection = - await this.objectMetadataRepository.find({ - where: { workspaceId }, - relations: [ - 'fields.object', - 'fields', - 'fields.fromRelationMetadata', - 'fields.toRelationMetadata', - 'fields.fromRelationMetadata.toObjectMetadata', - ], - }); - - await this.workspaceCacheStorageService.setObjectMetadataCollection( - workspaceId, - freshObjectMetadataCollection, - ); - - cachedObjectMetadataCollection = freshObjectMetadataCollection; - } - const workspaceDataSource = await workspaceDataSourceCacheInstance.execute( `${workspaceId}-${latestWorkspaceMetadataVersion}`, async () => { - const logId = v4(); + let cachedObjectMetadataCollection = + await this.workspaceCacheStorageService.getObjectMetadataCollection( + workspaceId, + ); + + if (!cachedObjectMetadataCollection) { + const freshObjectMetadataCollection = + await this.objectMetadataRepository.find({ + where: { workspaceId }, + relations: [ + 'fields.object', + 'fields', + 'fields.fromRelationMetadata', + 'fields.toRelationMetadata', + 'fields.fromRelationMetadata.toObjectMetadata', + ], + }); + + await this.workspaceCacheStorageService.setObjectMetadataCollection( + workspaceId, + freshObjectMetadataCollection, + ); + + cachedObjectMetadataCollection = freshObjectMetadataCollection; + } - console.log('Creating workspace fresh data source...' + logId); const dataSourceMetadata = await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId( workspaceId, @@ -104,7 +94,6 @@ export class WorkspaceDatasourceFactory { ); } - console.time('create entity schema' + logId); const cachedEntitySchemaOptions = await this.workspaceCacheStorageService.getORMEntitySchema( workspaceId, @@ -130,9 +119,7 @@ export class WorkspaceDatasourceFactory { cachedEntitySchemas = entitySchemas; } - console.timeEnd('create entity schema' + logId); - console.time('create workspace data source' + logId); const workspaceDataSource = new WorkspaceDataSource( { workspaceId, @@ -156,11 +143,7 @@ export class WorkspaceDatasourceFactory { }, ); - console.timeEnd('create workspace data source' + logId); - - console.time('initialize workspace data source' + logId); await workspaceDataSource.initialize(); - console.timeEnd('initialize workspace data source' + logId); return workspaceDataSource; }, diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts index 6dee55b50..fcd40d6f3 100644 --- a/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts +++ b/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts @@ -16,7 +16,6 @@ export class CacheManager { return this.cache.get(cacheKey)!; } - // Remove old entries with the same workspaceId for (const key of this.cache.keys()) { if (key.startsWith(`${workspaceId}-`)) { await onDelete?.(this.cache.get(key)!); @@ -24,7 +23,6 @@ export class CacheManager { } } - // Create a new value using the factory callback const value = await factory(); if (!value) { diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts index 6d24e6f8e..f4d8190c5 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts @@ -1,7 +1,6 @@ import { Injectable, Type } from '@nestjs/common'; import { ObjectLiteral } from 'typeorm'; -import { v4 } from 'uuid'; import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory'; import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; @@ -37,16 +36,11 @@ export class TwentyORMGlobalManager { ); } - const logId = v4(); - - console.time(`createDataSource in orm ${logId}`); const workspaceDataSource = await this.workspaceDataSourceFactory.create( workspaceId, null, ); - console.timeEnd(`createDataSource in orm ${logId}`); - const repository = workspaceDataSource.getRepository(objectMetadataName); return repository;