admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Remove performance logs (#6709)

Browse Source 
We have found the root cause of the issue:
- when using a datasource (including the cached ones), we are fetching
ObjectMetadataCollection from cache (700kB). Datasource usage is
happening any time we are using twentyORM, which is everywhere in the
jobs and in some resolvers (including the GetCurrentUser one). This is
leading to a high load on redis and leading to the performance issues we
are seeing.
- we actually don't need to fetch this objectMetadataCollection while
using a cached datasource, only when we instantiate a new one
This commit is contained in:
Charles Bochet
2024-08-21 14:17:11 +02:00
committed by GitHub
parent aa4ae53fb4
commit 612a875c7b
5 changed files with 25 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { EntitySchema, Repository } from 'typeorm';
import { v4 } from 'uuid';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@ -29,17 +28,11 @@ export class WorkspaceDatasourceFactory {
workspaceId: string,
workspaceMetadataVersion: string | null,
): Promise<WorkspaceDataSource> {
const logId = v4();
console.time(`fetch in datasource factory ${logId}`);
const latestWorkspaceMetadataVersion =
await this.workspaceMetadataVersionService.getMetadataVersion(
workspaceId,
);
console.timeEnd(`fetch in datasource factory ${logId}`);
const desiredWorkspaceMetadataVersion =
workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;
@ -55,38 +48,35 @@ export class WorkspaceDatasourceFactory {
);
}
let cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!cachedObjectMetadataCollection) {
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
freshObjectMetadataCollection,
);
cachedObjectMetadataCollection = freshObjectMetadataCollection;
}
const workspaceDataSource = await workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${latestWorkspaceMetadataVersion}`,
async () => {
const logId = v4();
let cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!cachedObjectMetadataCollection) {
const freshObjectMetadataCollection =
await this.objectMetadataRepository.find({
where: { workspaceId },
relations: [
'fields.object',
'fields',
'fields.fromRelationMetadata',
'fields.toRelationMetadata',
'fields.fromRelationMetadata.toObjectMetadata',
],
});
await this.workspaceCacheStorageService.setObjectMetadataCollection(
workspaceId,
freshObjectMetadataCollection,
);
cachedObjectMetadataCollection = freshObjectMetadataCollection;
}
console.log('Creating workspace fresh data source...' + logId);
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
workspaceId,
@ -104,7 +94,6 @@ export class WorkspaceDatasourceFactory {
);
}
console.time('create entity schema' + logId);
const cachedEntitySchemaOptions =
await this.workspaceCacheStorageService.getORMEntitySchema(
workspaceId,
@ -130,9 +119,7 @@ export class WorkspaceDatasourceFactory {
cachedEntitySchemas = entitySchemas;
}
console.timeEnd('create entity schema' + logId);
console.time('create workspace data source' + logId);
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
@ -156,11 +143,7 @@ export class WorkspaceDatasourceFactory {
},
);
console.timeEnd('create workspace data source' + logId);
console.time('initialize workspace data source' + logId);
await workspaceDataSource.initialize();
console.timeEnd('initialize workspace data source' + logId);
return workspaceDataSource;
},

2
packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts
View File

@ -16,7 +16,6 @@ export class CacheManager<T> {
return this.cache.get(cacheKey)!;
}
// Remove old entries with the same workspaceId
for (const key of this.cache.keys()) {
if (key.startsWith(`${workspaceId}-`)) {
await onDelete?.(this.cache.get(key)!);
@ -24,7 +23,6 @@ export class CacheManager<T> {
}
}
// Create a new value using the factory callback
const value = await factory();
if (!value) {

6
packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts
View File

@ -1,7 +1,6 @@
import { Injectable, Type } from '@nestjs/common';
import { ObjectLiteral } from 'typeorm';
import { v4 } from 'uuid';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
@ -37,16 +36,11 @@ export class TwentyORMGlobalManager {
);
}
const logId = v4();
console.time(`createDataSource in orm ${logId}`);
const workspaceDataSource = await this.workspaceDataSourceFactory.create(
workspaceId,
null,
);
console.timeEnd(`createDataSource in orm ${logId}`);
const repository = workspaceDataSource.getRepository<T>(objectMetadataName);
return repository;