admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

5x Fix cache performance issues (#6616)

Browse Source 
Calling `getObjectMetadata` from `WorkspaceCacheStorageService` in every
query was causing big performance issues. The `objectMetadataCollection`
is now part of the `WorkspaceInternalContext` so we only instance it
once in the `WorkspaceDatasourceFactory`.
Queries are now much faster, for instance for TimelineCalendar, it went
from ~450ms to 80ms.
This commit is contained in:
Raphaël Bosi
2024-08-13 17:54:55 +02:00
committed by GitHub
parent 65a961ff3e
commit 40bbee8d9f
6 changed files with 27 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts
View File

@ -3,8 +3,8 @@ import { Injectable } from '@nestjs/common';
import { EntitySchemaRelationOptions } from 'typeorm';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
import { determineRelationDetails } from 'src/engine/twenty-orm/utils/determine-relation-details.util';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
type EntitySchemaRelationMap = {
@ -37,11 +37,19 @@ export class EntitySchemaRelationFactory {
);
}
const objectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
);
if (!objectMetadataCollection) {
throw new Error('Object metadata collection not found');
}
const relationDetails = await determineRelationDetails(
workspaceId,
fieldMetadata,
relationMetadata,
this.workspaceCacheStorageService,
objectMetadataCollection,
);
entitySchemaRelationMap[fieldMetadata.name] = {

2
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -93,7 +93,7 @@ export class WorkspaceDatasourceFactory {
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
workspaceCacheStorage: this.workspaceCacheStorageService,
objectMetadataCollection: cachedObjectMetadataCollection,
},
{
url:

4
packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts
View File

@ -1,6 +1,6 @@
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
export interface WorkspaceInternalContext {
workspaceId: string;
workspaceCacheStorage: WorkspaceCacheStorageService;
objectMetadataCollection: ObjectMetadataEntity[];
}

26
packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts
View File

@ -18,9 +18,9 @@ import {
SaveOptions,
UpdateResult,
} from 'typeorm';
import { PickKeysByType } from 'typeorm/common/PickKeysByType';
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
import { UpsertOptions } from 'typeorm/repository/UpsertOptions';
import { PickKeysByType } from 'typeorm/common/PickKeysByType';
import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface';
@ -235,6 +235,7 @@ export class WorkspaceRepository<
formattedEntityOrEntities,
options,
);
const formattedResult = await this.formatResult(result);
return formattedResult;
@ -423,6 +424,7 @@ export class WorkspaceRepository<
entityManager?: EntityManager,
): Promise<InsertResult> {
const manager = entityManager || this.manager;
const formatedEntity = await this.formatData(entity);
const result = await manager.insert(this.target, formatedEntity);
const formattedResult = await this.formatResult(result);
@ -621,22 +623,15 @@ export class WorkspaceRepository<
throw new Error('Object metadata name is missing');
}
const objectMetadata =
await this.internalContext.workspaceCacheStorage.getObjectMetadata(
this.internalContext.workspaceId,
(objectMetadata) => objectMetadata.nameSingular === objectMetadataName,
);
const objectMetadata = this.internalContext.objectMetadataCollection.find(
(objectMetadata) => objectMetadata.nameSingular === objectMetadataName,
);
if (!objectMetadata) {
const objectMetadataCollection =
await this.internalContext.workspaceCacheStorage.getObjectMetadataCollection(
this.internalContext.workspaceId,
);
throw new Error(
`Object metadata for object "${objectMetadataName}" is missing ` +
`in workspace "${this.internalContext.workspaceId}" ` +
`with object metadata collection length: ${objectMetadataCollection?.length}`,
`with object metadata collection length: ${this.internalContext.objectMetadataCollection.length}`,
);
}
@ -806,15 +801,13 @@ export class WorkspaceRepository<
if (relationMetadata) {
const toObjectMetadata =
await this.internalContext.workspaceCacheStorage.getObjectMetadata(
relationMetadata.workspaceId,
this.internalContext.objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.toObjectMetadataId,
);
const fromObjectMetadata =
await this.internalContext.workspaceCacheStorage.getObjectMetadata(
relationMetadata.workspaceId,
this.internalContext.objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.fromObjectMetadataId,
);
@ -833,6 +826,7 @@ export class WorkspaceRepository<
newData[key] = await this.formatResult(
value,
relationType === 'one-to-many'
? toObjectMetadata
: fromObjectMetadata,

8
packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts
View File

@ -4,7 +4,6 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { computeRelationType } from 'src/engine/twenty-orm/utils/compute-relation-type.util';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
interface RelationDetails {
relationType: RelationType;
@ -14,10 +13,9 @@ interface RelationDetails {
}
export async function determineRelationDetails(
workspaceId: string,
fieldMetadata: FieldMetadataEntity,
relationMetadata: RelationMetadataEntity,
workspaceCacheStorageService: WorkspaceCacheStorageService,
objectMetadataCollection: ObjectMetadataEntity[],
): Promise<RelationDetails> {
const relationType = computeRelationType(fieldMetadata, relationMetadata);
let fromObjectMetadata: ObjectMetadataEntity | undefined =
@ -28,8 +26,8 @@ export async function determineRelationDetails(
// RelationMetadata always store the relation from the perspective of the `from` object, MANY_TO_ONE relations are not stored yet
if (relationType === 'many-to-one') {
fromObjectMetadata = fieldMetadata.object;
toObjectMetadata = await workspaceCacheStorageService.getObjectMetadata(
workspaceId,
toObjectMetadata = objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.fromObjectMetadataId,
);

15
packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts
View File

@ -63,21 +63,6 @@ export class WorkspaceCacheStorageService {
);
}
async getObjectMetadata(
workspaceId: string,
predicate: (objectMetadata: ObjectMetadataEntity) => boolean,
): Promise<ObjectMetadataEntity | undefined> {
const objectMetadataCollection = await this.workspaceSchemaCache.get<
ObjectMetadataEntity[]
>(`objectMetadataCollection:${workspaceId}`);
if (!objectMetadataCollection) {
return;
}
return objectMetadataCollection.find(predicate);
}
setTypeDefs(workspaceId: string, typeDefs: string): Promise<void> {
return this.workspaceSchemaCache.set<string>(
`typeDefs:${workspaceId}`,