admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance twenty orm (#6691)

Browse Source 
## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts
View File

@ -11,16 +11,16 @@ export class ScopedWorkspaceContextFactory {
public create(): {
workspaceId: string | null;
cacheVersion: string | null;
workspaceMetadataVersion: string | null;
} {
const workspaceId: string | undefined =
this.request?.['req']?.['workspaceId'];
const cacheVersion: string | undefined =
this.request?.['req']?.['cacheVersion'];
const workspaceMetadataVersion: string | undefined =
this.request?.['req']?.['workspaceMetadataVersion'];
return {
workspaceId: workspaceId ?? null,
cacheVersion: cacheVersion ?? null,
workspaceMetadataVersion: workspaceMetadataVersion ?? null,
};
}
}

77
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -1,12 +1,12 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { EntitySchema, Repository } from 'typeorm';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { workspaceDataSourceCacheInstance } from 'src/engine/twenty-orm/twenty-orm-core.module';
@ -18,7 +18,7 @@ export class WorkspaceDatasourceFactory {
private readonly dataSourceService: DataSourceService,
private readonly environmentService: EnvironmentService,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly entitySchemaFactory: EntitySchemaFactory,
@ -26,21 +26,29 @@ export class WorkspaceDatasourceFactory {
public async create(
workspaceId: string,
workspaceSchemaVersion: string | null,
workspaceMetadataVersion: string | null,
): Promise<WorkspaceDataSource> {
const desiredWorkspaceSchemaVersion =
workspaceSchemaVersion ??
(await this.workspaceCacheVersionService.getVersion(workspaceId));
const desiredWorkspaceMetadataVersion =
workspaceMetadataVersion ??
(await this.workspaceMetadataVersionService.getMetadataVersion(
workspaceId,
));
if (!desiredWorkspaceSchemaVersion) {
throw new Error('Cache version not found');
if (!desiredWorkspaceMetadataVersion) {
throw new Error(
`Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`,
);
}
const latestWorkspaceSchemaVersion =
await this.workspaceCacheVersionService.getVersion(workspaceId);
const latestWorkspaceMetadataVersion =
await this.workspaceMetadataVersionService.getMetadataVersion(
workspaceId,
);
if (latestWorkspaceSchemaVersion !== desiredWorkspaceSchemaVersion) {
throw new Error('Cache version mismatch');
if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) {
throw new Error(
`Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`,
);
}
let cachedObjectMetadataCollection =
@ -70,7 +78,7 @@ export class WorkspaceDatasourceFactory {
}
const workspaceDataSource = await workspaceDataSourceCacheInstance.execute(
`${workspaceId}-${latestWorkspaceSchemaVersion}`,
`${workspaceId}-${latestWorkspaceMetadataVersion}`,
async () => {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
@ -78,18 +86,43 @@ export class WorkspaceDatasourceFactory {
);
if (!dataSourceMetadata) {
throw new Error('Data source metadata not found');
throw new Error(
`Data source metadata not found for workspace ${workspaceId}`,
);
}
if (!cachedObjectMetadataCollection) {
throw new Error('Object metadata collection not found');
throw new Error(
`Object metadata collection not found for workspace ${workspaceId}`,
);
}
const cachedEntitySchemaOptions =
await this.workspaceCacheStorageService.getORMEntitySchema(
workspaceId,
);
let cachedEntitySchemas: EntitySchema[];
if (cachedEntitySchemaOptions) {
cachedEntitySchemas = cachedEntitySchemaOptions.map(
(option) => new EntitySchema(option),
);
} else {
const entitySchemas = await Promise.all(
cachedObjectMetadataCollection.map((objectMetadata) =>
this.entitySchemaFactory.create(workspaceId, objectMetadata),
),
);
await this.workspaceCacheStorageService.setORMEntitySchema(
workspaceId,
entitySchemas.map((entitySchema) => entitySchema.options),
);
cachedEntitySchemas = entitySchemas;
}
const entities = await Promise.all(
cachedObjectMetadataCollection.map((objectMetadata) =>
this.entitySchemaFactory.create(workspaceId, objectMetadata),
),
);
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
@ -104,7 +137,7 @@ export class WorkspaceDatasourceFactory {
? ['query', 'error']
: ['error'],
schema: dataSourceMetadata.schema,
entities,
entities: cachedEntitySchemas,
ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED')
? {
rejectUnauthorized: false,