Improve performance twenty orm (#6691)

## Context As we grow, the messaging scripts are experiencing performance issues forcing us to temporarily disable them on the cloud. While investigating the performance, I have noticed that generating the entity schema (for twentyORM) in the repository is taking ~500ms locally on my Mac M2 so likely more on pods. Caching the entitySchema then! I'm also clarifying naming around schemaVersion and cacheVersions ==> both are renamed workspaceMetadataVersion and migrated to the workspace table (the workspaceCacheVersion table is dropped).
2024-08-20 19:42:02 +02:00
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
--- a/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts
@ -1,44 +0,0 @@
-import { Inject, Type } from '@nestjs/common';
-import { ModuleRef, createContextId } from '@nestjs/core';
-import { Injector } from '@nestjs/core/injector/injector';
-
-import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
-
-export class LoadServiceWithWorkspaceContext {
-  private readonly injector = new Injector();
-
-  constructor(
-    @Inject(ModuleRef)
-    private readonly moduleRef: ModuleRef,
-    private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
-  ) {}
-
-  async load<T>(service: T, workspaceId: string): Promise<T> {
-    const modules = this.moduleRef['container'].getModules();
-    const host = [...modules.values()].find((module) =>
-      module.providers.has((service as Type<T>).constructor),
-    );
-
-    if (!host) {
-      throw new Error('Host module not found for the service');
-    }
-
-    const contextId = createContextId();
-    const cacheVersion =
-      await this.workspaceCacheVersionService.getVersion(workspaceId);
-
-    if (this.moduleRef.registerRequestByContextId) {
-      this.moduleRef.registerRequestByContextId(
-        { req: { workspaceId, cacheVersion } },
-        contextId,
-      );
-    }
-
-    return this.injector.loadPerContext(
-      service,
-      host,
-      new Map(host.providers),
-      contextId,
-    );
-  }
-}
--- a/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts
@ -11,16 +11,16 @@ export class ScopedWorkspaceContextFactory {

  public create(): {
    workspaceId: string | null;
-    cacheVersion: string | null;
+    workspaceMetadataVersion: string | null;
  } {
    const workspaceId: string | undefined =
      this.request?.['req']?.['workspaceId'];
-    const cacheVersion: string | undefined =
-      this.request?.['req']?.['cacheVersion'];
+    const workspaceMetadataVersion: string | undefined =
+      this.request?.['req']?.['workspaceMetadataVersion'];

    return {
      workspaceId: workspaceId ?? null,
-      cacheVersion: cacheVersion ?? null,
+      workspaceMetadataVersion: workspaceMetadataVersion ?? null,
    };
  }
 }
--- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
@ -1,12 +1,12 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';

-import { Repository } from 'typeorm';
+import { EntitySchema, Repository } from 'typeorm';

 import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
-import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
+import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
 import { workspaceDataSourceCacheInstance } from 'src/engine/twenty-orm/twenty-orm-core.module';
@ -18,7 +18,7 @@ export class WorkspaceDatasourceFactory {
    private readonly dataSourceService: DataSourceService,
    private readonly environmentService: EnvironmentService,
    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
-    private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
+    private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
    @InjectRepository(ObjectMetadataEntity, 'metadata')
    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
    private readonly entitySchemaFactory: EntitySchemaFactory,
@ -26,21 +26,29 @@ export class WorkspaceDatasourceFactory {

  public async create(
    workspaceId: string,
-    workspaceSchemaVersion: string | null,
+    workspaceMetadataVersion: string | null,
  ): Promise<WorkspaceDataSource> {
-    const desiredWorkspaceSchemaVersion =
-      workspaceSchemaVersion ??
-      (await this.workspaceCacheVersionService.getVersion(workspaceId));
+    const desiredWorkspaceMetadataVersion =
+      workspaceMetadataVersion ??
+      (await this.workspaceMetadataVersionService.getMetadataVersion(
+        workspaceId,
+      ));

-    if (!desiredWorkspaceSchemaVersion) {
-      throw new Error('Cache version not found');
+    if (!desiredWorkspaceMetadataVersion) {
+      throw new Error(
+        `Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`,
+      );
    }

-    const latestWorkspaceSchemaVersion =
-      await this.workspaceCacheVersionService.getVersion(workspaceId);
+    const latestWorkspaceMetadataVersion =
+      await this.workspaceMetadataVersionService.getMetadataVersion(
+        workspaceId,
+      );

-    if (latestWorkspaceSchemaVersion !== desiredWorkspaceSchemaVersion) {
-      throw new Error('Cache version mismatch');
+    if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) {
+      throw new Error(
+        `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`,
+      );
    }

    let cachedObjectMetadataCollection =
@ -70,7 +78,7 @@ export class WorkspaceDatasourceFactory {
    }

    const workspaceDataSource = await workspaceDataSourceCacheInstance.execute(
-      `${workspaceId}-${latestWorkspaceSchemaVersion}`,
+      `${workspaceId}-${latestWorkspaceMetadataVersion}`,
      async () => {
        const dataSourceMetadata =
          await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
@ -78,18 +86,43 @@ export class WorkspaceDatasourceFactory {
          );

        if (!dataSourceMetadata) {
-          throw new Error('Data source metadata not found');
+          throw new Error(
+            `Data source metadata not found for workspace ${workspaceId}`,
+          );
        }

        if (!cachedObjectMetadataCollection) {
-          throw new Error('Object metadata collection not found');
+          throw new Error(
+            `Object metadata collection not found for workspace ${workspaceId}`,
+          );
+        }
+
+        const cachedEntitySchemaOptions =
+          await this.workspaceCacheStorageService.getORMEntitySchema(
+            workspaceId,
+          );
+
+        let cachedEntitySchemas: EntitySchema[];
+
+        if (cachedEntitySchemaOptions) {
+          cachedEntitySchemas = cachedEntitySchemaOptions.map(
+            (option) => new EntitySchema(option),
+          );
+        } else {
+          const entitySchemas = await Promise.all(
+            cachedObjectMetadataCollection.map((objectMetadata) =>
+              this.entitySchemaFactory.create(workspaceId, objectMetadata),
+            ),
+          );
+
+          await this.workspaceCacheStorageService.setORMEntitySchema(
+            workspaceId,
+            entitySchemas.map((entitySchema) => entitySchema.options),
+          );
+
+          cachedEntitySchemas = entitySchemas;
        }

-        const entities = await Promise.all(
-          cachedObjectMetadataCollection.map((objectMetadata) =>
-            this.entitySchemaFactory.create(workspaceId, objectMetadata),
-          ),
-        );
        const workspaceDataSource = new WorkspaceDataSource(
          {
            workspaceId,
@ -104,7 +137,7 @@ export class WorkspaceDatasourceFactory {
              ? ['query', 'error']
              : ['error'],
            schema: dataSourceMetadata.schema,
-            entities,
+            entities: cachedEntitySchemas,
            ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED')
              ? {
                  rejectUnauthorized: false,
--- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts
@ -14,8 +14,7 @@ import {

 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
-import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
-import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context';
+import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import { entitySchemaFactories } from 'src/engine/twenty-orm/factories';
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
@ -33,21 +32,15 @@ export const workspaceDataSourceCacheInstance =
  imports: [
    TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
    DataSourceModule,
-    WorkspaceCacheVersionModule,
    WorkspaceCacheStorageModule,
+    WorkspaceMetadataVersionModule,
  ],
  providers: [
    ...entitySchemaFactories,
    TwentyORMManager,
    TwentyORMGlobalManager,
-    LoadServiceWithWorkspaceContext,
-  ],
-  exports: [
-    EntitySchemaFactory,
-    TwentyORMManager,
-    LoadServiceWithWorkspaceContext,
-    TwentyORMGlobalManager,
  ],
+  exports: [EntitySchemaFactory, TwentyORMManager, TwentyORMGlobalManager],
 })
 export class TwentyORMCoreModule
  extends ConfigurableModuleClass
--- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts
@ -25,7 +25,7 @@ export class TwentyORMManager {
  async getRepository<T extends ObjectLiteral>(
    workspaceEntityOrobjectMetadataName: Type<T> | string,
  ): Promise<WorkspaceRepository<T>> {
-    const { workspaceId, cacheVersion } =
+    const { workspaceId, workspaceMetadataVersion } =
      this.scopedWorkspaceContextFactory.create();

    let objectMetadataName: string;
@ -44,20 +44,23 @@ export class TwentyORMManager {

    const workspaceDataSource = await this.workspaceDataSourceFactory.create(
      workspaceId,
-      cacheVersion,
+      workspaceMetadataVersion,
    );

    return workspaceDataSource.getRepository<T>(objectMetadataName);
  }

  async getDatasource() {
-    const { workspaceId, cacheVersion } =
+    const { workspaceId, workspaceMetadataVersion } =
      this.scopedWorkspaceContextFactory.create();

    if (!workspaceId) {
      throw new Error('Workspace not found');
    }

-    return this.workspaceDataSourceFactory.create(workspaceId, cacheVersion);
+    return this.workspaceDataSourceFactory.create(
+      workspaceId,
+      workspaceMetadataVersion,
+    );
  }
 }