Improve performance on metadata computation (#12785)

In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig

packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-column.factory.ts

@@ -12,7 +12,7 @@ import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-me
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
 import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
 import { serializeDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-default-value';
-import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
 import {
   TwentyORMException,
@@ -26,10 +26,14 @@ type EntitySchemaColumnMap = {
 
 @Injectable()
 export class EntitySchemaColumnFactory {
-  create(fieldMetadataMapByName: FieldMetadataMap): EntitySchemaColumnMap {
+  create(
+    objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
+  ): EntitySchemaColumnMap {
     let entitySchemaColumnMap: EntitySchemaColumnMap = {};
 
-    const fieldMetadataCollection = Object.values(fieldMetadataMapByName);
+    const fieldMetadataCollection = Object.values(
+      objectMetadataItemWithFieldMaps.fieldsById,
+    );
 
     for (const fieldMetadata of fieldMetadataCollection) {
       const key = fieldMetadata.name;

packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts

@@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';
 import { FieldMetadataType } from 'twenty-shared/types';
 import { EntitySchemaRelationOptions } from 'typeorm';
 
-import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { determineSchemaRelationDetails } from 'src/engine/twenty-orm/utils/determine-schema-relation-details.util';
 import { isFieldMetadataInterfaceOfType } from 'src/engine/utils/is-field-metadata-of-type.util';
@@ -17,12 +17,14 @@ export class EntitySchemaRelationFactory {
   constructor() {}
 
   async create(
-    fieldMetadataMapByName: FieldMetadataMap,
+    objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
     objectMetadataMaps: ObjectMetadataMaps,
   ): Promise<EntitySchemaRelationMap> {
     const entitySchemaRelationMap: EntitySchemaRelationMap = {};
 
-    const fieldMetadataCollection = Object.values(fieldMetadataMapByName);
+    const fieldMetadataCollection = Object.values(
+      objectMetadataItemWithFieldMaps.fieldsById,
+    );
 
     for (const fieldMetadata of fieldMetadataCollection) {
       if (

packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts

@@ -24,12 +24,10 @@ export class EntitySchemaFactory {
     objectMetadata: ObjectMetadataItemWithFieldMaps,
     objectMetadataMaps: ObjectMetadataMaps,
   ): Promise<EntitySchema> {
-    const columns = this.entitySchemaColumnFactory.create(
-      objectMetadata.fieldsByName,
-    );
+    const columns = this.entitySchemaColumnFactory.create(objectMetadata);
 
     const relations = await this.entitySchemaRelationFactory.create(
-      objectMetadata.fieldsByName,
+      objectMetadata,
       objectMetadataMaps,
     );
 

packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts

@@ -11,7 +11,6 @@ export class ScopedWorkspaceContextFactory {
 
   public create(): {
     workspaceId: string | null;
-    workspaceMetadataVersion: number | null;
     userWorkspaceId: string | null;
     isExecutedByApiKey: boolean;
   } {
@@ -22,13 +21,9 @@ export class ScopedWorkspaceContextFactory {
       this.request?.['params']?.['workspaceId'] ||
       // @ts-expect-error legacy noImplicitAny
       this.request?.['workspace']?.['id']; // rest api
-    const workspaceMetadataVersion: number | undefined =
-      // @ts-expect-error legacy noImplicitAny
-      this.request?.['req']?.['workspaceMetadataVersion'];
 
     return {
       workspaceId: workspaceId ?? null,
-      workspaceMetadataVersion: workspaceMetadataVersion ?? null,
       userWorkspaceId:
         // @ts-expect-error legacy noImplicitAny
         this.request?.['req']?.['userWorkspaceId'] ??

packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts

@@ -1,23 +1,17 @@
 import { Injectable, Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
 
 import { ObjectRecordsPermissionsByRoleId } from 'twenty-shared/types';
 import { isDefined } from 'twenty-shared/utils';
-import { EntitySchema } from 'typeorm';
+import { EntitySchema, Repository } from 'typeorm';
 
 import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
 
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service';
-import {
-  WorkspaceMetadataCacheException,
-  WorkspaceMetadataCacheExceptionCode,
-} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception';
 import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
-import {
-  WorkspaceMetadataVersionException,
-  WorkspaceMetadataVersionExceptionCode,
-} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception';
 import { WorkspacePermissionsCacheStorageService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache-storage.service';
 import {
   ROLES_PERMISSIONS,
@@ -55,6 +49,8 @@ export class WorkspaceDatasourceFactory {
     private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
     private readonly workspacePermissionsCacheStorageService: WorkspacePermissionsCacheStorageService,
     private readonly workspaceFeatureFlagsMapCacheService: WorkspaceFeatureFlagsMapCacheService,
+    @InjectRepository(Workspace, 'core')
+    private readonly workspaceRepository: Repository<Workspace>,
   ) {}
 
   private async conditionalDestroyDataSource(
@@ -103,16 +99,9 @@ export class WorkspaceDatasourceFactory {
     }
   }
 
-  public async create(
-    workspaceId: string,
-    workspaceMetadataVersion: number | null,
-    shouldFailIfMetadataNotFound = true,
-  ): Promise<WorkspaceDataSource> {
-    const cachedWorkspaceMetadataVersion =
-      await this.getWorkspaceMetadataVersionFromCache(
-        workspaceId,
-        shouldFailIfMetadataNotFound,
-      );
+  public async create(workspaceId: string): Promise<WorkspaceDataSource> {
+    const dataSourceMetadataVersion =
+      await this.getWorkspaceMetadataVersionFromCacheOrFromDB(workspaceId);
 
     const { data: cachedFeatureFlagMap, version: cachedFeatureFlagMapVersion } =
       await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion(
@@ -126,17 +115,7 @@ export class WorkspaceDatasourceFactory {
       workspaceId,
     });
 
-    if (
-      workspaceMetadataVersion !== null &&
-      cachedWorkspaceMetadataVersion !== workspaceMetadataVersion
-    ) {
-      throw new TwentyORMException(
-        `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${cachedWorkspaceMetadataVersion}. Desired version: ${workspaceMetadataVersion}`,
-        TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
-      );
-    }
-
-    const cacheKey: CacheKey = `${workspaceId}-${cachedWorkspaceMetadataVersion}`;
+    const cacheKey: CacheKey = `${workspaceId}-${dataSourceMetadataVersion}`;
 
     const workspaceDataSource =
       await this.promiseMemoizer.memoizePromiseAndExecute(
@@ -157,21 +136,27 @@ export class WorkspaceDatasourceFactory {
           const cachedEntitySchemaOptions =
             await this.workspaceCacheStorageService.getORMEntitySchema(
               workspaceId,
-              cachedWorkspaceMetadataVersion,
+              dataSourceMetadataVersion,
             );
 
           let cachedEntitySchemas: EntitySchema[];
 
-          const cachedObjectMetadataMaps =
-            await this.workspaceCacheStorageService.getObjectMetadataMaps(
-              workspaceId,
-              cachedWorkspaceMetadataVersion,
+          const {
+            objectMetadataMaps: cachedObjectMetadataMaps,
+            metadataVersion: metadataVersionForFinalUpToDateCheck,
+          } =
+            await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
+              {
+                workspaceId,
+              },
             );
 
-          if (!cachedObjectMetadataMaps) {
-            throw new WorkspaceMetadataCacheException(
-              `Object metadata collection not found for workspace ${workspaceId}`,
-              WorkspaceMetadataCacheExceptionCode.OBJECT_METADATA_COLLECTION_NOT_FOUND,
+          if (
+            metadataVersionForFinalUpToDateCheck !== dataSourceMetadataVersion
+          ) {
+            throw new TwentyORMException(
+              `Workspace metadata version mismatch detected for workspace ${workspaceId}. Latest version: ${metadataVersionForFinalUpToDateCheck}. Built version: ${dataSourceMetadataVersion}`,
+              TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
             );
           }
 
@@ -185,7 +170,7 @@ export class WorkspaceDatasourceFactory {
                 (objectMetadata) =>
                   this.entitySchemaFactory.create(
                     workspaceId,
-                    cachedWorkspaceMetadataVersion,
+                    dataSourceMetadataVersion,
                     objectMetadata,
                     cachedObjectMetadataMaps,
                   ),
@@ -194,7 +179,7 @@ export class WorkspaceDatasourceFactory {
 
             await this.workspaceCacheStorageService.setORMEntitySchema(
               workspaceId,
-              cachedWorkspaceMetadataVersion,
+              dataSourceMetadataVersion,
               entitySchemas.map((entitySchema) => entitySchema.options),
             );
 
@@ -354,39 +339,28 @@ export class WorkspaceDatasourceFactory {
     });
   }
 
-  private async getWorkspaceMetadataVersionFromCache(
+  private async getWorkspaceMetadataVersionFromCacheOrFromDB(
     workspaceId: string,
-    shouldFailIfMetadataNotFound = true,
   ): Promise<number> {
-    let latestWorkspaceMetadataVersion =
+    const latestWorkspaceMetadataVersion =
       await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
 
-    if (!isDefined(latestWorkspaceMetadataVersion)) {
-      if (shouldFailIfMetadataNotFound) {
-        throw new WorkspaceMetadataVersionException(
-          `Metadata version not found while fetching datasource for workspace ${workspaceId}`,
-          WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
-        );
-      } else {
-        await this.workspaceMetadataCacheService.recomputeMetadataCache({
-          workspaceId,
-          ignoreLock: !shouldFailIfMetadataNotFound,
-        });
-        latestWorkspaceMetadataVersion =
-          await this.workspaceCacheStorageService.getMetadataVersion(
-            workspaceId,
-          );
-      }
+    if (isDefined(latestWorkspaceMetadataVersion)) {
+      return latestWorkspaceMetadataVersion;
     }
 
-    if (!isDefined(latestWorkspaceMetadataVersion)) {
-      throw new WorkspaceMetadataVersionException(
-        `Metadata version not found after recompute`,
-        WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
+    const workspace = await this.workspaceRepository.findOne({
+      where: { id: workspaceId },
+    });
+
+    if (!workspace) {
+      throw new TwentyORMException(
+        `Workspace not found for workspace ${workspaceId}`,
+        TwentyORMExceptionCode.WORKSPACE_NOT_FOUND,
       );
     }
 
-    return latestWorkspaceMetadataVersion;
+    return workspace.metadataVersion;
   }
 
   public async destroy(workspaceId: string) {