Improve performance on metadata computation (#12785)

In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig

packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts

@@ -2,9 +2,10 @@ import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 
 import { isDefined } from 'twenty-shared/utils';
-import { Repository } from 'typeorm';
+import { In, Repository } from 'typeorm';
 
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { generateObjectMetadataMaps } from 'src/engine/metadata-modules/utils/generate-object-metadata-maps.util';
@@ -14,6 +15,11 @@ import {
 } from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 
+type getExistingOrRecomputeMetadataMapsResult = {
+  objectMetadataMaps: ObjectMetadataMaps;
+  metadataVersion: number;
+};
+
 @Injectable()
 export class WorkspaceMetadataCacheService {
   logger = new Logger(WorkspaceMetadataCacheService.name);
@@ -24,21 +30,15 @@ export class WorkspaceMetadataCacheService {
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
     @InjectRepository(ObjectMetadataEntity, 'core')
     private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    @InjectRepository(IndexMetadataEntity, 'core')
+    private readonly indexMetadataRepository: Repository<IndexMetadataEntity>,
   ) {}
 
-  async recomputeMetadataCache({
+  async getExistingOrRecomputeMetadataMaps({
     workspaceId,
-    ignoreLock = false,
   }: {
     workspaceId: string;
-    ignoreLock?: boolean;
-  }): Promise<
-    | {
-        recomputedObjectMetadataMaps: ObjectMetadataMaps;
-        recomputedMetadataVersion: number;
-      }
-    | undefined
-  > {
+  }): Promise<getExistingOrRecomputeMetadataMapsResult> {
     const currentCacheVersion =
       await this.getMetadataVersionFromCache(workspaceId);
 
@@ -52,68 +52,94 @@ export class WorkspaceMetadataCacheService {
       );
     }
 
-    if (currentDatabaseVersion === currentCacheVersion) {
-      return;
-    }
+    const shouldRecompute =
+      !isDefined(currentCacheVersion) ||
+      currentCacheVersion !== currentDatabaseVersion;
 
-    if (!ignoreLock) {
-      const isAlreadyCaching =
-        await this.workspaceCacheStorageService.getObjectMetadataOngoingCachingLock(
-          workspaceId,
-          currentDatabaseVersion,
-        );
-
-      if (isAlreadyCaching) {
-        return;
-      }
-    }
-
-    if (currentCacheVersion !== undefined) {
-      this.workspaceCacheStorageService.flushVersionedMetadata(
-        workspaceId,
-        currentCacheVersion,
-      );
-    }
-
-    try {
-      await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock(
+    const existingObjectMetadataMaps =
+      await this.workspaceCacheStorageService.getObjectMetadataMaps(
         workspaceId,
         currentDatabaseVersion,
       );
 
-      const objectMetadataItems = await this.objectMetadataRepository.find({
-        where: { workspaceId },
-        relations: [
-          'fields',
-          'indexMetadatas',
-          'indexMetadatas.indexFieldMetadatas',
-        ],
+    if (isDefined(existingObjectMetadataMaps) && !shouldRecompute) {
+      return {
+        objectMetadataMaps: existingObjectMetadataMaps,
+        metadataVersion: currentDatabaseVersion,
+      };
+    }
+
+    const { objectMetadataMaps, metadataVersion } =
+      await this.recomputeMetadataCache({
+        workspaceId,
       });
 
-      const freshObjectMetadataMaps =
-        generateObjectMetadataMaps(objectMetadataItems);
+    return {
+      objectMetadataMaps,
+      metadataVersion,
+    };
+  }
 
-      await this.workspaceCacheStorageService.setObjectMetadataMaps(
-        workspaceId,
-        currentDatabaseVersion,
-        freshObjectMetadataMaps,
-      );
+  async recomputeMetadataCache({
+    workspaceId,
+  }: {
+    workspaceId: string;
+  }): Promise<getExistingOrRecomputeMetadataMapsResult> {
+    const currentDatabaseVersion =
+      await this.getMetadataVersionFromDatabase(workspaceId);
 
-      await this.workspaceCacheStorageService.setMetadataVersion(
-        workspaceId,
-        currentDatabaseVersion,
-      );
-
-      return {
-        recomputedObjectMetadataMaps: freshObjectMetadataMaps,
-        recomputedMetadataVersion: currentDatabaseVersion,
-      };
-    } finally {
-      await this.workspaceCacheStorageService.removeObjectMetadataOngoingCachingLock(
-        workspaceId,
-        currentDatabaseVersion,
+    if (!isDefined(currentDatabaseVersion)) {
+      throw new WorkspaceMetadataVersionException(
+        'Metadata version not found in the database',
+        WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
       );
     }
+
+    await this.workspaceCacheStorageService.flushVersionedMetadata(workspaceId);
+
+    const objectMetadataItems = await this.objectMetadataRepository.find({
+      where: { workspaceId },
+      relations: ['fields'],
+    });
+
+    const objectMetadataItemsIds = objectMetadataItems.map(
+      (objectMetadataItem) => objectMetadataItem.id,
+    );
+
+    const indexMetadataItems = await this.indexMetadataRepository.find({
+      where: { objectMetadataId: In(objectMetadataItemsIds) },
+      relations: ['indexFieldMetadatas'],
+    });
+
+    const objectMetadataItemsWithIndexMetadatas = objectMetadataItems.map(
+      (objectMetadataItem) => ({
+        ...objectMetadataItem,
+        indexMetadatas: indexMetadataItems.filter(
+          (indexMetadataItem) =>
+            indexMetadataItem.objectMetadataId === objectMetadataItem.id,
+        ),
+      }),
+    );
+
+    const freshObjectMetadataMaps = generateObjectMetadataMaps(
+      objectMetadataItemsWithIndexMetadatas,
+    );
+
+    await this.workspaceCacheStorageService.setObjectMetadataMaps(
+      workspaceId,
+      currentDatabaseVersion,
+      freshObjectMetadataMaps,
+    );
+
+    await this.workspaceCacheStorageService.setMetadataVersion(
+      workspaceId,
+      currentDatabaseVersion,
+    );
+
+    return {
+      objectMetadataMaps: freshObjectMetadataMaps,
+      metadataVersion: currentDatabaseVersion,
+    };
   }
 
   private async getMetadataVersionFromDatabase(

packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts

@@ -2,14 +2,17 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
 
 @Module({
   imports: [
-    TypeOrmModule.forFeature([Workspace], 'core'),
-    TypeOrmModule.forFeature([ObjectMetadataEntity], 'core'),
+    TypeOrmModule.forFeature(
+      [Workspace, ObjectMetadataEntity, IndexMetadataEntity],
+      'core',
+    ),
     WorkspaceCacheStorageModule,
   ],
   exports: [WorkspaceMetadataCacheService],