Improve performance on metadata computation (#12785)

In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig

packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts

@@ -2,9 +2,12 @@ import DataLoader from 'dataloader';
 
 import {
   FieldMetadataLoaderPayload,
+  IndexMetadataLoaderPayload,
   RelationLoaderPayload,
 } from 'src/engine/dataloaders/dataloader.service';
+import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { IndexMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-metadata.dto';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 
 export interface IDataloaders {
@@ -20,6 +23,11 @@ export interface IDataloaders {
 
   fieldMetadataLoader: DataLoader<
     FieldMetadataLoaderPayload,
-    FieldMetadataEntity[]
+    FieldMetadataDTO[]
+  >;
+
+  indexMetadataLoader: DataLoader<
+    IndexMetadataLoaderPayload,
+    IndexMetadataDTO[]
   >;
 }

packages/twenty-server/src/engine/dataloaders/dataloader.module.ts

@@ -2,9 +2,10 @@ import { Module } from '@nestjs/common';
 
 import { DataloaderService } from 'src/engine/dataloaders/dataloader.service';
 import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
+import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
 
 @Module({
-  imports: [FieldMetadataModule],
+  imports: [FieldMetadataModule, WorkspaceMetadataCacheModule],
   providers: [DataloaderService],
   exports: [DataloaderService],
 })

packages/twenty-server/src/engine/dataloaders/dataloader.service.ts

@@ -6,10 +6,13 @@ import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metada
 import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
 
 import { IDataloaders } from 'src/engine/dataloaders/dataloader.interface';
+import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
 import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
 import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
 import { FieldMetadataRelationService } from 'src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service';
+import { IndexMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-metadata.dto';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 
 export type RelationMetadataLoaderPayload = {
   workspaceId: string;
@@ -36,20 +39,28 @@ export type FieldMetadataLoaderPayload = {
   objectMetadata: Pick<ObjectMetadataInterface, 'id'>;
 };
 
+export type IndexMetadataLoaderPayload = {
+  workspaceId: string;
+  objectMetadata: Pick<ObjectMetadataInterface, 'id'>;
+};
+
 @Injectable()
 export class DataloaderService {
   constructor(
     private readonly fieldMetadataRelationService: FieldMetadataRelationService,
     private readonly fieldMetadataService: FieldMetadataService,
+    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
   ) {}
 
   createLoaders(): IDataloaders {
     const relationLoader = this.createRelationLoader();
     const fieldMetadataLoader = this.createFieldMetadataLoader();
+    const indexMetadataLoader = this.createIndexMetadataLoader();
 
     return {
       relationLoader,
       fieldMetadataLoader,
+      indexMetadataLoader,
     };
   }
 
@@ -78,20 +89,67 @@ export class DataloaderService {
     });
   }
 
-  private createFieldMetadataLoader() {
-    return new DataLoader<FieldMetadataLoaderPayload, FieldMetadataEntity[]>(
-      async (dataLoaderParams: FieldMetadataLoaderPayload[]) => {
+  private createIndexMetadataLoader() {
+    return new DataLoader<IndexMetadataLoaderPayload, IndexMetadataDTO[]>(
+      async (dataLoaderParams: IndexMetadataLoaderPayload[]) => {
         const workspaceId = dataLoaderParams[0].workspaceId;
-        const objectMetadataItems = dataLoaderParams.map(
-          (dataLoaderParam) => dataLoaderParam.objectMetadata,
+        const objectMetadataIds = dataLoaderParams.map(
+          (dataLoaderParam) => dataLoaderParam.objectMetadata.id,
         );
 
-        const fieldMetadataCollection =
-          await this.fieldMetadataService.getFieldMetadataItemsByBatch(
-            objectMetadataItems.map((item) => item.id),
-            workspaceId,
+        const { objectMetadataMaps } =
+          await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
+            { workspaceId },
           );
 
+        const indexMetadataCollection = objectMetadataIds.map((id) =>
+          Object.values(objectMetadataMaps.byId[id].indexMetadatas).map(
+            (indexMetadata) => {
+              return {
+                ...indexMetadata,
+                createdAt: new Date(indexMetadata.createdAt),
+                updatedAt: new Date(indexMetadata.updatedAt),
+                id: indexMetadata.id,
+                indexWhereClause: indexMetadata.indexWhereClause ?? undefined,
+                objectMetadataId: id,
+                workspaceId: workspaceId,
+              };
+            },
+          ),
+        );
+
+        return indexMetadataCollection;
+      },
+    );
+  }
+
+  private createFieldMetadataLoader() {
+    return new DataLoader<FieldMetadataLoaderPayload, FieldMetadataDTO[]>(
+      async (dataLoaderParams: FieldMetadataLoaderPayload[]) => {
+        const workspaceId = dataLoaderParams[0].workspaceId;
+        const objectMetadataIds = dataLoaderParams.map(
+          (dataLoaderParam) => dataLoaderParam.objectMetadata.id,
+        );
+
+        const { objectMetadataMaps } =
+          await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
+            { workspaceId },
+          );
+
+        const fieldMetadataCollection = objectMetadataIds.map((id) =>
+          Object.values(objectMetadataMaps.byId[id].fieldsById).map(
+            // TODO: fix this as we should merge FieldMetadataEntity and FieldMetadataInterface
+            (fieldMetadata) => {
+              return {
+                ...fieldMetadata,
+                createdAt: new Date(fieldMetadata.createdAt),
+                updatedAt: new Date(fieldMetadata.updatedAt),
+                workspaceId: workspaceId,
+              };
+            },
+          ),
+        );
+
         return fieldMetadataCollection;
       },
     );