Refactor metadata caching (#7011)

This PR introduces the following changes:
- add the metadataVersion to all our metadata cache keys to ease
troubleshooting:
<img width="1146" alt="image"
src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12">
- introduce a cache recompute lock to avoid overloading the database to
recompute the cache many time

packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts

@@ -0,0 +1,12 @@
+import { CustomException } from 'src/utils/custom-exception';
+
+export class WorkspaceMetadataCacheException extends CustomException {
+  code: WorkspaceMetadataCacheExceptionCode;
+  constructor(message: string, code: WorkspaceMetadataCacheExceptionCode) {
+    super(message, code);
+  }
+}
+
+export enum WorkspaceMetadataCacheExceptionCode {
+  METADATA_VERSION_NOT_FOUND = 'METADATA_VERSION_NOT_FOUND',
+}

packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts

@@ -0,0 +1,112 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Repository } from 'typeorm';
+
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import {
+  WorkspaceMetadataCacheException,
+  WorkspaceMetadataCacheExceptionCode,
+} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception';
+import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
+
+@Injectable()
+export class WorkspaceMetadataCacheService {
+  logger = new Logger(WorkspaceMetadataCacheService.name);
+
+  constructor(
+    @InjectRepository(Workspace, 'core')
+    private readonly workspaceRepository: Repository<Workspace>,
+    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
+    @InjectRepository(ObjectMetadataEntity, 'metadata')
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+  ) {}
+
+  async recomputeMetadataCache(
+    workspaceId: string,
+    force = false,
+  ): Promise<void> {
+    const currentCacheVersion =
+      await this.getMetadataVersionFromCache(workspaceId);
+
+    const currentDatabaseVersion =
+      await this.getMetadataVersionFromDatabase(workspaceId);
+
+    if (currentDatabaseVersion === undefined) {
+      throw new WorkspaceMetadataCacheException(
+        'Metadata version not found in the database',
+        WorkspaceMetadataCacheExceptionCode.METADATA_VERSION_NOT_FOUND,
+      );
+    }
+
+    if (!force && currentCacheVersion === currentDatabaseVersion) {
+      return;
+    }
+
+    const isAlreadyCaching =
+      await this.workspaceCacheStorageService.getObjectMetadataCollectionOngoingCachingLock(
+        workspaceId,
+        currentDatabaseVersion,
+      );
+
+    if (isAlreadyCaching) {
+      return;
+    }
+
+    if (currentCacheVersion !== undefined) {
+      this.workspaceCacheStorageService.flush(workspaceId, currentCacheVersion);
+    }
+
+    await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock(
+      workspaceId,
+      currentDatabaseVersion,
+    );
+
+    await this.workspaceCacheStorageService.setMetadataVersion(
+      workspaceId,
+      currentDatabaseVersion,
+    );
+
+    const freshObjectMetadataCollection =
+      await this.objectMetadataRepository.find({
+        where: { workspaceId },
+        relations: [
+          'fields.object',
+          'fields',
+          'fields.fromRelationMetadata',
+          'fields.toRelationMetadata',
+          'fields.fromRelationMetadata.toObjectMetadata',
+        ],
+      });
+
+    await this.workspaceCacheStorageService.setObjectMetadataCollection(
+      workspaceId,
+      currentDatabaseVersion,
+      freshObjectMetadataCollection,
+    );
+
+    await this.workspaceCacheStorageService.removeObjectMetadataCollectionOngoingCachingLock(
+      workspaceId,
+      currentDatabaseVersion,
+    );
+  }
+
+  private async getMetadataVersionFromDatabase(
+    workspaceId: string,
+  ): Promise<number | undefined> {
+    const workspace = await this.workspaceRepository.findOne({
+      where: { id: workspaceId },
+    });
+
+    return workspace?.metadataVersion;
+  }
+
+  private async getMetadataVersionFromCache(
+    workspaceId: string,
+  ): Promise<number | undefined> {
+    return await this.workspaceCacheStorageService.getMetadataVersion(
+      workspaceId,
+    );
+  }
+}

packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts

@@ -0,0 +1,18 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
+import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature([Workspace], 'core'),
+    TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'),
+    WorkspaceCacheStorageModule,
+  ],
+  exports: [WorkspaceMetadataCacheService],
+  providers: [WorkspaceMetadataCacheService],
+})
+export class WorkspaceMetadataCacheModule {}