Refactor metadata caching (#7011)

This PR introduces the following changes:
- add the metadataVersion to all our metadata cache keys to ease
troubleshooting:
<img width="1146" alt="image"
src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12">
- introduce a cache recompute lock to avoid overloading the database to
recompute the cache many time

packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts

@@ -4,25 +4,27 @@ import { makeExecutableSchema } from '@graphql-tools/schema';
 import { GraphQLSchema, printSchema } from 'graphql';
 import { gql } from 'graphql-tag';
 
+import {
+  GraphqlQueryRunnerException,
+  GraphqlQueryRunnerExceptionCode,
+} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
 import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
 import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories';
 import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory';
 import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
 import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
-import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
-import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
+import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 @Injectable()
 export class WorkspaceSchemaFactory {
   constructor(
     private readonly dataSourceService: DataSourceService,
-    private readonly objectMetadataService: ObjectMetadataService,
     private readonly scalarsExplorerService: ScalarsExplorerService,
     private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory,
     private readonly workspaceResolverFactory: WorkspaceResolverFactory,
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
-    private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
+    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
   ) {}
 
   async createGraphQLSchema(authContext: AuthContext): Promise<GraphQLSchema> {
@@ -35,42 +37,50 @@ export class WorkspaceSchemaFactory {
         authContext.workspace.id,
       );
 
-    // Can'f find any data sources for this workspace
     if (!dataSourcesMetadata || dataSourcesMetadata.length === 0) {
       return new GraphQLSchema({});
     }
 
-    // Validate cache version
-    await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated(
-      authContext.workspace.id,
-    );
-
-    // Get object metadata from cache
-    let objectMetadataCollection =
-      await this.workspaceCacheStorageService.getObjectMetadataCollection(
+    const currentCacheVersion =
+      await this.workspaceCacheStorageService.getMetadataVersion(
         authContext.workspace.id,
       );
 
-    // If object metadata is not cached, get it from the database
-    if (!objectMetadataCollection) {
-      objectMetadataCollection =
-        await this.objectMetadataService.findManyWithinWorkspace(
-          authContext.workspace.id,
-        );
-
-      await this.workspaceCacheStorageService.setObjectMetadataCollection(
+    if (currentCacheVersion === undefined) {
+      await this.workspaceMetadataCacheService.recomputeMetadataCache(
         authContext.workspace.id,
-        objectMetadataCollection,
+      );
+      throw new GraphqlQueryRunnerException(
+        'Metadata cache version not found',
+        GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
+      );
+    }
+
+    const objectMetadataCollection =
+      await this.workspaceCacheStorageService.getObjectMetadataCollection(
+        authContext.workspace.id,
+        currentCacheVersion,
+      );
+
+    if (!objectMetadataCollection) {
+      await this.workspaceMetadataCacheService.recomputeMetadataCache(
+        authContext.workspace.id,
+      );
+      throw new GraphqlQueryRunnerException(
+        'Object metadata collection not found',
+        GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
       );
     }
 
     // Get typeDefs from cache
     let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs(
       authContext.workspace.id,
+      currentCacheVersion,
     );
     let usedScalarNames =
       await this.workspaceCacheStorageService.getGraphQLUsedScalarNames(
         authContext.workspace.id,
+        currentCacheVersion,
       );
 
     // If typeDefs are not cached, generate them
@@ -87,10 +97,12 @@ export class WorkspaceSchemaFactory {
 
       await this.workspaceCacheStorageService.setGraphQLTypeDefs(
         authContext.workspace.id,
+        currentCacheVersion,
         typeDefs,
       );
       await this.workspaceCacheStorageService.setGraphQLUsedScalarNames(
         authContext.workspace.id,
+        currentCacheVersion,
         usedScalarNames,
       );
     }