Refactor metadata caching (#7011)

This PR introduces the following changes:
- add the metadataVersion to all our metadata cache keys to ease
troubleshooting:
<img width="1146" alt="image"
src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12">
- introduce a cache recompute lock to avoid overloading the database to
recompute the cache many time

packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts

@@ -6,7 +6,7 @@ import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-
 import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
 import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
-import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
+import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 
 describe('WorkspaceSchemaFactory', () => {
@@ -41,7 +41,7 @@ describe('WorkspaceSchemaFactory', () => {
           useValue: {},
         },
         {
-          provide: WorkspaceMetadataVersionService,
+          provide: WorkspaceMetadataCacheService,
           useValue: {},
         },
       ],

packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts

@@ -4,25 +4,18 @@ import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-
 import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module';
 import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module';
 import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module';
-import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
+import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
 
 import { WorkspaceSchemaFactory } from './workspace-schema.factory';
 
 @Module({
   imports: [
-    // TODO: Seems like it's breaking /metadata query and mutation arguments
-    // we should investigate this issue
-    // GraphQLModule.forRootAsync<YogaDriverConfig>({
-    //   driver: YogaDriver,
-    //   imports: [CoreEngineModule, GraphQLConfigModule],
-    //   useClass: GraphQLConfigService,
-    // }),
     MetadataEngineModule,
     WorkspaceSchemaBuilderModule,
     WorkspaceResolverBuilderModule,
     WorkspaceCacheStorageModule,
-    WorkspaceMetadataVersionModule,
+    WorkspaceMetadataCacheModule,
   ],
   providers: [WorkspaceSchemaFactory, ScalarsExplorerService],
   exports: [WorkspaceSchemaFactory],

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts

@@ -19,4 +19,5 @@ export enum GraphqlQueryRunnerExceptionCode {
   RECORD_NOT_FOUND = 'RECORD_NOT_FOUND',
   INVALID_ARGS_FIRST = 'INVALID_ARGS_FIRST',
   INVALID_ARGS_LAST = 'INVALID_ARGS_LAST',
+  METADATA_CACHE_VERSION_NOT_FOUND = 'METADATA_CACHE_VERSION_NOT_FOUND',
 }

packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts

@@ -4,25 +4,27 @@ import { makeExecutableSchema } from '@graphql-tools/schema';
 import { GraphQLSchema, printSchema } from 'graphql';
 import { gql } from 'graphql-tag';
 
+import {
+  GraphqlQueryRunnerException,
+  GraphqlQueryRunnerExceptionCode,
+} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
 import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
 import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories';
 import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory';
 import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
 import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
-import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
-import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
+import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 @Injectable()
 export class WorkspaceSchemaFactory {
   constructor(
     private readonly dataSourceService: DataSourceService,
-    private readonly objectMetadataService: ObjectMetadataService,
     private readonly scalarsExplorerService: ScalarsExplorerService,
     private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory,
     private readonly workspaceResolverFactory: WorkspaceResolverFactory,
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
-    private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
+    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
   ) {}
 
   async createGraphQLSchema(authContext: AuthContext): Promise<GraphQLSchema> {
@@ -35,42 +37,50 @@ export class WorkspaceSchemaFactory {
         authContext.workspace.id,
       );
 
-    // Can'f find any data sources for this workspace
     if (!dataSourcesMetadata || dataSourcesMetadata.length === 0) {
       return new GraphQLSchema({});
     }
 
-    // Validate cache version
-    await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated(
-      authContext.workspace.id,
-    );
-
-    // Get object metadata from cache
-    let objectMetadataCollection =
-      await this.workspaceCacheStorageService.getObjectMetadataCollection(
+    const currentCacheVersion =
+      await this.workspaceCacheStorageService.getMetadataVersion(
         authContext.workspace.id,
       );
 
-    // If object metadata is not cached, get it from the database
-    if (!objectMetadataCollection) {
-      objectMetadataCollection =
-        await this.objectMetadataService.findManyWithinWorkspace(
-          authContext.workspace.id,
-        );
-
-      await this.workspaceCacheStorageService.setObjectMetadataCollection(
+    if (currentCacheVersion === undefined) {
+      await this.workspaceMetadataCacheService.recomputeMetadataCache(
         authContext.workspace.id,
-        objectMetadataCollection,
+      );
+      throw new GraphqlQueryRunnerException(
+        'Metadata cache version not found',
+        GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
+      );
+    }
+
+    const objectMetadataCollection =
+      await this.workspaceCacheStorageService.getObjectMetadataCollection(
+        authContext.workspace.id,
+        currentCacheVersion,
+      );
+
+    if (!objectMetadataCollection) {
+      await this.workspaceMetadataCacheService.recomputeMetadataCache(
+        authContext.workspace.id,
+      );
+      throw new GraphqlQueryRunnerException(
+        'Object metadata collection not found',
+        GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
       );
     }
 
     // Get typeDefs from cache
     let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs(
       authContext.workspace.id,
+      currentCacheVersion,
     );
     let usedScalarNames =
       await this.workspaceCacheStorageService.getGraphQLUsedScalarNames(
         authContext.workspace.id,
+        currentCacheVersion,
       );
 
     // If typeDefs are not cached, generate them
@@ -87,10 +97,12 @@ export class WorkspaceSchemaFactory {
 
       await this.workspaceCacheStorageService.setGraphQLTypeDefs(
         authContext.workspace.id,
+        currentCacheVersion,
         typeDefs,
       );
       await this.workspaceCacheStorageService.setGraphQLUsedScalarNames(
         authContext.workspace.id,
+        currentCacheVersion,
         usedScalarNames,
       );
     }