Improve metadata version caching (#11775)

Investigating https://github.com/twentyhq/core-team-issues/issues/756, I
found that the error actually stemmed from "Object metadata collection
not found" error.

While this is planned to be fixed by metadata performance improvements
(as stated in [sentry-boss
doc](https://docs.google.com/document/d/1ytbC5W6ZFUSJ3PoJ4IfKi2IehKZYw65mqCnc24aP4RM/edit?tab=t.0)
in "known issues"), I tried some easy improvements to reduce the number
of errors.

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts

@@ -23,4 +23,5 @@ export enum GraphqlQueryRunnerExceptionCode {
   RELATION_SETTINGS_NOT_FOUND = 'RELATION_SETTINGS_NOT_FOUND',
   RELATION_TARGET_OBJECT_METADATA_NOT_FOUND = 'RELATION_TARGET_OBJECT_METADATA_NOT_FOUND',
   NOT_IMPLEMENTED = 'NOT_IMPLEMENTED',
+  OBJECT_METADATA_COLLECTION_NOT_FOUND = 'OBJECT_METADATA_COLLECTION_NOT_FOUND',
 }

packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts

@@ -4,6 +4,7 @@ import { makeExecutableSchema } from '@graphql-tools/schema';
 import chalk from 'chalk';
 import { GraphQLSchema, printSchema } from 'graphql';
 import { gql } from 'graphql-tag';
+import { isDefined } from 'twenty-shared/utils';
 
 import {
   GraphqlQueryRunnerException,
@@ -17,6 +18,7 @@ import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.typ
 import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
 import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 
@@ -58,34 +60,49 @@ export class WorkspaceSchemaFactory {
       return new GraphQLSchema({});
     }
 
-    const currentCacheVersion =
+    let currentCacheVersion =
       await this.workspaceCacheStorageService.getMetadataVersion(
         authContext.workspace.id,
       );
 
-    if (currentCacheVersion === undefined) {
+    let objectMetadataMaps: ObjectMetadataMaps | undefined;
-      await this.workspaceMetadataCacheService.recomputeMetadataCache({
-        workspaceId: authContext.workspace.id,
-      });
 
+    if (currentCacheVersion === undefined) {
+      const recomputed =
+        await this.workspaceMetadataCacheService.recomputeMetadataCache({
+          workspaceId: authContext.workspace.id,
+        });
+
+      objectMetadataMaps = recomputed?.recomputedObjectMetadataMaps;
+      currentCacheVersion = recomputed?.recomputedMetadataVersion;
+    } else {
+      objectMetadataMaps =
+        await this.workspaceCacheStorageService.getObjectMetadataMaps(
+          authContext.workspace.id,
+          currentCacheVersion,
+        );
+
+      if (!isDefined(objectMetadataMaps)) {
+        const recomputed =
+          await this.workspaceMetadataCacheService.recomputeMetadataCache({
+            workspaceId: authContext.workspace.id,
+          });
+
+        objectMetadataMaps = recomputed?.recomputedObjectMetadataMaps;
+        currentCacheVersion = recomputed?.recomputedMetadataVersion;
+      }
+    }
+
+    if (!objectMetadataMaps) {
       throw new GraphqlQueryRunnerException(
-        'Metadata cache version not found',
+        'Object metadata collection not found',
-        GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
+        GraphqlQueryRunnerExceptionCode.OBJECT_METADATA_COLLECTION_NOT_FOUND,
       );
     }
 
-    const objectMetadataMaps =
+    if (!currentCacheVersion) {
-      await this.workspaceCacheStorageService.getObjectMetadataMaps(
-        authContext.workspace.id,
-        currentCacheVersion,
-      );
-
-    if (!objectMetadataMaps) {
-      await this.workspaceMetadataCacheService.recomputeMetadataCache({
-        workspaceId: authContext.workspace.id,
-      });
       throw new GraphqlQueryRunnerException(
-        'Object metadata collection not found',
+        'Metadata cache version not found',
         GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
       );
     }

packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts

@@ -5,6 +5,7 @@ import { Repository } from 'typeorm';
 
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { generateObjectMetadataMaps } from 'src/engine/metadata-modules/utils/generate-object-metadata-maps.util';
 import {
   WorkspaceMetadataCacheException,
@@ -30,7 +31,13 @@ export class WorkspaceMetadataCacheService {
   }: {
     workspaceId: string;
     ignoreLock?: boolean;
-  }): Promise<void> {
+  }): Promise<
+    | {
+        recomputedObjectMetadataMaps: ObjectMetadataMaps;
+        recomputedMetadataVersion: number;
+      }
+    | undefined
+  > {
     const currentCacheVersion =
       await this.getMetadataVersionFromCache(workspaceId);
 
@@ -44,16 +51,22 @@ export class WorkspaceMetadataCacheService {
       );
     }
 
-    const isAlreadyCaching =
+    if (currentDatabaseVersion === currentCacheVersion) {
-      await this.workspaceCacheStorageService.getObjectMetadataOngoingCachingLock(
-        workspaceId,
-        currentDatabaseVersion,
-      );
-
-    if (!ignoreLock && isAlreadyCaching) {
       return;
     }
 
+    if (!ignoreLock) {
+      const isAlreadyCaching =
+        await this.workspaceCacheStorageService.getObjectMetadataOngoingCachingLock(
+          workspaceId,
+          currentDatabaseVersion,
+        );
+
+      if (isAlreadyCaching) {
+        return;
+      }
+    }
+
     if (currentCacheVersion !== undefined) {
       this.workspaceCacheStorageService.flushVersionedMetadata(
         workspaceId,
@@ -61,40 +74,47 @@ export class WorkspaceMetadataCacheService {
       );
     }
 
-    await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock(
+    try {
-      workspaceId,
+      await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock(
-      currentDatabaseVersion,
+        workspaceId,
-    );
+        currentDatabaseVersion,
+      );
 
-    await this.workspaceCacheStorageService.setMetadataVersion(
+      const objectMetadataItems = await this.objectMetadataRepository.find({
-      workspaceId,
+        where: { workspaceId },
-      currentDatabaseVersion,
+        relations: [
-    );
+          'fields',
+          'fields.fromRelationMetadata',
+          'fields.toRelationMetadata',
+          'indexMetadatas',
+          'indexMetadatas.indexFieldMetadatas',
+        ],
+      });
 
-    const objectMetadataItems = await this.objectMetadataRepository.find({
+      const freshObjectMetadataMaps =
-      where: { workspaceId },
+        generateObjectMetadataMaps(objectMetadataItems);
-      relations: [
-        'fields',
-        'fields.fromRelationMetadata',
-        'fields.toRelationMetadata',
-        'indexMetadatas',
-        'indexMetadatas.indexFieldMetadatas',
-      ],
-    });
 
-    const freshObjectMetadataMaps =
+      await this.workspaceCacheStorageService.setObjectMetadataMaps(
-      generateObjectMetadataMaps(objectMetadataItems);
+        workspaceId,
+        currentDatabaseVersion,
+        freshObjectMetadataMaps,
+      );
 
-    await this.workspaceCacheStorageService.setObjectMetadataMaps(
+      await this.workspaceCacheStorageService.setMetadataVersion(
-      workspaceId,
+        workspaceId,
-      currentDatabaseVersion,
+        currentDatabaseVersion,
-      freshObjectMetadataMaps,
+      );
-    );
 
-    await this.workspaceCacheStorageService.removeObjectMetadataOngoingCachingLock(
+      return {
-      workspaceId,
+        recomputedObjectMetadataMaps: freshObjectMetadataMaps,
-      currentDatabaseVersion,
+        recomputedMetadataVersion: currentDatabaseVersion,
-    );
+      };
+    } finally {
+      await this.workspaceCacheStorageService.removeObjectMetadataOngoingCachingLock(
+        workspaceId,
+        currentDatabaseVersion,
+      );
+    }
   }
 
   private async getMetadataVersionFromDatabase(