Refactor metadata caching (#7011)

This PR introduces the following changes:
- add the metadataVersion to all our metadata cache keys to ease
troubleshooting:
<img width="1146" alt="image"
src="https://github.com/user-attachments/assets/8427805b-e07f-465e-9e69-1403652c8b12">
- introduce a cache recompute lock to avoid overloading the database to
recompute the cache many time

packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts

@@ -19,6 +19,7 @@ export class EntitySchemaRelationFactory {
 
   async create(
     workspaceId: string,
+    metadataVersion: number,
     fieldMetadataCollection: FieldMetadataEntity[],
   ): Promise<EntitySchemaRelationMap> {
     const entitySchemaRelationMap: EntitySchemaRelationMap = {};
@@ -40,6 +41,7 @@ export class EntitySchemaRelationFactory {
       const objectMetadataCollection =
         await this.workspaceCacheStorageService.getObjectMetadataCollection(
           workspaceId,
+          metadataVersion,
         );
 
       if (!objectMetadataCollection) {

packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts

@@ -17,6 +17,7 @@ export class EntitySchemaFactory {
 
   async create(
     workspaceId: string,
+    metadataVersion: number,
     objectMetadata: ObjectMetadataEntity,
   ): Promise<EntitySchema> {
     const columns = this.entitySchemaColumnFactory.create(
@@ -26,6 +27,7 @@ export class EntitySchemaFactory {
 
     const relations = await this.entitySchemaRelationFactory.create(
       workspaceId,
+      metadataVersion,
       objectMetadata.fields,
     );
 

packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts

@@ -11,11 +11,11 @@ export class ScopedWorkspaceContextFactory {
 
   public create(): {
     workspaceId: string | null;
-    workspaceMetadataVersion: string | null;
+    workspaceMetadataVersion: number | null;
   } {
     const workspaceId: string | undefined =
       this.request?.['req']?.['workspaceId'];
-    const workspaceMetadataVersion: string | undefined =
+    const workspaceMetadataVersion: number | undefined =
       this.request?.['req']?.['workspaceMetadataVersion'];
 
     return {

packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts

@@ -1,13 +1,15 @@
 import { Injectable } from '@nestjs/common';
-import { InjectRepository } from '@nestjs/typeorm';
 
-import { EntitySchema, Repository } from 'typeorm';
+import { EntitySchema } from 'typeorm';
 
 import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
-import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
-import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
+import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
+import {
+  TwentyORMException,
+  TwentyORMExceptionCode,
+} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception';
 import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
 import { CacheManager } from 'src/engine/twenty-orm/storage/cache-manager.storage';
 import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@@ -20,63 +22,56 @@ export class WorkspaceDatasourceFactory {
     private readonly dataSourceService: DataSourceService,
     private readonly environmentService: EnvironmentService,
     private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
-    private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
-    @InjectRepository(ObjectMetadataEntity, 'metadata')
-    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
     private readonly entitySchemaFactory: EntitySchemaFactory,
   ) {}
 
   public async create(
     workspaceId: string,
-    workspaceMetadataVersion: string | null,
+    workspaceMetadataVersion: number | null,
   ): Promise<WorkspaceDataSource> {
     const latestWorkspaceMetadataVersion =
-      await this.workspaceMetadataVersionService.getMetadataVersion(
+      await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
+
+    if (latestWorkspaceMetadataVersion === undefined) {
+      await this.workspaceMetadataCacheService.recomputeMetadataCache(
         workspaceId,
       );
+      throw new TwentyORMException(
+        `Metadata version not found for workspace ${workspaceId}`,
+        TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND,
+      );
+    }
 
     const desiredWorkspaceMetadataVersion =
       workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;
 
-    if (!desiredWorkspaceMetadataVersion) {
-      throw new Error(
-        `Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`,
-      );
-    }
-
     if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) {
-      throw new Error(
+      throw new TwentyORMException(
         `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`,
+        TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
       );
     }
 
     const workspaceDataSource = await this.cacheManager.execute(
-      `${workspaceId}-${latestWorkspaceMetadataVersion}`,
+      `${workspaceId}-${desiredWorkspaceMetadataVersion}`,
       async () => {
-        let cachedObjectMetadataCollection =
+        const cachedObjectMetadataCollection =
           await this.workspaceCacheStorageService.getObjectMetadataCollection(
             workspaceId,
+            desiredWorkspaceMetadataVersion,
           );
 
         if (!cachedObjectMetadataCollection) {
-          const freshObjectMetadataCollection =
-            await this.objectMetadataRepository.find({
-              where: { workspaceId },
-              relations: [
-                'fields.object',
-                'fields',
-                'fields.fromRelationMetadata',
-                'fields.toRelationMetadata',
-                'fields.fromRelationMetadata.toObjectMetadata',
-              ],
-            });
-
-          await this.workspaceCacheStorageService.setObjectMetadataCollection(
+          await this.workspaceMetadataCacheService.recomputeMetadataCache(
             workspaceId,
-            freshObjectMetadataCollection,
+            true,
           );
 
-          cachedObjectMetadataCollection = freshObjectMetadataCollection;
+          throw new TwentyORMException(
+            `Object metadata collection not found for workspace ${workspaceId}`,
+            TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND,
+          );
         }
 
         const dataSourceMetadata =
@@ -85,20 +80,16 @@ export class WorkspaceDatasourceFactory {
           );
 
         if (!dataSourceMetadata) {
-          throw new Error(
-            `Data source metadata not found for workspace ${workspaceId}`,
-          );
-        }
-
-        if (!cachedObjectMetadataCollection) {
-          throw new Error(
-            `Object metadata collection not found for workspace ${workspaceId}`,
+          throw new TwentyORMException(
+            `Workspace Schema not found for workspace ${workspaceId}`,
+            TwentyORMExceptionCode.WORKSPACE_SCHEMA_NOT_FOUND,
           );
         }
 
         const cachedEntitySchemaOptions =
           await this.workspaceCacheStorageService.getORMEntitySchema(
             workspaceId,
+            desiredWorkspaceMetadataVersion,
           );
 
         let cachedEntitySchemas: EntitySchema[];
@@ -110,12 +101,17 @@ export class WorkspaceDatasourceFactory {
         } else {
           const entitySchemas = await Promise.all(
             cachedObjectMetadataCollection.map((objectMetadata) =>
-              this.entitySchemaFactory.create(workspaceId, objectMetadata),
+              this.entitySchemaFactory.create(
+                workspaceId,
+                desiredWorkspaceMetadataVersion,
+                objectMetadata,
+              ),
             ),
           );
 
           await this.workspaceCacheStorageService.setORMEntitySchema(
             workspaceId,
+            desiredWorkspaceMetadataVersion,
             entitySchemas.map((entitySchema) => entitySchema.options),
           );