Add fail on metadata cache miss (#7118)

- avoid failing when missing cache (used for command) - remove unused load cache function. Cache will be always re-created when trying to fetch if not existing
2024-09-18 15:40:24 +02:00
parent 44587b4908
commit 741a969cc1
5 changed files with 92 additions and 74 deletions
--- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts
@ -47,9 +47,9 @@ export class WorkspaceSchemaFactory {
      );

    if (currentCacheVersion === undefined) {
-      await this.workspaceMetadataCacheService.recomputeMetadataCache(
-        authContext.workspace.id,
-      );
+      await this.workspaceMetadataCacheService.recomputeMetadataCache({
+        workspaceId: authContext.workspace.id,
+      });
      throw new GraphqlQueryRunnerException(
        'Metadata cache version not found',
        GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
@ -63,9 +63,9 @@ export class WorkspaceSchemaFactory {
      );

    if (!objectMetadataMap) {
-      await this.workspaceMetadataCacheService.recomputeMetadataCache(
-        authContext.workspace.id,
-      );
+      await this.workspaceMetadataCacheService.recomputeMetadataCache({
+        workspaceId: authContext.workspace.id,
+      });
      throw new GraphqlQueryRunnerException(
        'Object metadata collection not found',
        GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND,
--- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts
@ -26,10 +26,13 @@ export class WorkspaceMetadataCacheService {
  ) {}

  @LogExecutionTime()
-  async recomputeMetadataCache(
-    workspaceId: string,
-    force = false,
-  ): Promise<void> {
+  async recomputeMetadataCache({
+    workspaceId,
+    ignoreLock = false,
+  }: {
+    workspaceId: string;
+    ignoreLock?: boolean;
+  }): Promise<void> {
    const currentCacheVersion =
      await this.getMetadataVersionFromCache(workspaceId);

@ -43,17 +46,13 @@ export class WorkspaceMetadataCacheService {
      );
    }

-    if (!force && currentCacheVersion === currentDatabaseVersion) {
-      return;
-    }
-
    const isAlreadyCaching =
      await this.workspaceCacheStorageService.getObjectMetadataOngoingCachingLock(
        workspaceId,
        currentDatabaseVersion,
      );

-    if (isAlreadyCaching) {
+    if (!ignoreLock && isAlreadyCaching) {
      return;
    }

--- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts
@ -45,8 +45,8 @@ export class WorkspaceMetadataVersionService {
      { metadataVersion: newMetadataVersion },
    );

-    await this.workspaceMetadataCacheService.recomputeMetadataCache(
+    await this.workspaceMetadataCacheService.recomputeMetadataCache({
      workspaceId,
-    );
+    });
  }
 }
--- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
@ -4,6 +4,7 @@ import { EntitySchema } from 'typeorm';

 import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
+import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
 import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import {
@ -33,14 +34,27 @@ export class WorkspaceDatasourceFactory {
  public async create(
    workspaceId: string,
    workspaceMetadataVersion: number | null,
+    failOnMetadataCacheMiss = true,
  ): Promise<WorkspaceDataSource> {
-    const desiredWorkspaceMetadataVersion =
-      await this.computeDesiredWorkspaceMetadataVersion(
-        workspaceId,
-        workspaceMetadataVersion,
-      );
+    const {
+      metadataVersion: cachedWorkspaceMetadataVersion,
+      objectMetadataMap: cachedObjectMetadataMap,
+    } = await this.getWorkspaceMetadataFromCache(
+      workspaceId,
+      failOnMetadataCacheMiss,
+    );

-    const cacheKey = `${workspaceId}-${desiredWorkspaceMetadataVersion}`;
+    if (
+      workspaceMetadataVersion !== null &&
+      cachedWorkspaceMetadataVersion !== workspaceMetadataVersion
+    ) {
+      throw new TwentyORMException(
+        `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${cachedWorkspaceMetadataVersion}. Desired version: ${workspaceMetadataVersion}`,
+        TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
+      );
+    }
+
+    const cacheKey = `${workspaceId}-${cachedWorkspaceMetadataVersion}`;

    if (cacheKey in this.cachedDatasourcePromise) {
      return this.cachedDatasourcePromise[cacheKey];
@ -52,25 +66,8 @@ export class WorkspaceDatasourceFactory {
          cacheKey as '`${string}-${string}`',
          async () => {
            this.logger.log(
-              `Creating workspace data source for workspace ${workspaceId} and metadata version ${desiredWorkspaceMetadataVersion}`,
+              `Creating workspace data source for workspace ${workspaceId} and metadata version ${cachedWorkspaceMetadataVersion}`,
            );
-            const cachedObjectMetadataMap =
-              await this.workspaceCacheStorageService.getObjectMetadataMap(
-                workspaceId,
-                desiredWorkspaceMetadataVersion,
-              );
-
-            if (!cachedObjectMetadataMap) {
-              await this.workspaceMetadataCacheService.recomputeMetadataCache(
-                workspaceId,
-                true,
-              );
-
-              throw new TwentyORMException(
-                `Object metadata map not found for workspace ${workspaceId}`,
-                TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND,
-              );
-            }

            const dataSourceMetadata =
              await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
@ -87,7 +84,7 @@ export class WorkspaceDatasourceFactory {
            const cachedEntitySchemaOptions =
              await this.workspaceCacheStorageService.getORMEntitySchema(
                workspaceId,
-                desiredWorkspaceMetadataVersion,
+                cachedWorkspaceMetadataVersion,
              );

            let cachedEntitySchemas: EntitySchema[];
@ -101,7 +98,7 @@ export class WorkspaceDatasourceFactory {
                Object.values(cachedObjectMetadataMap).map((objectMetadata) =>
                  this.entitySchemaFactory.create(
                    workspaceId,
-                    desiredWorkspaceMetadataVersion,
+                    cachedWorkspaceMetadataVersion,
                    objectMetadata,
                    cachedObjectMetadataMap,
                  ),
@ -110,7 +107,7 @@ export class WorkspaceDatasourceFactory {

              await this.workspaceCacheStorageService.setORMEntitySchema(
                workspaceId,
-                desiredWorkspaceMetadataVersion,
+                cachedWorkspaceMetadataVersion,
                entitySchemas.map((entitySchema) => entitySchema.options),
              );

@ -175,45 +172,67 @@ export class WorkspaceDatasourceFactory {
    return creationPromise;
  }

-  private async computeDesiredWorkspaceMetadataVersion(
+  public async destroy(workspaceId: string): Promise<void> {
+    const cachedWorkspaceMetadataVersion =
+      await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
+
+    await this.cacheManager.clearKey(
+      `${workspaceId}-${cachedWorkspaceMetadataVersion}`,
+    );
+  }
+
+  private async getWorkspaceMetadataFromCache(
    workspaceId: string,
-    workspaceMetadataVersion: number | null,
-  ): Promise<number> {
-    const latestWorkspaceMetadataVersion =
+    failOnMetadataCacheMiss = true,
+  ): Promise<{
+    metadataVersion: number;
+    objectMetadataMap: ObjectMetadataMap;
+  }> {
+    let latestWorkspaceMetadataVersion =
      await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);

    if (latestWorkspaceMetadataVersion === undefined) {
-      await this.workspaceMetadataCacheService.recomputeMetadataCache(
+      await this.workspaceMetadataCacheService.recomputeMetadataCache({
        workspaceId,
-      );
+        ignoreLock: !failOnMetadataCacheMiss,
+      });
+
+      if (failOnMetadataCacheMiss) {
+        throw new TwentyORMException(
+          `Metadata version not found for workspace ${workspaceId}`,
+          TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND,
+        );
+      } else {
+        latestWorkspaceMetadataVersion =
+          await this.workspaceCacheStorageService.getMetadataVersion(
+            workspaceId,
+          );
+      }
+    }
+
+    if (!latestWorkspaceMetadataVersion) {
      throw new TwentyORMException(
-        `Metadata version not found for workspace ${workspaceId}`,
+        `Metadata version not found after recompute for workspace ${workspaceId}`,
        TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND,
      );
    }

-    const desiredWorkspaceMetadataVersion =
-      workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;
+    const objectMetadataMap =
+      await this.workspaceCacheStorageService.getObjectMetadataMap(
+        workspaceId,
+        latestWorkspaceMetadataVersion,
+      );

-    if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) {
+    if (!objectMetadataMap) {
      throw new TwentyORMException(
-        `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`,
-        TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
+        `Object metadata map not found for workspace ${workspaceId}`,
+        TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND,
      );
    }

-    return desiredWorkspaceMetadataVersion;
-  }
-
-  public async destroy(
-    workspaceId: string,
-    metadataVersion: number | null,
-  ): Promise<void> {
-    const desiredWorkspaceMetadataVersion =
-      this.computeDesiredWorkspaceMetadataVersion(workspaceId, metadataVersion);
-
-    await this.cacheManager.clearKey(
-      `${workspaceId}-${desiredWorkspaceMetadataVersion}`,
-    );
+    return {
+      metadataVersion: latestWorkspaceMetadataVersion,
+      objectMetadataMap,
+    };
  }
 }
--- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts
@ -15,16 +15,19 @@ export class TwentyORMGlobalManager {
  async getRepositoryForWorkspace<T extends ObjectLiteral>(
    workspaceId: string,
    workspaceEntity: Type<T>,
+    failOnMetadataCacheMiss?: boolean,
  ): Promise<WorkspaceRepository<T>>;

  async getRepositoryForWorkspace<T extends ObjectLiteral>(
    workspaceId: string,
    objectMetadataName: string,
+    failOnMetadataCacheMiss?: boolean,
  ): Promise<WorkspaceRepository<T>>;

  async getRepositoryForWorkspace<T extends ObjectLiteral>(
    workspaceId: string,
    workspaceEntityOrobjectMetadataName: Type<T> | string,
+    failOnMetadataCacheMiss = true,
  ): Promise<WorkspaceRepository<T>> {
    let objectMetadataName: string;

@ -39,6 +42,7 @@ export class TwentyORMGlobalManager {
    const workspaceDataSource = await this.workspaceDataSourceFactory.create(
      workspaceId,
      null,
+      failOnMetadataCacheMiss,
    );

    const repository = workspaceDataSource.getRepository<T>(objectMetadataName);
@ -50,11 +54,7 @@ export class TwentyORMGlobalManager {
    return await this.workspaceDataSourceFactory.create(workspaceId, null);
  }

-  async loadDataSourceForWorkspace(workspaceId: string) {
-    await this.workspaceDataSourceFactory.create(workspaceId, null);
-  }
-
  async destroyDataSourceForWorkspace(workspaceId: string) {
-    await this.workspaceDataSourceFactory.destroy(workspaceId, null);
+    await this.workspaceDataSourceFactory.destroy(workspaceId);
  }
 }