From 741a969cc112ae39e9ffcbecf8c6ba712555c9e8 Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Wed, 18 Sep 2024 15:40:24 +0200 Subject: [PATCH] Add fail on metadata cache miss (#7118) - avoid failing when missing cache (used for command) - remove unused load cache function. Cache will be always re-created when trying to fetch if not existing --- .../api/graphql/workspace-schema.factory.ts | 12 +- .../workspace-metadata-cache.service.ts | 17 ++- .../workspace-metadata-version.service.ts | 4 +- .../factories/workspace-datasource.factory.ts | 123 ++++++++++-------- .../twenty-orm/twenty-orm-global.manager.ts | 10 +- 5 files changed, 92 insertions(+), 74 deletions(-) diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts index 3e7f20e7c..336fa825f 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts @@ -47,9 +47,9 @@ export class WorkspaceSchemaFactory { ); if (currentCacheVersion === undefined) { - await this.workspaceMetadataCacheService.recomputeMetadataCache( - authContext.workspace.id, - ); + await this.workspaceMetadataCacheService.recomputeMetadataCache({ + workspaceId: authContext.workspace.id, + }); throw new GraphqlQueryRunnerException( 'Metadata cache version not found', GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, @@ -63,9 +63,9 @@ export class WorkspaceSchemaFactory { ); if (!objectMetadataMap) { - await this.workspaceMetadataCacheService.recomputeMetadataCache( - authContext.workspace.id, - ); + await this.workspaceMetadataCacheService.recomputeMetadataCache({ + workspaceId: authContext.workspace.id, + }); throw new GraphqlQueryRunnerException( 'Object metadata collection not found', GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts index 758ffbee2..e1d5a0ae9 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts @@ -26,10 +26,13 @@ export class WorkspaceMetadataCacheService { ) {} @LogExecutionTime() - async recomputeMetadataCache( - workspaceId: string, - force = false, - ): Promise { + async recomputeMetadataCache({ + workspaceId, + ignoreLock = false, + }: { + workspaceId: string; + ignoreLock?: boolean; + }): Promise { const currentCacheVersion = await this.getMetadataVersionFromCache(workspaceId); @@ -43,17 +46,13 @@ export class WorkspaceMetadataCacheService { ); } - if (!force && currentCacheVersion === currentDatabaseVersion) { - return; - } - const isAlreadyCaching = await this.workspaceCacheStorageService.getObjectMetadataOngoingCachingLock( workspaceId, currentDatabaseVersion, ); - if (isAlreadyCaching) { + if (!ignoreLock && isAlreadyCaching) { return; } diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts index 615242dc8..c34f4577f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts @@ -45,8 +45,8 @@ export class WorkspaceMetadataVersionService { { metadataVersion: newMetadataVersion }, ); - await this.workspaceMetadataCacheService.recomputeMetadataCache( + await this.workspaceMetadataCacheService.recomputeMetadataCache({ workspaceId, - ); + }); } } diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 771c63559..bee750bae 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -4,6 +4,7 @@ import { EntitySchema } from 'typeorm'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util'; import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; import { @@ -33,14 +34,27 @@ export class WorkspaceDatasourceFactory { public async create( workspaceId: string, workspaceMetadataVersion: number | null, + failOnMetadataCacheMiss = true, ): Promise { - const desiredWorkspaceMetadataVersion = - await this.computeDesiredWorkspaceMetadataVersion( - workspaceId, - workspaceMetadataVersion, - ); + const { + metadataVersion: cachedWorkspaceMetadataVersion, + objectMetadataMap: cachedObjectMetadataMap, + } = await this.getWorkspaceMetadataFromCache( + workspaceId, + failOnMetadataCacheMiss, + ); - const cacheKey = `${workspaceId}-${desiredWorkspaceMetadataVersion}`; + if ( + workspaceMetadataVersion !== null && + cachedWorkspaceMetadataVersion !== workspaceMetadataVersion + ) { + throw new TwentyORMException( + `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${cachedWorkspaceMetadataVersion}. Desired version: ${workspaceMetadataVersion}`, + TwentyORMExceptionCode.METADATA_VERSION_MISMATCH, + ); + } + + const cacheKey = `${workspaceId}-${cachedWorkspaceMetadataVersion}`; if (cacheKey in this.cachedDatasourcePromise) { return this.cachedDatasourcePromise[cacheKey]; @@ -52,25 +66,8 @@ export class WorkspaceDatasourceFactory { cacheKey as '`${string}-${string}`', async () => { this.logger.log( - `Creating workspace data source for workspace ${workspaceId} and metadata version ${desiredWorkspaceMetadataVersion}`, + `Creating workspace data source for workspace ${workspaceId} and metadata version ${cachedWorkspaceMetadataVersion}`, ); - const cachedObjectMetadataMap = - await this.workspaceCacheStorageService.getObjectMetadataMap( - workspaceId, - desiredWorkspaceMetadataVersion, - ); - - if (!cachedObjectMetadataMap) { - await this.workspaceMetadataCacheService.recomputeMetadataCache( - workspaceId, - true, - ); - - throw new TwentyORMException( - `Object metadata map not found for workspace ${workspaceId}`, - TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND, - ); - } const dataSourceMetadata = await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId( @@ -87,7 +84,7 @@ export class WorkspaceDatasourceFactory { const cachedEntitySchemaOptions = await this.workspaceCacheStorageService.getORMEntitySchema( workspaceId, - desiredWorkspaceMetadataVersion, + cachedWorkspaceMetadataVersion, ); let cachedEntitySchemas: EntitySchema[]; @@ -101,7 +98,7 @@ export class WorkspaceDatasourceFactory { Object.values(cachedObjectMetadataMap).map((objectMetadata) => this.entitySchemaFactory.create( workspaceId, - desiredWorkspaceMetadataVersion, + cachedWorkspaceMetadataVersion, objectMetadata, cachedObjectMetadataMap, ), @@ -110,7 +107,7 @@ export class WorkspaceDatasourceFactory { await this.workspaceCacheStorageService.setORMEntitySchema( workspaceId, - desiredWorkspaceMetadataVersion, + cachedWorkspaceMetadataVersion, entitySchemas.map((entitySchema) => entitySchema.options), ); @@ -175,45 +172,67 @@ export class WorkspaceDatasourceFactory { return creationPromise; } - private async computeDesiredWorkspaceMetadataVersion( + public async destroy(workspaceId: string): Promise { + const cachedWorkspaceMetadataVersion = + await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); + + await this.cacheManager.clearKey( + `${workspaceId}-${cachedWorkspaceMetadataVersion}`, + ); + } + + private async getWorkspaceMetadataFromCache( workspaceId: string, - workspaceMetadataVersion: number | null, - ): Promise { - const latestWorkspaceMetadataVersion = + failOnMetadataCacheMiss = true, + ): Promise<{ + metadataVersion: number; + objectMetadataMap: ObjectMetadataMap; + }> { + let latestWorkspaceMetadataVersion = await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); if (latestWorkspaceMetadataVersion === undefined) { - await this.workspaceMetadataCacheService.recomputeMetadataCache( + await this.workspaceMetadataCacheService.recomputeMetadataCache({ workspaceId, - ); + ignoreLock: !failOnMetadataCacheMiss, + }); + + if (failOnMetadataCacheMiss) { + throw new TwentyORMException( + `Metadata version not found for workspace ${workspaceId}`, + TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND, + ); + } else { + latestWorkspaceMetadataVersion = + await this.workspaceCacheStorageService.getMetadataVersion( + workspaceId, + ); + } + } + + if (!latestWorkspaceMetadataVersion) { throw new TwentyORMException( - `Metadata version not found for workspace ${workspaceId}`, + `Metadata version not found after recompute for workspace ${workspaceId}`, TwentyORMExceptionCode.METADATA_VERSION_NOT_FOUND, ); } - const desiredWorkspaceMetadataVersion = - workspaceMetadataVersion ?? latestWorkspaceMetadataVersion; + const objectMetadataMap = + await this.workspaceCacheStorageService.getObjectMetadataMap( + workspaceId, + latestWorkspaceMetadataVersion, + ); - if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) { + if (!objectMetadataMap) { throw new TwentyORMException( - `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`, - TwentyORMExceptionCode.METADATA_VERSION_MISMATCH, + `Object metadata map not found for workspace ${workspaceId}`, + TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND, ); } - return desiredWorkspaceMetadataVersion; - } - - public async destroy( - workspaceId: string, - metadataVersion: number | null, - ): Promise { - const desiredWorkspaceMetadataVersion = - this.computeDesiredWorkspaceMetadataVersion(workspaceId, metadataVersion); - - await this.cacheManager.clearKey( - `${workspaceId}-${desiredWorkspaceMetadataVersion}`, - ); + return { + metadataVersion: latestWorkspaceMetadataVersion, + objectMetadataMap, + }; } } diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts index 811e8f27b..acfaabb18 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts @@ -15,16 +15,19 @@ export class TwentyORMGlobalManager { async getRepositoryForWorkspace( workspaceId: string, workspaceEntity: Type, + failOnMetadataCacheMiss?: boolean, ): Promise>; async getRepositoryForWorkspace( workspaceId: string, objectMetadataName: string, + failOnMetadataCacheMiss?: boolean, ): Promise>; async getRepositoryForWorkspace( workspaceId: string, workspaceEntityOrobjectMetadataName: Type | string, + failOnMetadataCacheMiss = true, ): Promise> { let objectMetadataName: string; @@ -39,6 +42,7 @@ export class TwentyORMGlobalManager { const workspaceDataSource = await this.workspaceDataSourceFactory.create( workspaceId, null, + failOnMetadataCacheMiss, ); const repository = workspaceDataSource.getRepository(objectMetadataName); @@ -50,11 +54,7 @@ export class TwentyORMGlobalManager { return await this.workspaceDataSourceFactory.create(workspaceId, null); } - async loadDataSourceForWorkspace(workspaceId: string) { - await this.workspaceDataSourceFactory.create(workspaceId, null); - } - async destroyDataSourceForWorkspace(workspaceId: string) { - await this.workspaceDataSourceFactory.destroy(workspaceId, null); + await this.workspaceDataSourceFactory.destroy(workspaceId); } }