From 23d71915f6ff8182ca609f7cf41a6c0e8787d0d6 Mon Sep 17 00:00:00 2001 From: Marie <51697796+ijreilly@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:53:19 +0200 Subject: [PATCH] Improve metadata version caching (#11775) Investigating https://github.com/twentyhq/core-team-issues/issues/756, I found that the error actually stemmed from "Object metadata collection not found" error. While this is planned to be fixed by metadata performance improvements (as stated in [sentry-boss doc](https://docs.google.com/document/d/1ytbC5W6ZFUSJ3PoJ4IfKi2IehKZYw65mqCnc24aP4RM/edit?tab=t.0) in "known issues"), I tried some easy improvements to reduce the number of errors. --- .../errors/graphql-query-runner.exception.ts | 1 + .../api/graphql/workspace-schema.factory.ts | 53 +++++++---- .../workspace-metadata-cache.service.ts | 94 +++++++++++-------- 3 files changed, 93 insertions(+), 55 deletions(-) diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts index 5e7f0d2ac..7edd9233c 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception.ts @@ -23,4 +23,5 @@ export enum GraphqlQueryRunnerExceptionCode { RELATION_SETTINGS_NOT_FOUND = 'RELATION_SETTINGS_NOT_FOUND', RELATION_TARGET_OBJECT_METADATA_NOT_FOUND = 'RELATION_TARGET_OBJECT_METADATA_NOT_FOUND', NOT_IMPLEMENTED = 'NOT_IMPLEMENTED', + OBJECT_METADATA_COLLECTION_NOT_FOUND = 'OBJECT_METADATA_COLLECTION_NOT_FOUND', } diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts index c687f083d..526679fc9 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts @@ -4,6 +4,7 @@ import { makeExecutableSchema } from '@graphql-tools/schema'; import chalk from 'chalk'; import { GraphQLSchema, printSchema } from 'graphql'; import { gql } from 'graphql-tag'; +import { isDefined } from 'twenty-shared/utils'; import { GraphqlQueryRunnerException, @@ -17,6 +18,7 @@ import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.typ import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @@ -58,34 +60,49 @@ export class WorkspaceSchemaFactory { return new GraphQLSchema({}); } - const currentCacheVersion = + let currentCacheVersion = await this.workspaceCacheStorageService.getMetadataVersion( authContext.workspace.id, ); - if (currentCacheVersion === undefined) { - await this.workspaceMetadataCacheService.recomputeMetadataCache({ - workspaceId: authContext.workspace.id, - }); + let objectMetadataMaps: ObjectMetadataMaps | undefined; + if (currentCacheVersion === undefined) { + const recomputed = + await this.workspaceMetadataCacheService.recomputeMetadataCache({ + workspaceId: authContext.workspace.id, + }); + + objectMetadataMaps = recomputed?.recomputedObjectMetadataMaps; + currentCacheVersion = recomputed?.recomputedMetadataVersion; + } else { + objectMetadataMaps = + await this.workspaceCacheStorageService.getObjectMetadataMaps( + authContext.workspace.id, + currentCacheVersion, + ); + + if (!isDefined(objectMetadataMaps)) { + const recomputed = + await this.workspaceMetadataCacheService.recomputeMetadataCache({ + workspaceId: authContext.workspace.id, + }); + + objectMetadataMaps = recomputed?.recomputedObjectMetadataMaps; + currentCacheVersion = recomputed?.recomputedMetadataVersion; + } + } + + if (!objectMetadataMaps) { throw new GraphqlQueryRunnerException( - 'Metadata cache version not found', - GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, + 'Object metadata collection not found', + GraphqlQueryRunnerExceptionCode.OBJECT_METADATA_COLLECTION_NOT_FOUND, ); } - const objectMetadataMaps = - await this.workspaceCacheStorageService.getObjectMetadataMaps( - authContext.workspace.id, - currentCacheVersion, - ); - - if (!objectMetadataMaps) { - await this.workspaceMetadataCacheService.recomputeMetadataCache({ - workspaceId: authContext.workspace.id, - }); + if (!currentCacheVersion) { throw new GraphqlQueryRunnerException( - 'Object metadata collection not found', + 'Metadata cache version not found', GraphqlQueryRunnerExceptionCode.METADATA_CACHE_VERSION_NOT_FOUND, ); } diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts index 5ae6f7676..90f26706e 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts @@ -5,6 +5,7 @@ import { Repository } from 'typeorm'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { generateObjectMetadataMaps } from 'src/engine/metadata-modules/utils/generate-object-metadata-maps.util'; import { WorkspaceMetadataCacheException, @@ -30,7 +31,13 @@ export class WorkspaceMetadataCacheService { }: { workspaceId: string; ignoreLock?: boolean; - }): Promise { + }): Promise< + | { + recomputedObjectMetadataMaps: ObjectMetadataMaps; + recomputedMetadataVersion: number; + } + | undefined + > { const currentCacheVersion = await this.getMetadataVersionFromCache(workspaceId); @@ -44,16 +51,22 @@ export class WorkspaceMetadataCacheService { ); } - const isAlreadyCaching = - await this.workspaceCacheStorageService.getObjectMetadataOngoingCachingLock( - workspaceId, - currentDatabaseVersion, - ); - - if (!ignoreLock && isAlreadyCaching) { + if (currentDatabaseVersion === currentCacheVersion) { return; } + if (!ignoreLock) { + const isAlreadyCaching = + await this.workspaceCacheStorageService.getObjectMetadataOngoingCachingLock( + workspaceId, + currentDatabaseVersion, + ); + + if (isAlreadyCaching) { + return; + } + } + if (currentCacheVersion !== undefined) { this.workspaceCacheStorageService.flushVersionedMetadata( workspaceId, @@ -61,40 +74,47 @@ export class WorkspaceMetadataCacheService { ); } - await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock( - workspaceId, - currentDatabaseVersion, - ); + try { + await this.workspaceCacheStorageService.addObjectMetadataCollectionOngoingCachingLock( + workspaceId, + currentDatabaseVersion, + ); - await this.workspaceCacheStorageService.setMetadataVersion( - workspaceId, - currentDatabaseVersion, - ); + const objectMetadataItems = await this.objectMetadataRepository.find({ + where: { workspaceId }, + relations: [ + 'fields', + 'fields.fromRelationMetadata', + 'fields.toRelationMetadata', + 'indexMetadatas', + 'indexMetadatas.indexFieldMetadatas', + ], + }); - const objectMetadataItems = await this.objectMetadataRepository.find({ - where: { workspaceId }, - relations: [ - 'fields', - 'fields.fromRelationMetadata', - 'fields.toRelationMetadata', - 'indexMetadatas', - 'indexMetadatas.indexFieldMetadatas', - ], - }); + const freshObjectMetadataMaps = + generateObjectMetadataMaps(objectMetadataItems); - const freshObjectMetadataMaps = - generateObjectMetadataMaps(objectMetadataItems); + await this.workspaceCacheStorageService.setObjectMetadataMaps( + workspaceId, + currentDatabaseVersion, + freshObjectMetadataMaps, + ); - await this.workspaceCacheStorageService.setObjectMetadataMaps( - workspaceId, - currentDatabaseVersion, - freshObjectMetadataMaps, - ); + await this.workspaceCacheStorageService.setMetadataVersion( + workspaceId, + currentDatabaseVersion, + ); - await this.workspaceCacheStorageService.removeObjectMetadataOngoingCachingLock( - workspaceId, - currentDatabaseVersion, - ); + return { + recomputedObjectMetadataMaps: freshObjectMetadataMaps, + recomputedMetadataVersion: currentDatabaseVersion, + }; + } finally { + await this.workspaceCacheStorageService.removeObjectMetadataOngoingCachingLock( + workspaceId, + currentDatabaseVersion, + ); + } } private async getMetadataVersionFromDatabase(