admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts
View File

@ -27,17 +27,13 @@ export enum WorkspaceCacheKeys {
ORMEntitySchemas = 'orm:entity-schemas',
GraphQLFeatureFlag = 'graphql:feature-flag',
MetadataObjectMetadataMaps = 'metadata:object-metadata-maps',
MetadataObjectMetadataOngoingCachingLock = 'metadata:object-metadata-ongoing-caching-lock',
MetadataVersion = 'metadata:workspace-metadata-version',
FeatureFlagMap = 'feature-flag:feature-flag-map',
FeatureFlagMapVersion = 'feature-flag:feature-flag-map-version',
FeatureFlagMapOngoingCachingLock = 'feature-flag-map-ongoing-caching-lock',
MetadataPermissionsRolesPermissions = 'metadata:permissions:roles-permissions',
MetadataPermissionsRolesPermissionsVersion = 'metadata:permissions:roles-permissions-version',
MetadataPermissionsRolesPermissionsOngoingCachingLock = 'metadata:permissions:roles-permissions-ongoing-caching-lock',
MetadataPermissionsUserWorkspaceRoleMap = 'metadata:permissions:user-workspace-role-map',
MetadataPermissionsUserWorkspaceRoleMapVersion = 'metadata:permissions:user-workspace-role-map-version',
MetadataPermissionsUserWorkspaceRoleMapOngoingCachingLock = 'metadata:permissions:user-workspace-role-map-ongoing-caching-lock',
}
const TTL_INFINITE = 0;
@ -91,35 +87,6 @@ export class WorkspaceCacheStorageService {
);
}
addObjectMetadataCollectionOngoingCachingLock(
workspaceId: string,
metadataVersion: number,
) {
return this.cacheStorageService.set<boolean>(
`${WorkspaceCacheKeys.MetadataObjectMetadataOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
true,
1_000 * 60, // 1 minute
);
}
removeObjectMetadataOngoingCachingLock(
workspaceId: string,
metadataVersion: number,
) {
return this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataObjectMetadataOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
);
}
getObjectMetadataOngoingCachingLock(
workspaceId: string,
metadataVersion: number,
): Promise<boolean | undefined> {
return this.cacheStorageService.get<boolean>(
`${WorkspaceCacheKeys.MetadataObjectMetadataOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
);
}
setObjectMetadataMaps(
workspaceId: string,
metadataVersion: number,
@ -252,59 +219,33 @@ export class WorkspaceCacheStorageService {
);
}
addFeatureFlagMapOngoingCachingLock(workspaceId: string) {
return this.cacheStorageService.set<boolean>(
`${WorkspaceCacheKeys.FeatureFlagMapOngoingCachingLock}:${workspaceId}`,
true,
1_000 * 60, // 1 minute
);
}
removeFeatureFlagsMapOngoingCachingLock(workspaceId: string) {
return this.cacheStorageService.del(
`${WorkspaceCacheKeys.FeatureFlagMapOngoingCachingLock}:${workspaceId}`,
);
}
getFeatureFlagsMapOngoingCachingLock(
workspaceId: string,
): Promise<boolean | undefined> {
return this.cacheStorageService.get<boolean>(
`${WorkspaceCacheKeys.FeatureFlagMapOngoingCachingLock}:${workspaceId}`,
);
}
async flushVersionedMetadata(
workspaceId: string,
metadataVersion: number,
metadataVersion?: number,
): Promise<void> {
const metadataVersionSuffix = isDefined(metadataVersion)
? `${metadataVersion}`
: '*';
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataObjectMetadataMaps}:${workspaceId}:${metadataVersion}`,
`${WorkspaceCacheKeys.MetadataObjectMetadataMaps}:${workspaceId}:${metadataVersionSuffix}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}:${metadataVersion}`,
`${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}:${metadataVersionSuffix}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersion}`,
`${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}:${metadataVersionSuffix}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersion}`,
`${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}:${metadataVersionSuffix}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersion}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataObjectMetadataOngoingCachingLock}:${workspaceId}:${metadataVersion}`,
`${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}:${metadataVersionSuffix}`,
);
}
async flush(
workspaceId: string,
metadataVersion: number | undefined,
): Promise<void> {
if (isDefined(metadataVersion)) {
await this.flushVersionedMetadata(workspaceId, metadataVersion);
}
async flush(workspaceId: string, metadataVersion?: number): Promise<void> {
await this.flushVersionedMetadata(workspaceId, metadataVersion);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataPermissionsRolesPermissions}:${workspaceId}`,
@ -314,10 +255,6 @@ export class WorkspaceCacheStorageService {
`${WorkspaceCacheKeys.MetadataPermissionsRolesPermissionsVersion}:${workspaceId}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataPermissionsRolesPermissionsOngoingCachingLock}:${workspaceId}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataPermissionsUserWorkspaceRoleMap}:${workspaceId}`,
);
@ -326,10 +263,6 @@ export class WorkspaceCacheStorageService {
`${WorkspaceCacheKeys.MetadataPermissionsUserWorkspaceRoleMapVersion}:${workspaceId}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.MetadataPermissionsUserWorkspaceRoleMapOngoingCachingLock}:${workspaceId}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.FeatureFlagMap}:${workspaceId}`,
);
@ -337,9 +270,5 @@ export class WorkspaceCacheStorageService {
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.FeatureFlagMapVersion}:${workspaceId}`,
);
await this.cacheStorageService.del(
`${WorkspaceCacheKeys.FeatureFlagMapOngoingCachingLock}:${workspaceId}`,
);
}
}