admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts
View File

@ -3,26 +3,18 @@ import { Injectable } from '@nestjs/common';
import { makeExecutableSchema } from '@graphql-tools/schema';
import { GraphQLSchema, printSchema } from 'graphql';
import { gql } from 'graphql-tag';
import { isDefined } from 'twenty-shared/utils';
import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service';
import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories';
import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory';
import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory';
import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
import {
WorkspaceMetadataCacheException,
WorkspaceMetadataCacheExceptionCode,
} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception';
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
import {
WorkspaceMetadataVersionException,
WorkspaceMetadataVersionExceptionCode,
} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
@ -34,8 +26,6 @@ export class WorkspaceSchemaFactory {
private readonly workspaceResolverFactory: WorkspaceResolverFactory,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
private readonly featureFlagService: FeatureFlagService,
private readonly twentyConfigService: TwentyConfigService,
) {}
async createGraphQLSchema(authContext: AuthContext): Promise<GraphQLSchema> {
@ -52,38 +42,12 @@ export class WorkspaceSchemaFactory {
return new GraphQLSchema({});
}
let currentCacheVersion =
await this.workspaceCacheStorageService.getMetadataVersion(
authContext.workspace.id,
);
let objectMetadataMaps: ObjectMetadataMaps | undefined;
if (currentCacheVersion === undefined) {
const recomputed =
await this.workspaceMetadataCacheService.recomputeMetadataCache({
const { objectMetadataMaps, metadataVersion } =
await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
{
workspaceId: authContext.workspace.id,
});
objectMetadataMaps = recomputed?.recomputedObjectMetadataMaps;
currentCacheVersion = recomputed?.recomputedMetadataVersion;
} else {
objectMetadataMaps =
await this.workspaceCacheStorageService.getObjectMetadataMaps(
authContext.workspace.id,
currentCacheVersion,
);
if (!isDefined(objectMetadataMaps)) {
const recomputed =
await this.workspaceMetadataCacheService.recomputeMetadataCache({
workspaceId: authContext.workspace.id,
});
objectMetadataMaps = recomputed?.recomputedObjectMetadataMaps;
currentCacheVersion = recomputed?.recomputedMetadataVersion;
}
}
},
);
if (!objectMetadataMaps) {
throw new WorkspaceMetadataCacheException(
@ -92,17 +56,10 @@ export class WorkspaceSchemaFactory {
);
}
if (!currentCacheVersion) {
throw new WorkspaceMetadataVersionException(
'Metadata cache version not found',
WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
);
}
const objectMetadataCollection = Object.values(objectMetadataMaps.byId).map(
(objectMetadataItem) => ({
...objectMetadataItem,
fields: objectMetadataItem.fields,
fields: Object.values(objectMetadataItem.fieldsById),
indexes: objectMetadataItem.indexMetadatas,
}),
);
@ -110,12 +67,12 @@ export class WorkspaceSchemaFactory {
// Get typeDefs from cache
let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs(
authContext.workspace.id,
currentCacheVersion,
metadataVersion,
);
let usedScalarNames =
await this.workspaceCacheStorageService.getGraphQLUsedScalarNames(
authContext.workspace.id,
currentCacheVersion,
metadataVersion,
);
// If typeDefs are not cached, generate them
@ -133,12 +90,12 @@ export class WorkspaceSchemaFactory {
await this.workspaceCacheStorageService.setGraphQLTypeDefs(
authContext.workspace.id,
currentCacheVersion,
metadataVersion,
typeDefs,
);
await this.workspaceCacheStorageService.setGraphQLUsedScalarNames(
authContext.workspace.id,
currentCacheVersion,
metadataVersion,
usedScalarNames,
);
}