admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts
View File

@ -13,4 +13,5 @@ export enum TwentyORMExceptionCode {
FEATURE_FLAG_MAP_VERSION_NOT_FOUND = 'FEATURE_FLAG_MAP_VERSION_NOT_FOUND',
USER_WORKSPACE_ROLE_MAP_VERSION_NOT_FOUND = 'USER_WORKSPACE_ROLE_MAP_VERSION_NOT_FOUND',
MALFORMED_METADATA = 'MALFORMED_METADATA',
WORKSPACE_NOT_FOUND = 'WORKSPACE_NOT_FOUND',
}

10
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-column.factory.ts
View File

@ -12,7 +12,7 @@ import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-me
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
import { serializeDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-default-value';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
import {
TwentyORMException,
@ -26,10 +26,14 @@ type EntitySchemaColumnMap = {
@Injectable()
export class EntitySchemaColumnFactory {
create(fieldMetadataMapByName: FieldMetadataMap): EntitySchemaColumnMap {
create(
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
): EntitySchemaColumnMap {
let entitySchemaColumnMap: EntitySchemaColumnMap = {};
const fieldMetadataCollection = Object.values(fieldMetadataMapByName);
const fieldMetadataCollection = Object.values(
objectMetadataItemWithFieldMaps.fieldsById,
);
for (const fieldMetadata of fieldMetadataCollection) {
const key = fieldMetadata.name;

8
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts
View File

@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';
import { FieldMetadataType } from 'twenty-shared/types';
import { EntitySchemaRelationOptions } from 'typeorm';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
import { determineSchemaRelationDetails } from 'src/engine/twenty-orm/utils/determine-schema-relation-details.util';
import { isFieldMetadataInterfaceOfType } from 'src/engine/utils/is-field-metadata-of-type.util';
@ -17,12 +17,14 @@ export class EntitySchemaRelationFactory {
constructor() {}
async create(
fieldMetadataMapByName: FieldMetadataMap,
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
objectMetadataMaps: ObjectMetadataMaps,
): Promise<EntitySchemaRelationMap> {
const entitySchemaRelationMap: EntitySchemaRelationMap = {};
const fieldMetadataCollection = Object.values(fieldMetadataMapByName);
const fieldMetadataCollection = Object.values(
objectMetadataItemWithFieldMaps.fieldsById,
);
for (const fieldMetadata of fieldMetadataCollection) {
if (

6
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts
View File

@ -24,12 +24,10 @@ export class EntitySchemaFactory {
objectMetadata: ObjectMetadataItemWithFieldMaps,
objectMetadataMaps: ObjectMetadataMaps,
): Promise<EntitySchema> {
const columns = this.entitySchemaColumnFactory.create(
objectMetadata.fieldsByName,
);
const columns = this.entitySchemaColumnFactory.create(objectMetadata);
const relations = await this.entitySchemaRelationFactory.create(
objectMetadata.fieldsByName,
objectMetadata,
objectMetadataMaps,
);

5
packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts
View File

@ -11,7 +11,6 @@ export class ScopedWorkspaceContextFactory {
public create(): {
workspaceId: string | null;
workspaceMetadataVersion: number | null;
userWorkspaceId: string | null;
isExecutedByApiKey: boolean;
} {
@ -22,13 +21,9 @@ export class ScopedWorkspaceContextFactory {
this.request?.['params']?.['workspaceId'] ||
// @ts-expect-error legacy noImplicitAny
this.request?.['workspace']?.['id']; // rest api
const workspaceMetadataVersion: number | undefined =
// @ts-expect-error legacy noImplicitAny
this.request?.['req']?.['workspaceMetadataVersion'];
return {
workspaceId: workspaceId ?? null,
workspaceMetadataVersion: workspaceMetadataVersion ?? null,
userWorkspaceId:
// @ts-expect-error legacy noImplicitAny
this.request?.['req']?.['userWorkspaceId'] ??

104
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -1,23 +1,17 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { ObjectRecordsPermissionsByRoleId } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { EntitySchema } from 'typeorm';
import { EntitySchema, Repository } from 'typeorm';
import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service';
import {
WorkspaceMetadataCacheException,
WorkspaceMetadataCacheExceptionCode,
} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception';
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
import {
WorkspaceMetadataVersionException,
WorkspaceMetadataVersionExceptionCode,
} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception';
import { WorkspacePermissionsCacheStorageService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache-storage.service';
import {
ROLES_PERMISSIONS,
@ -55,6 +49,8 @@ export class WorkspaceDatasourceFactory {
private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService,
private readonly workspacePermissionsCacheStorageService: WorkspacePermissionsCacheStorageService,
private readonly workspaceFeatureFlagsMapCacheService: WorkspaceFeatureFlagsMapCacheService,
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
) {}
private async conditionalDestroyDataSource(
@ -103,16 +99,9 @@ export class WorkspaceDatasourceFactory {
}
}
public async create(
workspaceId: string,
workspaceMetadataVersion: number | null,
shouldFailIfMetadataNotFound = true,
): Promise<WorkspaceDataSource> {
const cachedWorkspaceMetadataVersion =
await this.getWorkspaceMetadataVersionFromCache(
workspaceId,
shouldFailIfMetadataNotFound,
);
public async create(workspaceId: string): Promise<WorkspaceDataSource> {
const dataSourceMetadataVersion =
await this.getWorkspaceMetadataVersionFromCacheOrFromDB(workspaceId);
const { data: cachedFeatureFlagMap, version: cachedFeatureFlagMapVersion } =
await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion(
@ -126,17 +115,7 @@ export class WorkspaceDatasourceFactory {
workspaceId,
});
if (
workspaceMetadataVersion !== null &&
cachedWorkspaceMetadataVersion !== workspaceMetadataVersion
) {
throw new TwentyORMException(
`Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${cachedWorkspaceMetadataVersion}. Desired version: ${workspaceMetadataVersion}`,
TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
);
}
const cacheKey: CacheKey = `${workspaceId}-${cachedWorkspaceMetadataVersion}`;
const cacheKey: CacheKey = `${workspaceId}-${dataSourceMetadataVersion}`;
const workspaceDataSource =
await this.promiseMemoizer.memoizePromiseAndExecute(
@ -157,21 +136,27 @@ export class WorkspaceDatasourceFactory {
const cachedEntitySchemaOptions =
await this.workspaceCacheStorageService.getORMEntitySchema(
workspaceId,
cachedWorkspaceMetadataVersion,
dataSourceMetadataVersion,
);
let cachedEntitySchemas: EntitySchema[];
const cachedObjectMetadataMaps =
await this.workspaceCacheStorageService.getObjectMetadataMaps(
workspaceId,
cachedWorkspaceMetadataVersion,
const {
objectMetadataMaps: cachedObjectMetadataMaps,
metadataVersion: metadataVersionForFinalUpToDateCheck,
} =
await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
{
workspaceId,
},
);
if (!cachedObjectMetadataMaps) {
throw new WorkspaceMetadataCacheException(
`Object metadata collection not found for workspace ${workspaceId}`,
WorkspaceMetadataCacheExceptionCode.OBJECT_METADATA_COLLECTION_NOT_FOUND,
if (
metadataVersionForFinalUpToDateCheck !== dataSourceMetadataVersion
) {
throw new TwentyORMException(
`Workspace metadata version mismatch detected for workspace ${workspaceId}. Latest version: ${metadataVersionForFinalUpToDateCheck}. Built version: ${dataSourceMetadataVersion}`,
TwentyORMExceptionCode.METADATA_VERSION_MISMATCH,
);
}
@ -185,7 +170,7 @@ export class WorkspaceDatasourceFactory {
(objectMetadata) =>
this.entitySchemaFactory.create(
workspaceId,
cachedWorkspaceMetadataVersion,
dataSourceMetadataVersion,
objectMetadata,
cachedObjectMetadataMaps,
),
@ -194,7 +179,7 @@ export class WorkspaceDatasourceFactory {
await this.workspaceCacheStorageService.setORMEntitySchema(
workspaceId,
cachedWorkspaceMetadataVersion,
dataSourceMetadataVersion,
entitySchemas.map((entitySchema) => entitySchema.options),
);
@ -354,39 +339,28 @@ export class WorkspaceDatasourceFactory {
});
}
private async getWorkspaceMetadataVersionFromCache(
private async getWorkspaceMetadataVersionFromCacheOrFromDB(
workspaceId: string,
shouldFailIfMetadataNotFound = true,
): Promise<number> {
let latestWorkspaceMetadataVersion =
const latestWorkspaceMetadataVersion =
await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
if (!isDefined(latestWorkspaceMetadataVersion)) {
if (shouldFailIfMetadataNotFound) {
throw new WorkspaceMetadataVersionException(
`Metadata version not found while fetching datasource for workspace ${workspaceId}`,
WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
);
} else {
await this.workspaceMetadataCacheService.recomputeMetadataCache({
workspaceId,
ignoreLock: !shouldFailIfMetadataNotFound,
});
latestWorkspaceMetadataVersion =
await this.workspaceCacheStorageService.getMetadataVersion(
workspaceId,
);
}
if (isDefined(latestWorkspaceMetadataVersion)) {
return latestWorkspaceMetadataVersion;
}
if (!isDefined(latestWorkspaceMetadataVersion)) {
throw new WorkspaceMetadataVersionException(
`Metadata version not found after recompute`,
WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND,
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
if (!workspace) {
throw new TwentyORMException(
`Workspace not found for workspace ${workspaceId}`,
TwentyORMExceptionCode.WORKSPACE_NOT_FOUND,
);
}
return latestWorkspaceMetadataVersion;
return workspace.metadataVersion;
}
public async destroy(workspaceId: string) {

25
packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts
View File

@ -17,7 +17,6 @@ export class TwentyORMGlobalManager {
workspaceEntity: Type<T>,
options?: {
shouldBypassPermissionChecks?: boolean;
shouldFailIfMetadataNotFound?: boolean;
},
): Promise<WorkspaceRepository<T>>;
@ -26,7 +25,6 @@ export class TwentyORMGlobalManager {
objectMetadataName: string,
options?: {
shouldBypassPermissionChecks?: boolean;
shouldFailIfMetadataNotFound?: boolean;
},
): Promise<WorkspaceRepository<T>>;
@ -35,10 +33,8 @@ export class TwentyORMGlobalManager {
workspaceEntityOrObjectMetadataName: Type<T> | string,
options: {
shouldBypassPermissionChecks?: boolean;
shouldFailIfMetadataNotFound?: boolean;
} = {
shouldBypassPermissionChecks: false,
shouldFailIfMetadataNotFound: true,
},
): Promise<WorkspaceRepository<T>> {
let objectMetadataName: string;
@ -51,11 +47,8 @@ export class TwentyORMGlobalManager {
);
}
const workspaceDataSource = await this.workspaceDataSourceFactory.create(
workspaceId,
null,
options.shouldFailIfMetadataNotFound,
);
const workspaceDataSource =
await this.workspaceDataSourceFactory.create(workspaceId);
const repository = workspaceDataSource.getRepository<T>(
objectMetadataName,
@ -65,18 +58,8 @@ export class TwentyORMGlobalManager {
return repository;
}
async getDataSourceForWorkspace({
workspaceId,
shouldFailIfMetadataNotFound = true,
}: {
workspaceId: string;
shouldFailIfMetadataNotFound?: boolean;
}) {
return await this.workspaceDataSourceFactory.create(
workspaceId,
null,
shouldFailIfMetadataNotFound,
);
async getDataSourceForWorkspace({ workspaceId }: { workspaceId: string }) {
return await this.workspaceDataSourceFactory.create(workspaceId);
}
async destroyDataSourceForWorkspace(workspaceId: string) {

22
packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts
View File

@ -30,12 +30,8 @@ export class TwentyORMManager {
async getRepository<T extends ObjectLiteral>(
workspaceEntityOrObjectMetadataName: Type<T> | string,
): Promise<WorkspaceRepository<T>> {
const {
workspaceId,
workspaceMetadataVersion,
userWorkspaceId,
isExecutedByApiKey,
} = this.scopedWorkspaceContextFactory.create();
const { workspaceId, userWorkspaceId, isExecutedByApiKey } =
this.scopedWorkspaceContextFactory.create();
let objectMetadataName: string;
@ -51,10 +47,8 @@ export class TwentyORMManager {
throw new Error('Workspace not found');
}
const workspaceDataSource = await this.workspaceDataSourceFactory.create(
workspaceId,
workspaceMetadataVersion,
);
const workspaceDataSource =
await this.workspaceDataSourceFactory.create(workspaceId);
let roleId: string | undefined;
@ -79,16 +73,12 @@ export class TwentyORMManager {
}
async getDatasource() {
const { workspaceId, workspaceMetadataVersion } =
this.scopedWorkspaceContextFactory.create();
const { workspaceId } = this.scopedWorkspaceContextFactory.create();
if (!workspaceId) {
throw new Error('Workspace not found');
}
return this.workspaceDataSourceFactory.create(
workspaceId,
workspaceMetadataVersion,
);
return this.workspaceDataSourceFactory.create(workspaceId);
}
}

3
packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts
View File

@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
@ -22,7 +23,7 @@ import { PgPoolSharedModule } from './pg-shared-pool/pg-shared-pool.module';
@Module({
imports: [
TypeOrmModule.forFeature(
[ObjectMetadataEntity, UserWorkspaceRoleEntity],
[ObjectMetadataEntity, UserWorkspaceRoleEntity, Workspace],
'core',
),
DataSourceModule,

25
packages/twenty-server/src/engine/twenty-orm/utils/format-data.util.ts
View File

@ -7,7 +7,6 @@ import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-meta
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { isFieldMetadataInterfaceOfType } from 'src/engine/utils/is-field-metadata-of-type.util';
export function formatData<T>(
data: T,
@ -25,28 +24,14 @@ export function formatData<T>(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const newData: Record<string, any> = {};
const fieldMetadataByJoinColumnName =
objectMetadataItemWithFieldMaps.fields.reduce((acc, fieldMetadata) => {
if (
isFieldMetadataInterfaceOfType(
fieldMetadata,
FieldMetadataType.RELATION,
)
) {
const joinColumnName = fieldMetadata.settings?.joinColumnName;
if (joinColumnName) {
acc.set(joinColumnName, fieldMetadata);
}
}
return acc;
}, new Map<string, FieldMetadataInterface>());
for (const [key, value] of Object.entries(data)) {
const fieldMetadataId =
objectMetadataItemWithFieldMaps.fieldIdByName[key] ||
objectMetadataItemWithFieldMaps.fieldIdByJoinColumnName[key];
const fieldMetadata =
objectMetadataItemWithFieldMaps.fieldsByName[key] ||
fieldMetadataByJoinColumnName.get(key);
objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId];
if (!fieldMetadata) {
throw new Error(

24
packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.ts
View File

@ -44,15 +44,15 @@ export function formatResult<T>(
);
const newData: object = {};
const objectMetadaItemFieldsByName =
objectMetadataMaps.byId[objectMetadataItemWithFieldMaps.id]?.fieldsByName;
for (const [key, value] of Object.entries(data)) {
const compositePropertyArgs = compositeFieldMetadataMap.get(key);
const fieldMetadata = objectMetadataItemWithFieldMaps.fieldsByName[key] as
| FieldMetadataInterface<FieldMetadataType>
| undefined;
const fieldMetadataId = objectMetadataItemWithFieldMaps.fieldIdByName[key];
const fieldMetadata = objectMetadataItemWithFieldMaps.fieldsById[
fieldMetadataId
] as FieldMetadataInterface<FieldMetadataType> | undefined;
const isRelation = fieldMetadata
? isFieldMetadataInterfaceOfType(
@ -69,12 +69,9 @@ export function formatResult<T>(
objectMetadataItemWithFieldMaps,
objectMetadataMaps,
);
} else if (objectMetadaItemFieldsByName[key]) {
} else if (fieldMetadata) {
// @ts-expect-error legacy noImplicitAny
newData[key] = formatFieldMetadataValue(
value,
objectMetadaItemFieldsByName[key],
);
newData[key] = formatFieldMetadataValue(value, fieldMetadata);
} else {
// @ts-expect-error legacy noImplicitAny
newData[key] = value;
@ -123,10 +120,9 @@ export function formatResult<T>(
newData[parentField][compositeProperty.name] = value;
}
const dateFieldMetadataCollection =
objectMetadataItemWithFieldMaps.fields.filter(
(field) => field.type === FieldMetadataType.DATE,
);
const dateFieldMetadataCollection = Object.values(
objectMetadataItemWithFieldMaps.fieldsById,
).filter((field) => field.type === FieldMetadataType.DATE);
// This is a temporary fix to handle a bug in the frontend where the date gets returned in the wrong timezone,
// thus returning the wrong date.