admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
packages/twenty-server/src/engine/core-modules/__mocks__/mockObjectMetadataItemsWithFieldMaps.ts
View File

@ -23,25 +23,6 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
labelIdentifierFieldMetadataId: 'nameFieldMetadataId',
imageIdentifierFieldMetadataId: '',
workspaceId: '',
fields: [
{
id: 'nameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.FULL_NAME,
icon: 'test-field-icon',
name: 'name',
label: 'Name',
defaultValue: {
lastName: "''",
firstName: "''",
},
description: 'Contact’s name',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
],
indexMetadatas: [],
fieldsById: {
nameFieldMetadataId: {
@ -60,28 +41,15 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
isNullable: true,
isUnique: false,
workspaceId: '',
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
},
},
fieldsByName: {
name: {
id: 'nameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.FULL_NAME,
icon: 'test-field-icon',
name: 'name',
label: 'Name',
defaultValue: {
lastName: "''",
firstName: "''",
},
description: 'Contact’s name',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
fieldIdByName: {
name: 'nameFieldMetadataId',
},
fieldsByJoinColumnName: {},
fieldIdByJoinColumnName: {},
},
{
id: '',
@ -102,34 +70,6 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
labelIdentifierFieldMetadataId: 'nameFieldMetadataId',
imageIdentifierFieldMetadataId: '',
workspaceId: '',
fields: [
{
id: 'nameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.TEXT,
icon: 'test-field-icon',
name: 'name',
label: 'Name',
defaultValue: '',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
{
id: 'domainNameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.LINKS,
icon: 'test-field-icon',
name: 'domainName',
label: 'Domain Name',
defaultValue: '',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
],
indexMetadatas: [],
fieldsById: {
nameFieldMetadataId: {
@ -144,6 +84,9 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
isNullable: true,
isUnique: false,
workspaceId: '',
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
},
domainNameFieldMetadataId: {
id: 'domainNameFieldMetadataId',
@ -157,40 +100,16 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
isNullable: true,
isUnique: false,
workspaceId: '',
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
},
},
fieldsByName: {
name: {
id: 'nameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.TEXT,
icon: 'test-field-icon',
name: 'name',
label: 'Name',
defaultValue: {
lastName: "''",
firstName: "''",
},
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
domainName: {
id: 'domainNameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.LINKS,
icon: 'test-field-icon',
name: 'domainName',
label: 'Domain Name',
defaultValue: '',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
fieldIdByName: {
name: 'nameFieldMetadataId',
domainName: 'domainNameFieldMetadataId',
},
fieldsByJoinColumnName: {},
fieldIdByJoinColumnName: {},
},
{
id: '',
@ -211,34 +130,6 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
labelIdentifierFieldMetadataId: 'nameFieldMetadataId',
imageIdentifierFieldMetadataId: 'imageIdentifierFieldMetadataId',
workspaceId: '',
fields: [
{
id: 'nameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.TEXT,
icon: 'test-field-icon',
name: 'name',
label: 'Name',
defaultValue: '',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
{
id: 'imageIdentifierFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.TEXT,
icon: 'test-field-icon',
name: 'imageIdentifierFieldName',
label: 'Image Identifier Field Name',
defaultValue: '',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
],
indexMetadatas: [],
fieldsById: {
nameFieldMetadataId: {
@ -253,6 +144,9 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
isNullable: true,
isUnique: false,
workspaceId: '',
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
},
imageIdentifierFieldMetadataId: {
id: 'imageIdentifierFieldMetadataId',
@ -266,40 +160,16 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
isNullable: true,
isUnique: false,
workspaceId: '',
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
},
},
fieldsByName: {
name: {
id: 'nameFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.TEXT,
icon: 'test-field-icon',
name: 'name',
label: 'Name',
defaultValue: {
lastName: "''",
firstName: "''",
},
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
imageIdentifierFieldName: {
id: 'imageIdentifierFieldMetadataId',
objectMetadataId: '',
type: FieldMetadataType.TEXT,
icon: 'test-field-icon',
name: 'imageIdentifierFieldName',
label: 'Image Identifier Field Name',
defaultValue: '',
isCustom: false,
isNullable: true,
isUnique: false,
workspaceId: '',
},
fieldIdByName: {
name: 'nameFieldMetadataId',
imageIdentifierFieldName: 'imageIdentifierFieldMetadataId',
},
fieldsByJoinColumnName: {},
fieldIdByJoinColumnName: {},
},
{
id: '',
@ -320,10 +190,9 @@ export const mockObjectMetadataItemsWithFieldMaps: ObjectMetadataItemWithFieldMa
labelIdentifierFieldMetadataId: '',
imageIdentifierFieldMetadataId: '',
workspaceId: '',
fields: [],
indexMetadatas: [],
fieldsById: {},
fieldsByName: {},
fieldsByJoinColumnName: {},
fieldIdByName: {},
fieldIdByJoinColumnName: {},
},
];

5
packages/twenty-server/src/engine/core-modules/auth/token/services/access-token.service.ts
View File

@ -21,13 +21,13 @@ import {
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { userWorkspaceValidator } from 'src/engine/core-modules/user-workspace/user-workspace.validate';
import { User } from 'src/engine/core-modules/user/user.entity';
import { userValidator } from 'src/engine/core-modules/user/user.validate';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { workspaceValidator } from 'src/engine/core-modules/workspace/workspace.validate';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { userWorkspaceValidator } from 'src/engine/core-modules/user-workspace/user-workspace.validate';
@Injectable()
export class AccessTokenService {
@ -78,9 +78,6 @@ export class AccessTokenService {
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{
shouldFailIfMetadataNotFound: false,
},
);
const workspaceMember = await workspaceMemberRepository.findOne({

2
packages/twenty-server/src/engine/core-modules/event-emitter/types/object-record.base.event.ts
View File

@ -13,6 +13,6 @@ export class ObjectRecordBaseEvent<T = object> {
recordId: string;
userId?: string;
workspaceMemberId?: string;
objectMetadata: ObjectMetadataInterface;
objectMetadata: Omit<ObjectMetadataInterface, 'indexMetadatas'>;
properties: Properties<T>;
}

1
packages/twenty-server/src/engine/core-modules/event-emitter/utils/__tests__/object-record-changed-values.spec.ts
View File

@ -4,6 +4,7 @@ import { objectRecordChangedValues } from 'src/engine/core-modules/event-emitter
const mockObjectMetadata: ObjectMetadataInterface = {
id: '1',
icon: 'Icon123',
nameSingular: 'Object',
namePlural: 'Objects',
labelSingular: 'Object',

10
packages/twenty-server/src/engine/core-modules/feature-flag/services/feature-flag.service.ts
View File

@ -74,10 +74,7 @@ export class FeatureFlagService {
);
await this.workspaceFeatureFlagsMapCacheService.recomputeFeatureFlagsMapCache(
{
workspaceId,
ignoreLock: true,
},
{ workspaceId },
);
}
}
@ -132,10 +129,7 @@ export class FeatureFlagService {
const result = await this.featureFlagRepository.save(featureFlagToSave);
await this.workspaceFeatureFlagsMapCacheService.recomputeFeatureFlagsMapCache(
{
workspaceId,
ignoreLock: true,
},
{ workspaceId },
);
return result;

14
packages/twenty-server/src/engine/core-modules/record-transformer/services/record-input-transformer.service.ts
View File

@ -3,8 +3,6 @@ import { Injectable } from '@nestjs/common';
import { FieldMetadataType } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { transformLinksValue } from 'src/engine/core-modules/record-transformer/utils/transform-links-value.util';
import { transformPhonesValue } from 'src/engine/core-modules/record-transformer/utils/transform-phones-value.util';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
@ -29,19 +27,11 @@ export class RecordInputTransformerService {
return recordInput;
}
const fieldMetadataByFieldName = objectMetadataMapItem.fields.reduce(
(acc, field) => {
acc[field.name] = field;
return acc;
},
{} as Record<string, FieldMetadataInterface>,
);
let transformedEntries = {};
for (const [key, value] of Object.entries(recordInput)) {
const fieldMetadata = fieldMetadataByFieldName[key];
const fieldMetadataId = objectMetadataMapItem.fieldIdByName[key];
const fieldMetadata = objectMetadataMapItem.fieldsById[fieldMetadataId];
if (!fieldMetadata) {
transformedEntries = { ...transformedEntries, [key]: value };

10
packages/twenty-server/src/engine/core-modules/search/services/search.service.ts
View File

@ -163,9 +163,13 @@ export class SearchService {
const queryBuilder = entityManager.createQueryBuilder();
const queryParser = new GraphqlQueryParser(
objectMetadataItem.fieldsByName,
objectMetadataItem.fieldsByJoinColumnName,
generateObjectMetadataMaps([objectMetadataItem]),
objectMetadataItem,
generateObjectMetadataMaps([
{
...objectMetadataItem,
fields: Object.values(objectMetadataItem.fieldsById),
},
]),
);
queryParser.applyFilterToBuilder(