mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-create-many.handler.ts
View File

@ -5,6 +5,8 @@ import { isDefined } from 'twenty-shared/utils';
import { RestApiBaseHandler } from 'src/engine/api/rest/core/interfaces/rest-api-base.handler';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
@Injectable()
export class RestApiCreateManyHandler extends RestApiBaseHandler {
async handle(request: Request) {
@ -57,7 +59,9 @@ export class RestApiCreateManyHandler extends RestApiBaseHandler {
this.apiEventEmitterService.emitCreateEvents({
records: createdRecords,
authContext: this.getAuthContextFromRequest(request),
objectMetadataItem: objectMetadata.objectMetadataMapItem,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadata.objectMetadataMapItem,
),
});
const records = await this.getRecord({

6
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-create-one.handler.ts
View File

@ -5,6 +5,8 @@ import { isDefined } from 'twenty-shared/utils';
import { RestApiBaseHandler } from 'src/engine/api/rest/core/interfaces/rest-api-base.handler';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
@Injectable()
export class RestApiCreateOneHandler extends RestApiBaseHandler {
async handle(request: Request) {
@ -40,7 +42,9 @@ export class RestApiCreateOneHandler extends RestApiBaseHandler {
this.apiEventEmitterService.emitCreateEvents({
records: [createdRecord],
authContext: this.getAuthContextFromRequest(request),
objectMetadataItem: objectMetadata.objectMetadataMapItem,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadata.objectMetadataMapItem,
),
});
const records = await this.getRecord({

5
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-delete-one.handler.ts
View File

@ -5,6 +5,7 @@ import { Request } from 'express';
import { RestApiBaseHandler } from 'src/engine/api/rest/core/interfaces/rest-api-base.handler';
import { parseCorePath } from 'src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
@Injectable()
export class RestApiDeleteOneHandler extends RestApiBaseHandler {
@ -26,7 +27,9 @@ export class RestApiDeleteOneHandler extends RestApiBaseHandler {
this.apiEventEmitterService.emitDestroyEvents({
records: [recordToDelete],
authContext: this.getAuthContextFromRequest(request),
objectMetadataItem: objectMetadata.objectMetadataMapItem,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadata.objectMetadataMapItem,
),
});
return this.formatResult({

4
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-find-duplicates.handler.ts
View File

@ -4,14 +4,14 @@ import { Request } from 'express';
import isEmpty from 'lodash.isempty';
import { In } from 'typeorm';
import { ObjectRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import {
FormatResult,
RestApiBaseHandler,
} from 'src/engine/api/rest/core/interfaces/rest-api-base.handler';
import { ObjectRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
@Injectable()
export class RestApiFindDuplicatesHandler extends RestApiBaseHandler {

5
packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-update-one.handler.ts
View File

@ -6,6 +6,7 @@ import { isDefined } from 'twenty-shared/utils';
import { RestApiBaseHandler } from 'src/engine/api/rest/core/interfaces/rest-api-base.handler';
import { parseCorePath } from 'src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
@Injectable()
export class RestApiUpdateOneHandler extends RestApiBaseHandler {
@ -38,7 +39,9 @@ export class RestApiUpdateOneHandler extends RestApiBaseHandler {
records: [updatedRecord],
updatedFields: Object.keys(request.body),
authContext: this.getAuthContextFromRequest(request),
objectMetadataItem: objectMetadata.objectMetadataMapItem,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadata.objectMetadataMapItem,
),
});
const records = await this.getRecord({

65
packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts
View File

@ -160,32 +160,34 @@ export abstract class RestApiBaseHandler {
const relations: string[] = [];
objectMetadata.objectMetadataMapItem.fields.forEach((field) => {
if (field.type === FieldMetadataType.RELATION) {
if (
depth === MAX_DEPTH &&
isDefined(field.relationTargetObjectMetadataId)
) {
const relationTargetObjectMetadata =
objectMetadata.objectMetadataMaps.byId[
field.relationTargetObjectMetadataId
];
const depth2Relations = this.getRelations({
objectMetadata: {
objectMetadataMaps: objectMetadata.objectMetadataMaps,
objectMetadataMapItem: relationTargetObjectMetadata,
},
depth: 1,
});
Object.values(objectMetadata.objectMetadataMapItem.fieldsById).forEach(
(field) => {
if (field.type === FieldMetadataType.RELATION) {
if (
depth === MAX_DEPTH &&
isDefined(field.relationTargetObjectMetadataId)
) {
const relationTargetObjectMetadata =
objectMetadata.objectMetadataMaps.byId[
field.relationTargetObjectMetadataId
];
const depth2Relations = this.getRelations({
objectMetadata: {
objectMetadataMaps: objectMetadata.objectMetadataMaps,
objectMetadataMapItem: relationTargetObjectMetadata,
},
depth: 1,
});
depth2Relations.forEach((depth2Relation) => {
relations.push(`${field.name}.${depth2Relation}`);
});
} else {
relations.push(`${field.name}`);
depth2Relations.forEach((depth2Relation) => {
relations.push(`${field.name}.${depth2Relation}`);
});
} else {
relations.push(`${field.name}`);
}
}
}
});
},
);
return relations;
}
@ -305,9 +307,7 @@ export abstract class RestApiBaseHandler {
objectMetadataMaps: ObjectMetadataMaps;
objectMetadataMapItem: ObjectMetadataItemWithFieldMaps;
};
objectMetadataItemWithFieldsMaps:
| ObjectMetadataItemWithFieldMaps
| undefined;
objectMetadataItemWithFieldsMaps: ObjectMetadataItemWithFieldMaps;
extraFilters?: Partial<ObjectRecordFilter>;
}) {
const objectMetadataNameSingular =
@ -321,17 +321,10 @@ export abstract class RestApiBaseHandler {
objectMetadata,
);
const fieldMetadataMapByName =
objectMetadataItemWithFieldsMaps?.fieldsByName || {};
const fieldMetadataMapByJoinColumnName =
objectMetadataItemWithFieldsMaps?.fieldsByJoinColumnName || {};
const isForwardPagination = !inputs.endingBefore;
const graphqlQueryParser = new GraphqlQueryParser(
fieldMetadataMapByName,
fieldMetadataMapByJoinColumnName,
objectMetadataItemWithFieldsMaps,
objectMetadata.objectMetadataMaps,
);
@ -442,7 +435,7 @@ export abstract class RestApiBaseHandler {
const cursorArgFilter = computeCursorArgFilter(
this.parseCursor(cursor),
inputs.orderBy || [],
objectMetadata.objectMetadataMapItem.fieldsByName,
objectMetadata.objectMetadataMapItem,
isForwardPagination,
);

2
packages/twenty-server/src/engine/api/rest/core/query-builder/factories/create-many-query.factory.ts
View File

@ -26,7 +26,7 @@ export class CreateManyQueryFactory {
mutation Create${objectNamePlural}($data: [${objectNameSingular}CreateInput!]) {
create${objectNamePlural}(data: $data) {
id
${objectMetadata.objectMetadataMapItem.fields
${Object.values(objectMetadata.objectMetadataMapItem.fieldsById)
.map((field) =>
mapFieldMetadataToGraphqlQuery(
objectMetadata.objectMetadataMaps,

2
packages/twenty-server/src/engine/api/rest/core/query-builder/factories/find-duplicates-query.factory.ts
View File

@ -31,7 +31,7 @@ export class FindDuplicatesQueryFactory {
}
edges{
node {
${objectMetadata.objectMetadataMapItem.fields
${Object.values(objectMetadata.objectMetadataMapItem.fieldsById)
.map((field) =>
mapFieldMetadataToGraphqlQuery(
objectMetadata.objectMetadataMaps,

14
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/__tests__/check-fields.utils.spec.ts
View File

@ -17,21 +17,23 @@ describe('checkFields', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldNumberMock.isNullable,
defaultValue: fieldNumberMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const fieldsById: FieldMetadataMap = {
'field-number-id': completeFieldNumberMock,
};
const fieldsByName: FieldMetadataMap = {
[completeFieldNumberMock.name]: completeFieldNumberMock,
};
const mockObjectMetadataWithFieldMaps = {
...objectMetadataItemMock,
fieldsById,
fieldsByName,
fieldsByJoinColumnName: {},
fieldIdByName: {
[completeFieldNumberMock.name]: completeFieldNumberMock.id,
},
fieldIdByJoinColumnName: {},
indexMetadatas: [],
};
it('should check field types', () => {

14
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/__tests__/get-field-type.utils.spec.ts
View File

@ -18,21 +18,23 @@ describe('getFieldType', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldNumberMock.isNullable,
defaultValue: fieldNumberMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const fieldsById: FieldMetadataMap = {
'field-number-id': completeFieldNumberMock,
};
const fieldsByName: FieldMetadataMap = {
[completeFieldNumberMock.name]: completeFieldNumberMock,
};
const mockObjectMetadataWithFieldMaps = {
...objectMetadataItemMock,
fieldsById,
fieldsByName,
fieldsByJoinColumnName: {},
fieldIdByName: {
[completeFieldNumberMock.name]: completeFieldNumberMock.id,
},
fieldIdByJoinColumnName: {},
indexMetadatas: [],
};
it('should get field type', () => {

28
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/__tests__/map-field-metadata-to-graphql-query.utils.spec.ts
View File

@ -24,6 +24,9 @@ describe('mapFieldMetadataToGraphqlQuery', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldNumberMock.isNullable,
defaultValue: fieldNumberMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const typedFieldTextMock: FieldMetadataInterface = {
@ -34,6 +37,9 @@ describe('mapFieldMetadataToGraphqlQuery', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldTextMock.isNullable,
defaultValue: fieldTextMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const typedFieldCurrencyMock: FieldMetadataInterface = {
@ -44,6 +50,9 @@ describe('mapFieldMetadataToGraphqlQuery', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldCurrencyMock.isNullable,
defaultValue: fieldCurrencyMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const fieldsById: FieldMetadataMap = {
@ -52,17 +61,16 @@ describe('mapFieldMetadataToGraphqlQuery', () => {
'field-currency-id': typedFieldCurrencyMock,
};
const fieldsByName: FieldMetadataMap = {
[typedFieldNumberMock.name]: typedFieldNumberMock,
[typedFieldTextMock.name]: typedFieldTextMock,
[typedFieldCurrencyMock.name]: typedFieldCurrencyMock,
};
const typedObjectMetadataItem: ObjectMetadataItemWithFieldMaps = {
...objectMetadataItemMock,
fieldsById,
fieldsByName,
fieldsByJoinColumnName: {},
fieldIdByName: {
[typedFieldNumberMock.name]: typedFieldNumberMock.id,
[typedFieldTextMock.name]: typedFieldTextMock.id,
[typedFieldCurrencyMock.name]: typedFieldCurrencyMock.id,
},
fieldIdByJoinColumnName: {},
indexMetadatas: [],
};
const objectMetadataMapsMock: ObjectMetadataMaps = {
@ -110,6 +118,10 @@ describe('mapFieldMetadataToGraphqlQuery', () => {
name: 'toObjectMetadataName',
label: 'Test Field',
objectMetadataId: 'object-metadata-id',
isNullable: true,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
if (fieldMetadataType === FieldMetadataType.RELATION) {

2
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/check-fields.utils.ts
View File

@ -9,7 +9,7 @@ export const checkFields = (
objectMetadataItem: ObjectMetadataItemWithFieldMaps,
fieldNames: string[],
): void => {
const fieldMetadataNames = objectMetadataItem.fields
const fieldMetadataNames = Object.values(objectMetadataItem.fieldsById)
.map((field) => {
if (isCompositeFieldMetadataType(field.type)) {
const compositeType = compositeTypeDefinitions.get(field.type);

2
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/check-order-by.utils.ts
View File

@ -11,7 +11,7 @@ export const checkArrayFields = (
objectMetadataItem: ObjectMetadataItemWithFieldMaps,
fields: Array<Partial<ObjectRecord>>,
): void => {
const fieldMetadataNames = objectMetadataItem.fields
const fieldMetadataNames = Object.values(objectMetadataItem.fieldsById)
.map((field) => {
if (isCompositeFieldMetadataType(field.type)) {
const compositeType = compositeTypeDefinitions.get(field.type);

14
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/filter-utils/__tests__/check-filter-enum-values.spec.ts
View File

@ -19,21 +19,23 @@ describe('checkFilterEnumValues', () => {
isNullable: fieldSelectMock.isNullable,
defaultValue: fieldSelectMock.defaultValue,
options: fieldSelectMock.options,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const fieldsById: FieldMetadataMap = {
'field-select-id': completeFieldSelectMock,
};
const fieldsByName: FieldMetadataMap = {
[completeFieldSelectMock.name]: completeFieldSelectMock,
};
const mockObjectMetadataWithFieldMaps = {
...objectMetadataItemMock,
fieldsById,
fieldsByName,
fieldsByJoinColumnName: {},
fieldIdByName: {
[completeFieldSelectMock.name]: completeFieldSelectMock.id,
},
fieldIdByJoinColumnName: {},
indexMetadatas: [],
};
it('should check properly', () => {

22
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/filter-utils/__tests__/parse-filter.utils.spec.ts
View File

@ -7,6 +7,7 @@ import {
} from 'src/engine/api/__mocks__/object-metadata-item.mock';
import { parseFilter } from 'src/engine/api/rest/core/query-builder/utils/filter-utils/parse-filter.utils';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
describe('parseFilter', () => {
const completeFieldNumberMock: FieldMetadataInterface = {
@ -17,6 +18,9 @@ describe('parseFilter', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldNumberMock.isNullable,
defaultValue: fieldNumberMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const completeFieldTextMock: FieldMetadataInterface = {
@ -27,6 +31,9 @@ describe('parseFilter', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldTextMock.isNullable,
defaultValue: fieldTextMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const fieldsById: FieldMetadataMap = {
@ -34,16 +41,15 @@ describe('parseFilter', () => {
'field-text-id': completeFieldTextMock,
};
const fieldsByName: FieldMetadataMap = {
[completeFieldNumberMock.name]: completeFieldNumberMock,
[completeFieldTextMock.name]: completeFieldTextMock,
};
const mockObjectMetadataWithFieldMaps = {
const mockObjectMetadataWithFieldMaps: ObjectMetadataItemWithFieldMaps = {
...objectMetadataItemMock,
fieldsById,
fieldsByName,
fieldsByJoinColumnName: {},
fieldIdByName: {
[completeFieldNumberMock.name]: completeFieldNumberMock.id,
[completeFieldTextMock.name]: completeFieldTextMock.id,
},
fieldIdByJoinColumnName: {},
indexMetadatas: [],
};
it('should parse string filter test 1', () => {

3
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/filter-utils/check-filter-enum-values.ts
View File

@ -18,7 +18,8 @@ export const checkFilterEnumValues = (
) {
return;
}
const field = objectMetadataItem.fieldsByName[fieldName];
const fieldMetadataId = objectMetadataItem.fieldIdByName[fieldName];
const field = objectMetadataItem.fieldsById[fieldMetadataId];
const values = /^\[.*\]$/.test(value)
? value.slice(1, -1).split(',')

5
packages/twenty-server/src/engine/api/rest/core/query-builder/utils/get-field-type.utils.ts
View File

@ -6,5 +6,8 @@ export const getFieldType = (
objectMetadataItem: ObjectMetadataItemWithFieldMaps,
fieldName: string,
): FieldMetadataType | undefined => {
return objectMetadataItem.fieldsByName[fieldName]?.type;
const fieldMetadataId = objectMetadataItem.fieldIdByName[fieldName];
const field = objectMetadataItem.fieldsById[fieldMetadataId];
return field?.type;
};

25
packages/twenty-server/src/engine/api/rest/input-factories/__tests__/filter-input.factory.spec.ts
View File

@ -10,6 +10,7 @@ import {
} from 'src/engine/api/__mocks__/object-metadata-item.mock';
import { FilterInputFactory } from 'src/engine/api/rest/input-factories/filter-input.factory';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
describe('FilterInputFactory', () => {
const completeFieldNumberMock: FieldMetadataInterface = {
@ -20,6 +21,9 @@ describe('FilterInputFactory', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldNumberMock.isNullable,
defaultValue: fieldNumberMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const completeFieldTextMock: FieldMetadataInterface = {
@ -30,6 +34,9 @@ describe('FilterInputFactory', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldTextMock.isNullable,
defaultValue: fieldTextMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const completeFieldCurrencyMock: FieldMetadataInterface = {
@ -40,6 +47,9 @@ describe('FilterInputFactory', () => {
objectMetadataId: 'object-metadata-id',
isNullable: fieldCurrencyMock.isNullable,
defaultValue: fieldCurrencyMock.defaultValue,
isLabelSyncedWithName: true,
createdAt: new Date(),
updatedAt: new Date(),
};
const fieldsById: FieldMetadataMap = {
@ -48,16 +58,15 @@ describe('FilterInputFactory', () => {
'field-currency-id': completeFieldCurrencyMock,
};
const fieldsByName: FieldMetadataMap = {
[completeFieldNumberMock.name]: completeFieldNumberMock,
[completeFieldTextMock.name]: completeFieldTextMock,
[completeFieldCurrencyMock.name]: completeFieldCurrencyMock,
};
const objectMetadataMapItem = {
const objectMetadataMapItem: ObjectMetadataItemWithFieldMaps = {
...objectMetadataMapItemMock,
fieldsById,
fieldsByName,
fieldIdByName: {
[completeFieldNumberMock.name]: completeFieldNumberMock.id,
[completeFieldTextMock.name]: completeFieldTextMock.id,
[completeFieldCurrencyMock.name]: completeFieldCurrencyMock.id,
},
fieldIdByJoinColumnName: {},
};
const objectMetadataMaps = {