mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/twenty-server/src/engine/api/utils/__tests__/build-duplicate-conditions.utils.spec.ts
View File

@ -1,11 +1,11 @@
import { mockPersonObjectMetadata } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonObjectMetadata';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
import { mockPersonObjectMetadataWithFieldMaps } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonObjectMetadata';
import { mockPersonRecords } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonRecords';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
describe('buildDuplicateConditions', () => {
it('should build conditions based on duplicate criteria from composite field', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['emailsPrimaryEmail']]),
mockPersonObjectMetadataWithFieldMaps([['emailsPrimaryEmail']]),
mockPersonRecords,
'recordId',
);
@ -13,8 +13,10 @@ describe('buildDuplicateConditions', () => {
expect(duplicateConditons).toEqual({
or: [
{
emailsPrimaryEmail: {
eq: 'test@test.fr',
emails: {
primaryEmail: {
eq: 'test@test.fr',
},
},
},
],
@ -26,7 +28,7 @@ describe('buildDuplicateConditions', () => {
it('should build conditions based on duplicate criteria from basic field', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonObjectMetadataWithFieldMaps([['jobTitle']]),
mockPersonRecords,
'recordId',
);
@ -47,7 +49,7 @@ describe('buildDuplicateConditions', () => {
it('should not build conditions based on duplicate criteria if record value is null or too small', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['linkedinLinkPrimaryLinkUrl']]),
mockPersonObjectMetadataWithFieldMaps([['linkedinLinkPrimaryLinkUrl']]),
mockPersonRecords,
'recordId',
);
@ -57,7 +59,7 @@ describe('buildDuplicateConditions', () => {
it('should build conditions based on duplicate criteria and without recordId filter', () => {
const duplicateConditons = buildDuplicateConditions(
mockPersonObjectMetadata([['jobTitle']]),
mockPersonObjectMetadataWithFieldMaps([['jobTitle']]),
mockPersonRecords,
);

111
packages/twenty-server/src/engine/api/utils/__tests__/compute-cursor-arg-filter.utils.spec.ts
View File

@ -4,35 +4,76 @@ import { OrderByDirection } from 'src/engine/api/graphql/workspace-query-builder
import { GraphqlQueryRunnerException } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { computeCursorArgFilter } from 'src/engine/api/utils/compute-cursor-arg-filter.utils';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
describe('computeCursorArgFilter', () => {
const mockFieldMetadataMap = {
name: {
type: FieldMetadataType.TEXT,
id: 'name-id',
name: 'name',
label: 'Name',
objectMetadataId: 'object-id',
const objectMetadataItemWithFieldMaps = {
id: 'object-id',
workspaceId: 'workspace-id',
nameSingular: 'person',
namePlural: 'people',
isCustom: false,
isRemote: false,
labelSingular: 'Person',
labelPlural: 'People',
targetTableName: 'person',
indexMetadatas: [],
isSystem: false,
isActive: true,
isAuditLogged: false,
isSearchable: false,
fieldIdByJoinColumnName: {},
icon: 'Icon123',
fieldIdByName: {
name: 'name-id',
age: 'age-id',
fullName: 'fullname-id',
},
age: {
type: FieldMetadataType.NUMBER,
id: 'age-id',
name: 'age',
label: 'Age',
objectMetadataId: 'object-id',
fieldsById: {
'name-id': {
type: FieldMetadataType.TEXT,
id: 'name-id',
name: 'name',
label: 'Name',
objectMetadataId: 'object-id',
isLabelSyncedWithName: true,
isNullable: true,
createdAt: new Date(),
updatedAt: new Date(),
},
'age-id': {
type: FieldMetadataType.NUMBER,
id: 'age-id',
name: 'age',
label: 'Age',
objectMetadataId: 'object-id',
isLabelSyncedWithName: true,
isNullable: true,
createdAt: new Date(),
updatedAt: new Date(),
},
'fullname-id': {
type: FieldMetadataType.FULL_NAME,
id: 'fullname-id',
name: 'fullName',
label: 'Full Name',
objectMetadataId: 'object-id',
isLabelSyncedWithName: true,
isNullable: true,
createdAt: new Date(),
updatedAt: new Date(),
},
},
fullName: {
type: FieldMetadataType.FULL_NAME,
id: 'fullname-id',
name: 'fullName',
label: 'Full Name',
objectMetadataId: 'object-id',
},
};
} satisfies ObjectMetadataItemWithFieldMaps;
describe('basic cursor filtering', () => {
it('should return empty array when cursor is empty', () => {
const result = computeCursorArgFilter({}, [], mockFieldMetadataMap, true);
const result = computeCursorArgFilter(
{},
[],
objectMetadataItemWithFieldMaps,
true,
);
expect(result).toEqual([]);
});
@ -44,7 +85,7 @@ describe('computeCursorArgFilter', () => {
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
objectMetadataItemWithFieldMaps,
true,
);
@ -58,7 +99,7 @@ describe('computeCursorArgFilter', () => {
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
objectMetadataItemWithFieldMaps,
false,
);
@ -77,7 +118,7 @@ describe('computeCursorArgFilter', () => {
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
objectMetadataItemWithFieldMaps,
true,
);
@ -105,7 +146,7 @@ describe('computeCursorArgFilter', () => {
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
objectMetadataItemWithFieldMaps,
true,
);
@ -151,7 +192,7 @@ describe('computeCursorArgFilter', () => {
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
objectMetadataItemWithFieldMaps,
true,
);
@ -180,7 +221,7 @@ describe('computeCursorArgFilter', () => {
const result = computeCursorArgFilter(
cursor,
orderBy,
mockFieldMetadataMap,
objectMetadataItemWithFieldMaps,
false,
);
@ -218,7 +259,12 @@ describe('computeCursorArgFilter', () => {
const orderBy = [{ invalidField: OrderByDirection.AscNullsLast }];
expect(() =>
computeCursorArgFilter(cursor, orderBy, mockFieldMetadataMap, true),
computeCursorArgFilter(
cursor,
orderBy,
objectMetadataItemWithFieldMaps,
true,
),
).toThrow(GraphqlQueryRunnerException);
});
@ -227,7 +273,12 @@ describe('computeCursorArgFilter', () => {
const orderBy = [{ age: OrderByDirection.AscNullsLast }];
expect(() =>
computeCursorArgFilter(cursor, orderBy, mockFieldMetadataMap, true),
computeCursorArgFilter(
cursor,
orderBy,
objectMetadataItemWithFieldMaps,
true,
),
).toThrow(GraphqlQueryRunnerException);
});
});

13
packages/twenty-server/src/engine/api/utils/build-cursor-where-condition.utils.ts
View File

@ -11,19 +11,19 @@ import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { buildCursorCompositeFieldWhereCondition } from 'src/engine/api/utils/build-cursor-composite-field-where-condition.utils';
import { computeOperator } from 'src/engine/api/utils/compute-operator.utils';
import { isAscendingOrder } from 'src/engine/api/utils/is-ascending-order.utils';
import { validateAndGetOrderByForScalarField } from 'src/engine/api/utils/validate-and-get-order-by.utils';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { buildCursorCompositeFieldWhereCondition } from 'src/engine/api/utils/build-cursor-composite-field-where-condition.utils';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
type BuildCursorWhereConditionParams = {
cursorKey: keyof ObjectRecord;
cursorValue:
| ObjectRecordCursorLeafScalarValue
| ObjectRecordCursorLeafCompositeValue;
fieldMetadataMapByName: FieldMetadataMap;
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps;
orderBy: ObjectRecordOrderBy;
isForwardPagination: boolean;
isEqualityCondition?: boolean;
@ -32,12 +32,15 @@ type BuildCursorWhereConditionParams = {
export const buildCursorWhereCondition = ({
cursorKey,
cursorValue,
fieldMetadataMapByName,
objectMetadataItemWithFieldMaps,
orderBy,
isForwardPagination,
isEqualityCondition = false,
}: BuildCursorWhereConditionParams): Record<string, unknown> => {
const fieldMetadata = fieldMetadataMapByName[cursorKey];
const fieldMetadataId =
objectMetadataItemWithFieldMaps.fieldIdByName[cursorKey];
const fieldMetadata =
objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId];
if (!fieldMetadata) {
throw new GraphqlQueryRunnerException(

8
packages/twenty-server/src/engine/api/utils/compute-cursor-arg-filter.utils.ts
View File

@ -9,13 +9,13 @@ import {
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { buildCursorCumulativeWhereCondition } from 'src/engine/api/utils/build-cursor-cumulative-where-conditions.utils';
import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
import { buildCursorWhereCondition } from 'src/engine/api/utils/build-cursor-where-condition.utils';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
export const computeCursorArgFilter = (
cursor: ObjectRecordCursor,
orderBy: ObjectRecordOrderBy,
fieldMetadataMapByName: FieldMetadataMap,
objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
isForwardPagination = true,
): ObjectRecordFilter[] => {
const cursorEntries = Object.entries(cursor)
@ -42,7 +42,7 @@ export const computeCursorArgFilter = (
buildCursorWhereCondition({
cursorKey,
cursorValue,
fieldMetadataMapByName,
objectMetadataItemWithFieldMaps,
orderBy,
isForwardPagination: true,
isEqualityCondition: true,
@ -51,7 +51,7 @@ export const computeCursorArgFilter = (
buildCursorWhereCondition({
cursorKey,
cursorValue,
fieldMetadataMapByName,
objectMetadataItemWithFieldMaps,
orderBy,
isForwardPagination,
}),