admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
packages/twenty-server/src/modules/workflow/workflow-builder/workflow-schema/utils/__tests__/should-generate-field-fake-value.spec.ts
View File

@ -1,6 +1,7 @@
import { FieldMetadataType } from 'twenty-shared/types';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { shouldGenerateFieldFakeValue } from 'src/modules/workflow/workflow-builder/workflow-schema/utils/should-generate-field-fake-value';
describe('shouldGenerateFieldFakeValue', () => {
@ -10,7 +11,9 @@ describe('shouldGenerateFieldFakeValue', () => {
isActive: true,
type: FieldMetadataType.TEXT,
name: 'testField',
} as FieldMetadataEntity;
createdAt: new Date(),
updatedAt: new Date(),
} as FieldMetadataInterface;
expect(shouldGenerateFieldFakeValue(field)).toBe(true);
});
@ -21,7 +24,9 @@ describe('shouldGenerateFieldFakeValue', () => {
isActive: true,
type: FieldMetadataType.UUID,
name: 'id',
} as FieldMetadataEntity;
createdAt: new Date(),
updatedAt: new Date(),
} as FieldMetadataInterface;
expect(shouldGenerateFieldFakeValue(field)).toBe(true);
});
@ -32,7 +37,9 @@ describe('shouldGenerateFieldFakeValue', () => {
isActive: false,
type: FieldMetadataType.TEXT,
name: 'testField',
} as FieldMetadataEntity;
createdAt: new Date(),
updatedAt: new Date(),
} as FieldMetadataInterface;
expect(shouldGenerateFieldFakeValue(field)).toBe(false);
});
@ -43,7 +50,9 @@ describe('shouldGenerateFieldFakeValue', () => {
isActive: true,
type: FieldMetadataType.TEXT,
name: 'testField',
} as FieldMetadataEntity;
createdAt: new Date(),
updatedAt: new Date(),
} as FieldMetadataInterface;
expect(shouldGenerateFieldFakeValue(field)).toBe(false);
});
@ -54,7 +63,9 @@ describe('shouldGenerateFieldFakeValue', () => {
isActive: true,
type: FieldMetadataType.RELATION,
name: 'testField',
} as FieldMetadataEntity;
createdAt: new Date(),
updatedAt: new Date(),
} as FieldMetadataInterface;
expect(shouldGenerateFieldFakeValue(field)).toBe(false);
});

77
packages/twenty-server/src/modules/workflow/workflow-builder/workflow-schema/utils/generate-object-record-fields.ts
View File

@ -18,44 +18,47 @@ export const generateObjectRecordFields = ({
}): BaseOutputSchema => {
const objectMetadata = objectMetadataInfo.objectMetadataItemWithFieldsMaps;
return objectMetadata.fields.reduce((acc: BaseOutputSchema, field) => {
if (!shouldGenerateFieldFakeValue(field)) {
return acc;
}
return Object.values(objectMetadata.fieldsById).reduce(
(acc: BaseOutputSchema, field) => {
if (!shouldGenerateFieldFakeValue(field)) {
return acc;
}
if (field.type !== FieldMetadataType.RELATION) {
acc[field.name] = generateFakeField({
type: field.type,
label: field.label,
icon: field.icon,
});
if (field.type !== FieldMetadataType.RELATION) {
acc[field.name] = generateFakeField({
type: field.type,
label: field.label,
icon: field.icon,
});
return acc;
}
if (
depth < MAXIMUM_DEPTH &&
isDefined(field.relationTargetObjectMetadataId)
) {
const relationTargetObjectMetadata =
objectMetadataInfo.objectMetadataMaps.byId[
field.relationTargetObjectMetadataId
];
acc[field.name] = {
isLeaf: false,
icon: field.icon,
label: field.label,
value: generateFakeObjectRecord({
objectMetadataInfo: {
objectMetadataItemWithFieldsMaps: relationTargetObjectMetadata,
objectMetadataMaps: objectMetadataInfo.objectMetadataMaps,
},
depth: depth + 1,
}),
};
}
return acc;
}
if (
depth < MAXIMUM_DEPTH &&
isDefined(field.relationTargetObjectMetadataId)
) {
const relationTargetObjectMetadata =
objectMetadataInfo.objectMetadataMaps.byId[
field.relationTargetObjectMetadataId
];
acc[field.name] = {
isLeaf: false,
icon: field.icon,
label: field.label,
value: generateFakeObjectRecord({
objectMetadataInfo: {
objectMetadataItemWithFieldsMaps: relationTargetObjectMetadata,
objectMetadataMaps: objectMetadataInfo.objectMetadataMaps,
},
depth: depth + 1,
}),
};
}
return acc;
}, {} as BaseOutputSchema);
},
{} as BaseOutputSchema,
);
};

5
packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/record-crud/create-record.workflow-action.ts
View File

@ -1,6 +1,7 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { isDefined } from 'class-validator';
import { Repository } from 'typeorm';
import { WorkflowExecutor } from 'src/modules/workflow/workflow-executor/interfaces/workflow-executor.interface';
@ -109,8 +110,8 @@ export class CreateRecordWorkflowAction implements WorkflowExecutor {
);
const validObjectRecord = Object.fromEntries(
Object.entries(workflowActionInput.objectRecord).filter(
([key]) => objectMetadataItemWithFieldsMaps.fieldsByName[key],
Object.entries(workflowActionInput.objectRecord).filter(([key]) =>
isDefined(objectMetadataItemWithFieldsMaps.fieldIdByName[key]),
),
);

3
packages/twenty-server/src/modules/workflow/workflow-executor/workflow-actions/record-crud/find-records.workflow-action.ts
View File

@ -90,8 +90,7 @@ export class FindRecordsWorkflowAction implements WorkflowExecutor {
);
const graphqlQueryParser = new GraphqlQueryParser(
objectMetadataItemWithFieldsMaps.fieldsByName,
objectMetadataItemWithFieldsMaps.fieldsByJoinColumnName,
objectMetadataItemWithFieldsMaps,
objectMetadataMaps,
);

24
packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/__tests__/database-event-trigger.listener.spec.ts
View File

@ -1,6 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { AutomatedTriggerType } from 'src/modules/workflow/common/standard-objects/workflow-automated-trigger.workspace-entity';
import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service';
@ -54,8 +55,26 @@ describe('DatabaseEventTriggerListener', () => {
},
},
objectMetadataItemWithFieldsMaps: {
fieldsByJoinColumnName: {},
},
id: 'test-object-metadata',
workspaceId: 'test-workspace',
nameSingular: 'testObject',
namePlural: 'testObjects',
labelSingular: 'Test Object',
labelPlural: 'Test Objects',
description: 'Test object for testing',
fieldIdByJoinColumnName: {},
fieldsById: {},
fieldIdByName: {},
indexMetadatas: [],
targetTableName: 'test_objects',
isSystem: false,
isCustom: false,
isActive: true,
isRemote: false,
isAuditLogged: true,
isSearchable: true,
icon: 'Icon123',
} satisfies ObjectMetadataItemWithFieldMaps,
}),
},
},
@ -97,6 +116,7 @@ describe('DatabaseEventTriggerListener', () => {
updatedAt: new Date(),
fields: [],
indexMetadatas: [],
icon: 'Icon123',
},
properties: {
updatedFields: ['field1', 'field2'],

11
packages/twenty-server/src/modules/workflow/workflow-trigger/automated-trigger/listeners/database-event-trigger.listener.ts
View File

@ -183,13 +183,12 @@ export class DatabaseEventTriggerListener {
workspaceId,
);
const fieldsByJoinColumnName =
objectMetadataItemWithFieldsMaps.fieldsByJoinColumnName;
for (const [joinColumn, joinField] of Object.entries(
fieldsByJoinColumnName,
for (const [joinColumnName, joinFieldId] of Object.entries(
objectMetadataItemWithFieldsMaps.fieldIdByJoinColumnName,
)) {
const joinRecordId = record[joinColumn];
const joinField =
objectMetadataItemWithFieldsMaps.fieldsById[joinFieldId];
const joinRecordId = record[joinColumnName];
const relatedObjectMetadataId = joinField.relationTargetObjectMetadataId;
if (!isDefined(relatedObjectMetadataId)) {