admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
packages/twenty-server/src/engine/metadata-modules/utils/__tests__/validate-field-name-availability.spec.ts
View File

@ -6,7 +6,7 @@ import {
FIELD_CURRENCY_MOCK_NAME,
FIELD_FULL_NAME_MOCK_NAME,
FIELD_LINKS_MOCK_NAME,
objectMetadataItemMock,
objectMetadataMapItemMock,
} from 'src/engine/api/__mocks__/object-metadata-item.mock';
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
@ -57,18 +57,22 @@ const validateFieldNameAvailabilityTestCases: ValidateFieldNameAvailabilityTestC
];
describe('validateFieldNameAvailabilityOrThrow', () => {
const objectMetadata = objectMetadataItemMock;
it.each(validateFieldNameAvailabilityTestCases)(
'$title',
({ context: { input, shouldNotThrow } }) => {
if (shouldNotThrow) {
expect(() =>
validateFieldNameAvailabilityOrThrow(input, objectMetadata),
validateFieldNameAvailabilityOrThrow(
input,
objectMetadataMapItemMock,
),
).not.toThrow();
} else {
expect(() =>
validateFieldNameAvailabilityOrThrow(input, objectMetadata),
validateFieldNameAvailabilityOrThrow(
input,
objectMetadataMapItemMock,
),
).toThrowErrorMatchingSnapshot();
}
},

26
packages/twenty-server/src/engine/metadata-modules/utils/generate-object-metadata-maps.util.ts
View File

@ -1,3 +1,4 @@
import omit from 'lodash.omit';
import { FieldMetadataType } from 'twenty-shared/types';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
@ -16,9 +17,7 @@ export const generateObjectMetadataMaps = (
};
for (const objectMetadata of objectMetadataCollection) {
const fieldsByIdMap: FieldMetadataMap = {};
const fieldsByNameMap: FieldMetadataMap = {};
const fieldsByJoinColumnNameMap: FieldMetadataMap = {};
const fieldIdByJoinColumnNameMap: Record<string, string> = {};
for (const fieldMetadata of objectMetadata.fields) {
if (
@ -28,20 +27,25 @@ export const generateObjectMetadataMaps = (
)
) {
if (fieldMetadata.settings?.joinColumnName) {
fieldsByJoinColumnNameMap[fieldMetadata.settings.joinColumnName] =
fieldMetadata;
fieldIdByJoinColumnNameMap[fieldMetadata.settings.joinColumnName] =
fieldMetadata.id;
}
}
fieldsByNameMap[fieldMetadata.name] = fieldMetadata;
fieldsByIdMap[fieldMetadata.id] = fieldMetadata;
}
const fieldsByIdMap = objectMetadata.fields.reduce((acc, field) => {
acc[field.id] = field;
return acc;
}, {} as FieldMetadataMap);
const processedObjectMetadata: ObjectMetadataItemWithFieldMaps = {
...objectMetadata,
...omit(objectMetadata, 'fields'),
fieldsById: fieldsByIdMap,
fieldsByName: fieldsByNameMap,
fieldsByJoinColumnName: fieldsByJoinColumnNameMap,
fieldIdByName: Object.fromEntries(
Object.entries(fieldsByIdMap).map(([id, field]) => [field.name, id]),
),
fieldIdByJoinColumnName: fieldIdByJoinColumnNameMap,
};
objectMetadataMaps.byId[objectMetadata.id] = processedObjectMetadata;

18
packages/twenty-server/src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps.ts Normal file
View File

@ -0,0 +1,18 @@
import omit from 'lodash.omit';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
export const getObjectMetadataFromObjectMetadataItemWithFieldMaps = (
objectMetadataMapItem: ObjectMetadataItemWithFieldMaps,
): ObjectMetadataInterface => {
return {
...omit(objectMetadataMapItem, [
'fieldsById',
'fieldIdByName',
'fieldIdByJoinColumnName',
]),
fields: Object.values(objectMetadataMapItem.fieldsById),
};
};

14
packages/twenty-server/src/engine/metadata-modules/utils/remove-field-maps-from-object-metadata.util.ts
View File

@ -1,14 +0,0 @@
import omit from 'lodash.omit';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
export const removeFieldMapsFromObjectMetadata = (
objectMetadata: ObjectMetadataItemWithFieldMaps,
): ObjectMetadataInterface =>
omit(objectMetadata, [
'fieldsById',
'fieldsByName',
'fieldsByJoinColumnName',
]);

14
packages/twenty-server/src/engine/metadata-modules/utils/validate-field-name-availability.utils.ts
View File

@ -1,18 +1,18 @@
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import {
InvalidMetadataException,
InvalidMetadataExceptionCode,
} from 'src/engine/metadata-modules/utils/exceptions/invalid-metadata.exception';
const getReservedCompositeFieldNames = (
objectMetadata: ObjectMetadataEntity,
objectMetadata: ObjectMetadataItemWithFieldMaps,
) => {
const reservedCompositeFieldsNames: string[] = [];
for (const field of objectMetadata.fields) {
for (const field of Object.values(objectMetadata.fieldsById)) {
if (isCompositeFieldMetadataType(field.type)) {
const base = field.name;
const compositeType = compositeTypeDefinitions.get(field.type);
@ -30,12 +30,16 @@ const getReservedCompositeFieldNames = (
export const validateFieldNameAvailabilityOrThrow = (
name: string,
objectMetadata: ObjectMetadataEntity,
objectMetadata: ObjectMetadataItemWithFieldMaps,
) => {
const reservedCompositeFieldsNames =
getReservedCompositeFieldNames(objectMetadata);
if (objectMetadata.fields.some((field) => field.name === name)) {
if (
Object.values(objectMetadata.fieldsById).some(
(field) => field.name === name,
)
) {
throw new InvalidMetadataException(
`Name "${name}" is not available`,
InvalidMetadataExceptionCode.NOT_AVAILABLE,

35
packages/twenty-server/src/engine/metadata-modules/utils/validate-no-other-object-with-same-name-exists-or-throw.util.ts Normal file
View File

@ -0,0 +1,35 @@
import {
ObjectMetadataException,
ObjectMetadataExceptionCode,
} from 'src/engine/metadata-modules/object-metadata/object-metadata.exception';
import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
type ValidateNoOtherObjectWithSameNameExistsOrThrowsParams = {
objectMetadataNameSingular: string;
objectMetadataNamePlural: string;
existingObjectMetadataId?: string;
objectMetadataMaps: ObjectMetadataMaps;
};
export const validatesNoOtherObjectWithSameNameExistsOrThrows = ({
objectMetadataNameSingular,
objectMetadataNamePlural,
existingObjectMetadataId,
objectMetadataMaps,
}: ValidateNoOtherObjectWithSameNameExistsOrThrowsParams) => {
const objectAlreadyExists = Object.values(objectMetadataMaps.byId).find(
(objectMetadata) =>
(objectMetadata.nameSingular === objectMetadataNameSingular ||
objectMetadata.namePlural === objectMetadataNamePlural ||
objectMetadata.nameSingular === objectMetadataNamePlural ||
objectMetadata.namePlural === objectMetadataNameSingular) &&
objectMetadata.id !== existingObjectMetadataId,
);
if (objectAlreadyExists) {
throw new ObjectMetadataException(
'Object already exists',
ObjectMetadataExceptionCode.OBJECT_ALREADY_EXISTS,
);
}
};