mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
packages/twenty-server/src/engine/twenty-orm/utils/format-data.util.ts
View File

@ -7,7 +7,6 @@ import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-meta
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { isFieldMetadataInterfaceOfType } from 'src/engine/utils/is-field-metadata-of-type.util';
export function formatData<T>(
data: T,
@ -25,28 +24,14 @@ export function formatData<T>(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const newData: Record<string, any> = {};
const fieldMetadataByJoinColumnName =
objectMetadataItemWithFieldMaps.fields.reduce((acc, fieldMetadata) => {
if (
isFieldMetadataInterfaceOfType(
fieldMetadata,
FieldMetadataType.RELATION,
)
) {
const joinColumnName = fieldMetadata.settings?.joinColumnName;
if (joinColumnName) {
acc.set(joinColumnName, fieldMetadata);
}
}
return acc;
}, new Map<string, FieldMetadataInterface>());
for (const [key, value] of Object.entries(data)) {
const fieldMetadataId =
objectMetadataItemWithFieldMaps.fieldIdByName[key] ||
objectMetadataItemWithFieldMaps.fieldIdByJoinColumnName[key];
const fieldMetadata =
objectMetadataItemWithFieldMaps.fieldsByName[key] ||
fieldMetadataByJoinColumnName.get(key);
objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId];
if (!fieldMetadata) {
throw new Error(

24
packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.ts
View File

@ -44,15 +44,15 @@ export function formatResult<T>(
);
const newData: object = {};
const objectMetadaItemFieldsByName =
objectMetadataMaps.byId[objectMetadataItemWithFieldMaps.id]?.fieldsByName;
for (const [key, value] of Object.entries(data)) {
const compositePropertyArgs = compositeFieldMetadataMap.get(key);
const fieldMetadata = objectMetadataItemWithFieldMaps.fieldsByName[key] as
| FieldMetadataInterface<FieldMetadataType>
| undefined;
const fieldMetadataId = objectMetadataItemWithFieldMaps.fieldIdByName[key];
const fieldMetadata = objectMetadataItemWithFieldMaps.fieldsById[
fieldMetadataId
] as FieldMetadataInterface<FieldMetadataType> | undefined;
const isRelation = fieldMetadata
? isFieldMetadataInterfaceOfType(
@ -69,12 +69,9 @@ export function formatResult<T>(
objectMetadataItemWithFieldMaps,
objectMetadataMaps,
);
} else if (objectMetadaItemFieldsByName[key]) {
} else if (fieldMetadata) {
// @ts-expect-error legacy noImplicitAny
newData[key] = formatFieldMetadataValue(
value,
objectMetadaItemFieldsByName[key],
);
newData[key] = formatFieldMetadataValue(value, fieldMetadata);
} else {
// @ts-expect-error legacy noImplicitAny
newData[key] = value;
@ -123,10 +120,9 @@ export function formatResult<T>(
newData[parentField][compositeProperty.name] = value;
}
const dateFieldMetadataCollection =
objectMetadataItemWithFieldMaps.fields.filter(
(field) => field.type === FieldMetadataType.DATE,
);
const dateFieldMetadataCollection = Object.values(
objectMetadataItemWithFieldMaps.fieldsById,
).filter((field) => field.type === FieldMetadataType.DATE);
// This is a temporary fix to handle a bug in the frontend where the date gets returned in the wrong timezone,
// thus returning the wrong date.