mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts
View File

@ -17,7 +17,6 @@ import {
import { FieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-default-value.interface';
import { FieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-options.interface';
import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { FieldStandardOverridesDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-standard-overrides.dto';
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-field-metadata.entity';
@ -41,8 +40,7 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
])
export class FieldMetadataEntity<
T extends FieldMetadataType = FieldMetadataType,
> implements FieldMetadataInterface<T>
{
> {
@PrimaryGeneratedColumn('uuid')
id: string;

2
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts
View File

@ -24,6 +24,7 @@ import { IsFieldMetadataOptions } from 'src/engine/metadata-modules/field-metada
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
@ -54,6 +55,7 @@ import { UpdateFieldInput } from './dtos/update-field.input';
ActorModule,
ViewModule,
PermissionsModule,
WorkspaceMetadataCacheModule,
],
services: [
IsFieldMetadataDefaultValue,

167
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
View File

@ -10,6 +10,7 @@ import { isDefined } from 'twenty-shared/utils';
import { DataSource, FindOneOptions, In, Repository } from 'typeorm';
import { v4 as uuidV4, v4 } from 'uuid';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';
import { settings } from 'src/engine/constants/settings';
@ -40,9 +41,9 @@ import { generateNullable } from 'src/engine/metadata-modules/field-metadata/uti
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
import { isSelectOrMultiSelectFieldMetadata } from 'src/engine/metadata-modules/field-metadata/utils/is-select-or-multi-select-field-metadata.util';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { RelationOnDeleteAction } from 'src/engine/metadata-modules/relation-metadata/relation-on-delete-action.type';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { InvalidMetadataException } from 'src/engine/metadata-modules/utils/exceptions/invalid-metadata.exception';
import { validateFieldNameAvailabilityOrThrow } from 'src/engine/metadata-modules/utils/validate-field-name-availability.utils';
import { validateMetadataNameOrThrow } from 'src/engine/metadata-modules/utils/validate-metadata-name.utils';
@ -50,6 +51,7 @@ import {
computeMetadataNameFromLabel,
validateNameAndLabelAreSyncOrThrow,
} from 'src/engine/metadata-modules/utils/validate-name-and-label-are-sync-or-throw.util';
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
@ -78,8 +80,8 @@ type ValidateFieldMetadataArgs<T extends UpdateFieldInput | CreateFieldInput> =
{
fieldMetadataType: FieldMetadataType;
fieldMetadataInput: T;
objectMetadata: ObjectMetadataEntity;
existingFieldMetadata?: FieldMetadataEntity;
objectMetadata: ObjectMetadataItemWithFieldMaps;
existingFieldMetadata?: FieldMetadataInterface;
};
@Injectable()
@ -89,8 +91,6 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
private readonly coreDataSource: DataSource,
@InjectRepository(FieldMetadataEntity, 'core')
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
@InjectRepository(ObjectMetadataEntity, 'core')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly workspaceMigrationFactory: WorkspaceMigrationFactory,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
@ -100,6 +100,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
private readonly fieldMetadataValidationService: FieldMetadataValidationService,
private readonly fieldMetadataRelatedRecordsService: FieldMetadataRelatedRecordsService,
private readonly viewService: ViewService,
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
) {
super(fieldMetadataRepository);
}
@ -123,54 +124,49 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
id: string,
fieldMetadataInput: UpdateFieldInput,
): Promise<FieldMetadataEntity> {
const { objectMetadataMaps } =
await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
{ workspaceId: fieldMetadataInput.workspaceId },
);
let existingFieldMetadata: FieldMetadataInterface | undefined;
for (const objectMetadataItem of Object.values(objectMetadataMaps.byId)) {
const fieldMetadata = objectMetadataItem.fieldsById[id];
if (fieldMetadata) {
existingFieldMetadata = fieldMetadata;
break;
}
}
if (!isDefined(existingFieldMetadata)) {
throw new FieldMetadataException(
'Field does not exist',
FieldMetadataExceptionCode.FIELD_METADATA_NOT_FOUND,
);
}
const objectMetadataItemWithFieldMaps =
objectMetadataMaps.byId[existingFieldMetadata.objectMetadataId];
const queryRunner = this.coreDataSource.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
try {
const fieldMetadataRepository =
queryRunner.manager.getRepository<FieldMetadataEntity>(
FieldMetadataEntity,
);
const [existingFieldMetadata] = await fieldMetadataRepository.find({
where: {
id,
workspaceId: fieldMetadataInput.workspaceId,
},
});
if (!isDefined(existingFieldMetadata)) {
throw new FieldMetadataException(
'Field does not exist',
FieldMetadataExceptionCode.FIELD_METADATA_NOT_FOUND,
);
}
const [objectMetadata] = await this.objectMetadataRepository.find({
where: {
id: existingFieldMetadata.objectMetadataId,
workspaceId: fieldMetadataInput.workspaceId,
},
relations: ['fields'],
order: {},
});
if (!isDefined(objectMetadata)) {
throw new FieldMetadataException(
'Object metadata does not exist',
FieldMetadataExceptionCode.OBJECT_METADATA_NOT_FOUND,
);
}
if (!isDefined(objectMetadata.labelIdentifierFieldMetadataId)) {
if (
!isDefined(
objectMetadataItemWithFieldMaps.labelIdentifierFieldMetadataId,
)
) {
throw new FieldMetadataException(
'Label identifier field metadata id does not exist',
FieldMetadataExceptionCode.LABEL_IDENTIFIER_FIELD_METADATA_ID_NOT_FOUND,
);
}
assertMutationNotOnRemoteObject(objectMetadata);
assertMutationNotOnRemoteObject(objectMetadataItemWithFieldMaps);
assertDoesNotNullifyDefaultValueForNonNullableField({
isNullable: existingFieldMetadata.isNullable,
@ -180,19 +176,9 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
if (fieldMetadataInput.isActive === false) {
checkCanDeactivateFieldOrThrow({
labelIdentifierFieldMetadataId:
objectMetadata.labelIdentifierFieldMetadataId,
objectMetadataItemWithFieldMaps.labelIdentifierFieldMetadataId,
existingFieldMetadata,
});
const viewsRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
fieldMetadataInput.workspaceId,
'view',
);
await viewsRepository.delete({
kanbanFieldMetadataId: id,
});
}
const updatableFieldInput =
@ -221,7 +207,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
fieldMetadataType: existingFieldMetadata.type,
existingFieldMetadata,
fieldMetadataInput: fieldMetadataForUpdate,
objectMetadata,
objectMetadata: objectMetadataItemWithFieldMaps,
});
const isLabelSyncedWithName =
@ -236,9 +222,9 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
}
// We're running field update under a transaction, so we can rollback if migration fails
await fieldMetadataRepository.update(id, fieldMetadataForUpdate);
await this.fieldMetadataRepository.update(id, fieldMetadataForUpdate);
const [updatedFieldMetadata] = await fieldMetadataRepository.find({
const [updatedFieldMetadata] = await this.fieldMetadataRepository.find({
where: { id },
});
@ -249,6 +235,18 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
);
}
if (fieldMetadataInput.isActive === false) {
const viewsRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
fieldMetadataInput.workspaceId,
'view',
);
await viewsRepository.delete({
kanbanFieldMetadataId: id,
});
}
if (
updatedFieldMetadata.isActive &&
isSelectOrMultiSelectFieldMetadata(updatedFieldMetadata) &&
@ -272,10 +270,10 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`update-${updatedFieldMetadata.name}`),
existingFieldMetadata.workspaceId,
fieldMetadataInput.workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
name: computeObjectTargetTable(objectMetadataItemWithFieldMaps),
action: WorkspaceMigrationTableActionType.ALTER,
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.ALTER,
@ -299,6 +297,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
throw error;
} finally {
await queryRunner.release();
await this.workspaceMetadataVersionService.incrementMetadataVersion(
fieldMetadataInput.workspaceId,
);
@ -470,28 +469,6 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
}
}
public async findOneOrFail(
id: string,
options?: FindOneOptions<FieldMetadataEntity>,
) {
const [fieldMetadata] = await this.fieldMetadataRepository.find({
...options,
where: {
...options?.where,
id,
},
});
if (!fieldMetadata) {
throw new FieldMetadataException(
'Field does not exist',
FieldMetadataExceptionCode.FIELD_METADATA_NOT_FOUND,
);
}
return fieldMetadata;
}
public async findOneWithinWorkspace(
workspaceId: string,
options: FindOneOptions<FieldMetadataEntity>,
@ -509,7 +486,10 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
private buildUpdatableStandardFieldInput(
fieldMetadataInput: UpdateFieldInput,
existingFieldMetadata: FieldMetadataEntity,
existingFieldMetadata: Pick<
FieldMetadataInterface,
'type' | 'isNullable' | 'defaultValue' | 'options'
>,
) {
const updatableStandardFieldInput: UpdateFieldInput & {
standardOverrides?: FieldStandardOverridesDTO;
@ -754,7 +734,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
private async validateAndCreateFieldMetadataItems(
fieldMetadataInput: CreateFieldInput,
objectMetadata: ObjectMetadataEntity,
objectMetadata: ObjectMetadataItemWithFieldMaps,
fieldMetadataRepository: Repository<FieldMetadataEntity>,
): Promise<FieldMetadataEntity[]> {
if (!fieldMetadataInput.isRemoteCreation) {
@ -841,7 +821,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
isRemoteCreation,
}: {
createdFieldMetadataItems: FieldMetadataEntity[];
objectMetadataMap: Record<string, ObjectMetadataEntity>;
objectMetadataMap: Record<string, ObjectMetadataItemWithFieldMaps>;
isRemoteCreation: boolean;
}): Promise<WorkspaceMigrationTableAction[]> {
if (isRemoteCreation) {
@ -886,6 +866,11 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
return [];
}
const { objectMetadataMaps } =
await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
{ workspaceId: fieldMetadataInputs[0].workspaceId },
);
const workspaceId = fieldMetadataInputs[0].workspaceId;
const queryRunner = this.coreDataSource.createQueryRunner();
@ -902,23 +887,11 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
this.groupFieldInputsByObjectId(fieldMetadataInputs);
const objectMetadataIds = Object.keys(inputsByObjectId);
const objectMetadatas = await this.objectMetadataRepository.find({
where: {
workspaceId,
},
relations: ['fields'],
});
const objectMetadataMap = objectMetadatas.reduce(
(acc, obj) => ({ ...acc, [obj.id]: obj }),
{} as Record<string, ObjectMetadataEntity>,
);
const createdFieldMetadatas: FieldMetadataEntity[] = [];
const migrationActions: WorkspaceMigrationTableAction[] = [];
for (const objectMetadataId of objectMetadataIds) {
const objectMetadata = objectMetadataMap[objectMetadataId];
const objectMetadata = objectMetadataMaps.byId[objectMetadataId];
if (!isDefined(objectMetadata)) {
throw new FieldMetadataException(
@ -941,7 +914,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
const fieldMigrationActions = await this.createMigrationActions({
createdFieldMetadataItems,
objectMetadataMap,
objectMetadataMap: objectMetadataMaps.byId,
isRemoteCreation: fieldMetadataInput.isRemoteCreation ?? false,
});

5
packages/twenty-server/src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface.ts
View File

@ -19,7 +19,7 @@ export interface FieldMetadataInterface<
workspaceId?: string;
description?: string;
icon?: string;
isNullable?: boolean;
isNullable: boolean;
isUnique?: boolean;
relationTargetFieldMetadataId?: string;
relationTargetFieldMetadata?: FieldMetadataInterface;
@ -30,4 +30,7 @@ export interface FieldMetadataInterface<
isActive?: boolean;
generatedType?: 'STORED' | 'VIRTUAL';
asExpression?: string;
isLabelSyncedWithName: boolean;
createdAt: Date;
updatedAt: Date;
}

2
packages/twenty-server/src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface.ts
View File

@ -13,7 +13,7 @@ export interface ObjectMetadataInterface {
labelSingular: string;
labelPlural: string;
description?: string;
icon?: string;
icon: string;
targetTableName: string;
fields: FieldMetadataInterface[];
indexMetadatas: IndexMetadataInterface[];

16
packages/twenty-server/src/engine/metadata-modules/field-metadata/relation/field-metadata-relation.service.ts
View File

@ -8,7 +8,7 @@ import {
FieldMetadataExceptionCode,
} from 'src/engine/metadata-modules/field-metadata/field-metadata.exception';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { removeFieldMapsFromObjectMetadata } from 'src/engine/metadata-modules/utils/remove-field-maps-from-object-metadata.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
@ -77,13 +77,15 @@ export class FieldMetadataRelationService {
}
return {
sourceObjectMetadata: removeFieldMapsFromObjectMetadata(
sourceObjectMetadata,
) as ObjectMetadataEntity,
sourceObjectMetadata:
getObjectMetadataFromObjectMetadataItemWithFieldMaps(
sourceObjectMetadata,
) as ObjectMetadataEntity,
sourceFieldMetadata: sourceFieldMetadata as FieldMetadataEntity,
targetObjectMetadata: removeFieldMapsFromObjectMetadata(
targetObjectMetadata,
) as ObjectMetadataEntity,
targetObjectMetadata:
getObjectMetadataFromObjectMetadataItemWithFieldMaps(
targetObjectMetadata,
) as ObjectMetadataEntity,
targetFieldMetadata: targetFieldMetadata as FieldMetadataEntity,
};
});

7
packages/twenty-server/src/engine/metadata-modules/field-metadata/services/field-metadata-enum-validation.service.ts
View File

@ -6,6 +6,7 @@ import { assertUnreachable, isDefined } from 'twenty-shared/utils';
import { z } from 'zod';
import { FieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-options.interface';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
import {
@ -13,7 +14,6 @@ import {
FieldMetadataDefaultOption,
} from 'src/engine/metadata-modules/field-metadata/dtos/options.input';
import { UpdateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/update-field.input';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import {
FieldMetadataException,
FieldMetadataExceptionCode,
@ -31,7 +31,10 @@ type Validator<T> = { validator: (str: T) => boolean; message: string };
type FieldMetadataUpdateCreateInput = CreateFieldInput | UpdateFieldInput;
type ValidateEnumFieldMetadataArgs = {
existingFieldMetadata?: FieldMetadataEntity;
existingFieldMetadata?: Pick<
FieldMetadataInterface,
'type' | 'isNullable' | 'defaultValue' | 'options'
>;
fieldMetadataInput: FieldMetadataUpdateCreateInput;
fieldMetadataType: EnumFieldMetadataUnionType;
};

8
packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/is-select-or-multi-select-field-metadata.util.ts
View File

@ -1,17 +1,15 @@
import { FieldMetadataType } from 'twenty-shared/types';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
export type SelectOrMultiSelectFieldMetadataEntity = FieldMetadataEntity<
FieldMetadataType.SELECT | FieldMetadataType.MULTI_SELECT
>;
export const isSelectOrMultiSelectFieldMetadata = (
fieldMetadata: unknown,
fieldMetadata: FieldMetadataInterface,
): fieldMetadata is SelectOrMultiSelectFieldMetadataEntity => {
if (!(fieldMetadata instanceof FieldMetadataEntity)) {
return false;
}
return [FieldMetadataType.SELECT, FieldMetadataType.MULTI_SELECT].includes(
fieldMetadata.type,
);