Improve performance on metadata computation (#12785)
In this PR: ## Improve recompute metadata cache performance. We are aiming for ~100ms Deleting relationMetadata table and FKs pointing on it Fetching indexMetadata and indexFieldMetadata in a separate query as typeorm is suboptimizing ## Remove caching lock As recomputing the metadata cache is lighter, we try to stop preventing multiple concurrent computations. This also simplifies interfaces ## Introduce self recovery mecanisms to recompute cache automatically if corrupted Aka getFreshObjectMetadataMaps ## custom object resolver performance improvement: 1sec to 200ms Double check queries and indexes used while creating a custom object Remove the queries to db to use the cached objectMetadataMap ## reduce objectMetadataMaps to 500kb <img width="222" alt="image" src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062" /> We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName). While this is great for devXP, this is not great for performances. Using the same mecanisme as for objectMetadataMap: we only keep byIdMap and introduce two otherMaps to idByName, idByJoinColumnName to make the bridge ## Add dataloader on IndexMetadata (aka indexMetadataList in the API) ## Improve field resolver performances too ## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
@ -4,11 +4,9 @@ import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { i18n } from '@lingui/core';
|
||||
import { Query, QueryOptions } from '@ptc-org/nestjs-query-core';
|
||||
import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
|
||||
import { APP_LOCALES, SOURCE_LOCALE } from 'twenty-shared/translations';
|
||||
import { APP_LOCALES } from 'twenty-shared/translations';
|
||||
import { capitalize, isDefined } from 'twenty-shared/utils';
|
||||
import { FindManyOptions, FindOneOptions, In, Not, Repository } from 'typeorm';
|
||||
|
||||
import { ObjectMetadataStandardIdToIdMap } from 'src/engine/metadata-modules/object-metadata/interfaces/object-metadata-standard-id-to-id-map';
|
||||
import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm';
|
||||
|
||||
import { generateMessageId } from 'src/engine/core-modules/i18n/utils/generateMessageId';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
@ -35,7 +33,10 @@ import {
|
||||
} from 'src/engine/metadata-modules/object-metadata/utils/validate-object-metadata-input.util';
|
||||
import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service';
|
||||
import { SearchVectorService } from 'src/engine/metadata-modules/search-vector/search-vector.service';
|
||||
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
|
||||
import { validateNameAndLabelAreSyncOrThrow } from 'src/engine/metadata-modules/utils/validate-name-and-label-are-sync-or-throw.util';
|
||||
import { validatesNoOtherObjectWithSameNameExistsOrThrows } from 'src/engine/metadata-modules/utils/validate-no-other-object-with-same-name-exists-or-throw.util';
|
||||
import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service';
|
||||
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
|
||||
import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service';
|
||||
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
|
||||
@ -58,6 +59,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
|
||||
private readonly remoteTableRelationsService: RemoteTableRelationsService,
|
||||
private readonly dataSourceService: DataSourceService,
|
||||
private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService,
|
||||
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
|
||||
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
|
||||
private readonly searchVectorService: SearchVectorService,
|
||||
@ -89,6 +91,13 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
override async createOne(
|
||||
objectMetadataInput: CreateObjectInput,
|
||||
): Promise<ObjectMetadataEntity> {
|
||||
const { objectMetadataMaps } =
|
||||
await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
|
||||
{
|
||||
workspaceId: objectMetadataInput.workspaceId,
|
||||
},
|
||||
);
|
||||
|
||||
const lastDataSourceMetadata =
|
||||
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
|
||||
objectMetadataInput.workspaceId,
|
||||
@ -131,13 +140,28 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
);
|
||||
}
|
||||
|
||||
await this.validatesNoOtherObjectWithSameNameExistsOrThrows({
|
||||
validatesNoOtherObjectWithSameNameExistsOrThrows({
|
||||
objectMetadataNamePlural: objectMetadataInput.namePlural,
|
||||
objectMetadataNameSingular: objectMetadataInput.nameSingular,
|
||||
workspaceId: objectMetadataInput.workspaceId,
|
||||
objectMetadataMaps,
|
||||
});
|
||||
|
||||
const createdObjectMetadata = await super.createOne({
|
||||
const baseCustomFields = buildDefaultFieldsForCustomObject(
|
||||
objectMetadataInput.workspaceId,
|
||||
);
|
||||
|
||||
const labelIdentifierFieldMetadataId = baseCustomFields.find(
|
||||
(field) => field.standardId === CUSTOM_OBJECT_STANDARD_FIELD_IDS.name,
|
||||
)?.id;
|
||||
|
||||
if (!labelIdentifierFieldMetadataId) {
|
||||
throw new ObjectMetadataException(
|
||||
'Label identifier field metadata not created properly',
|
||||
ObjectMetadataExceptionCode.MISSING_CUSTOM_OBJECT_DEFAULT_LABEL_IDENTIFIER_FIELD,
|
||||
);
|
||||
}
|
||||
|
||||
const createdObjectMetadata = await this.objectMetadataRepository.save({
|
||||
...objectMetadataInput,
|
||||
dataSourceId: lastDataSourceMetadata.id,
|
||||
targetTableName: 'DEPRECATED',
|
||||
@ -146,24 +170,8 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
isSystem: false,
|
||||
isRemote: objectMetadataInput.isRemote,
|
||||
isSearchable: !objectMetadataInput.isRemote,
|
||||
fields: objectMetadataInput.isRemote
|
||||
? []
|
||||
: buildDefaultFieldsForCustomObject(objectMetadataInput.workspaceId),
|
||||
});
|
||||
|
||||
const labelIdentifierFieldMetadata = createdObjectMetadata.fields.find(
|
||||
(field) => field.standardId === CUSTOM_OBJECT_STANDARD_FIELD_IDS.name,
|
||||
);
|
||||
|
||||
if (!labelIdentifierFieldMetadata) {
|
||||
throw new ObjectMetadataException(
|
||||
'Label identifier field metadata not created properly',
|
||||
ObjectMetadataExceptionCode.MISSING_CUSTOM_OBJECT_DEFAULT_LABEL_IDENTIFIER_FIELD,
|
||||
);
|
||||
}
|
||||
|
||||
await this.objectMetadataRepository.update(createdObjectMetadata.id, {
|
||||
labelIdentifierFieldMetadataId: labelIdentifierFieldMetadata.id,
|
||||
fields: objectMetadataInput.isRemote ? [] : baseCustomFields,
|
||||
labelIdentifierFieldMetadataId,
|
||||
});
|
||||
|
||||
if (objectMetadataInput.isRemote) {
|
||||
@ -174,6 +182,13 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
objectMetadataInput.primaryKeyColumnType,
|
||||
);
|
||||
} else {
|
||||
const createdRelatedObjectMetadataCollection =
|
||||
await this.objectMetadataFieldRelationService.createRelationsAndForeignKeysMetadata(
|
||||
objectMetadataInput.workspaceId,
|
||||
createdObjectMetadata,
|
||||
objectMetadataMaps,
|
||||
);
|
||||
|
||||
await this.objectMetadataMigrationService.createTableMigration(
|
||||
createdObjectMetadata,
|
||||
);
|
||||
@ -183,12 +198,6 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
createdObjectMetadata.fields,
|
||||
);
|
||||
|
||||
const createdRelatedObjectMetadataCollection =
|
||||
await this.objectMetadataFieldRelationService.createRelationsAndForeignKeysMetadata(
|
||||
objectMetadataInput.workspaceId,
|
||||
createdObjectMetadata,
|
||||
);
|
||||
|
||||
await this.objectMetadataMigrationService.createRelationMigrations(
|
||||
createdObjectMetadata,
|
||||
createdRelatedObjectMetadataCollection,
|
||||
@ -214,7 +223,6 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
|
||||
await this.workspacePermissionsCacheService.recomputeRolesPermissionsCache({
|
||||
workspaceId: objectMetadataInput.workspaceId,
|
||||
ignoreLock: true,
|
||||
});
|
||||
|
||||
return createdObjectMetadata;
|
||||
@ -224,6 +232,11 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
input: UpdateOneObjectInput,
|
||||
workspaceId: string,
|
||||
): Promise<ObjectMetadataEntity> {
|
||||
const { objectMetadataMaps } =
|
||||
await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps(
|
||||
{ workspaceId },
|
||||
);
|
||||
|
||||
const inputId = input.id;
|
||||
|
||||
const inputPayload = {
|
||||
@ -238,9 +251,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
|
||||
validateObjectMetadataInputNamesOrThrow(inputPayload);
|
||||
|
||||
const existingObjectMetadata = await this.objectMetadataRepository.findOne({
|
||||
where: { id: inputId, workspaceId: workspaceId },
|
||||
});
|
||||
const existingObjectMetadata = objectMetadataMaps.byId[inputId];
|
||||
|
||||
if (!existingObjectMetadata) {
|
||||
throw new ObjectMetadataException(
|
||||
@ -254,14 +265,14 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
...inputPayload,
|
||||
};
|
||||
|
||||
await this.validatesNoOtherObjectWithSameNameExistsOrThrows({
|
||||
validatesNoOtherObjectWithSameNameExistsOrThrows({
|
||||
objectMetadataNameSingular:
|
||||
existingObjectMetadataCombinedWithUpdateInput.nameSingular,
|
||||
objectMetadataNamePlural:
|
||||
existingObjectMetadataCombinedWithUpdateInput.namePlural,
|
||||
workspaceId: workspaceId,
|
||||
existingObjectMetadataId:
|
||||
existingObjectMetadataCombinedWithUpdateInput.id,
|
||||
objectMetadataMaps,
|
||||
});
|
||||
|
||||
if (existingObjectMetadataCombinedWithUpdateInput.isLabelSyncedWithName) {
|
||||
@ -395,7 +406,6 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
|
||||
await this.workspacePermissionsCacheService.recomputeRolesPermissionsCache({
|
||||
workspaceId,
|
||||
ignoreLock: true,
|
||||
});
|
||||
|
||||
return objectMetadata;
|
||||
@ -436,44 +446,26 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
});
|
||||
}
|
||||
|
||||
public async findMany(options?: FindManyOptions<ObjectMetadataEntity>) {
|
||||
return this.objectMetadataRepository.find({
|
||||
relations: ['fields'],
|
||||
...options,
|
||||
where: {
|
||||
...options?.where,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
public async deleteObjectsMetadata(workspaceId: string) {
|
||||
await this.objectMetadataRepository.delete({ workspaceId });
|
||||
}
|
||||
|
||||
public async getObjectMetadataStandardIdToIdMap(workspaceId: string) {
|
||||
const objectMetadata = await this.findManyWithinWorkspace(workspaceId);
|
||||
|
||||
const objectMetadataStandardIdToIdMap =
|
||||
objectMetadata.reduce<ObjectMetadataStandardIdToIdMap>((acc, object) => {
|
||||
acc[object.standardId ?? ''] = {
|
||||
id: object.id,
|
||||
fields: object.fields.reduce((acc, field) => {
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
acc[field.standardId ?? ''] = field.id;
|
||||
|
||||
return acc;
|
||||
}, {}),
|
||||
};
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
return { objectMetadataStandardIdToIdMap };
|
||||
}
|
||||
|
||||
private async handleObjectNameAndLabelUpdates(
|
||||
existingObjectMetadata: ObjectMetadataEntity,
|
||||
objectMetadataForUpdate: ObjectMetadataEntity,
|
||||
existingObjectMetadata: Pick<
|
||||
ObjectMetadataItemWithFieldMaps,
|
||||
'nameSingular' | 'isCustom' | 'id' | 'labelPlural' | 'icon' | 'fieldsById'
|
||||
>,
|
||||
objectMetadataForUpdate: Pick<
|
||||
ObjectMetadataItemWithFieldMaps,
|
||||
| 'nameSingular'
|
||||
| 'isCustom'
|
||||
| 'workspaceId'
|
||||
| 'id'
|
||||
| 'labelSingular'
|
||||
| 'labelPlural'
|
||||
| 'icon'
|
||||
| 'fieldsById'
|
||||
>,
|
||||
inputPayload: UpdateObjectPayload,
|
||||
) {
|
||||
const newTargetTableName = computeObjectTargetTable(
|
||||
@ -533,45 +525,6 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
}
|
||||
}
|
||||
|
||||
private validatesNoOtherObjectWithSameNameExistsOrThrows = async ({
|
||||
objectMetadataNameSingular,
|
||||
objectMetadataNamePlural,
|
||||
workspaceId,
|
||||
existingObjectMetadataId,
|
||||
}: {
|
||||
objectMetadataNameSingular: string;
|
||||
objectMetadataNamePlural: string;
|
||||
workspaceId: string;
|
||||
existingObjectMetadataId?: string;
|
||||
}): Promise<void> => {
|
||||
const baseWhereConditions = [
|
||||
{ nameSingular: objectMetadataNameSingular, workspaceId },
|
||||
{ nameSingular: objectMetadataNamePlural, workspaceId },
|
||||
{ namePlural: objectMetadataNameSingular, workspaceId },
|
||||
{ namePlural: objectMetadataNamePlural, workspaceId },
|
||||
];
|
||||
|
||||
const whereConditions = baseWhereConditions.map((condition) => {
|
||||
return {
|
||||
...condition,
|
||||
...(isDefined(existingObjectMetadataId)
|
||||
? { id: Not(In([existingObjectMetadataId])) }
|
||||
: {}),
|
||||
};
|
||||
});
|
||||
|
||||
const objectAlreadyExists = await this.objectMetadataRepository.findOne({
|
||||
where: whereConditions,
|
||||
});
|
||||
|
||||
if (objectAlreadyExists) {
|
||||
throw new ObjectMetadataException(
|
||||
'Object already exists',
|
||||
ObjectMetadataExceptionCode.OBJECT_ALREADY_EXISTS,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
async resolveOverridableString(
|
||||
objectMetadata: ObjectMetadataDTO,
|
||||
labelKey: 'labelPlural' | 'labelSingular' | 'description' | 'icon',
|
||||
@ -581,17 +534,6 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
|
||||
return objectMetadata[labelKey];
|
||||
}
|
||||
|
||||
if (!locale || locale === SOURCE_LOCALE) {
|
||||
if (
|
||||
objectMetadata.standardOverrides &&
|
||||
isDefined(objectMetadata.standardOverrides[labelKey])
|
||||
) {
|
||||
return objectMetadata.standardOverrides[labelKey] as string;
|
||||
}
|
||||
|
||||
return objectMetadata[labelKey];
|
||||
}
|
||||
|
||||
const translationValue =
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
objectMetadata.standardOverrides?.translations?.[locale]?.[labelKey];
|
||||
|
||||
Reference in New Issue
Block a user