Files
twenty_crm/packages/twenty-server/src/engine/metadata-modules/object-metadata/services/object-metadata-migration.service.ts
Charles Bochet d5c974054d Improve performance on metadata computation (#12785)
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
2025-06-23 21:06:17 +02:00

337 lines
12 KiB
TypeScript

import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { FieldMetadataType } from 'twenty-shared/types';
import { Repository } from 'typeorm';
import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { buildMigrationsForCustomObjectRelations } from 'src/engine/metadata-modules/object-metadata/utils/build-migrations-for-custom-object-relations.util';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,
WorkspaceMigrationColumnDrop,
WorkspaceMigrationTableAction,
WorkspaceMigrationTableActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationFactory } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.factory';
import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service';
import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import { isFieldMetadataInterfaceOfType } from 'src/engine/utils/is-field-metadata-of-type.util';
import { RELATION_MIGRATION_PRIORITY_PREFIX } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
@Injectable()
export class ObjectMetadataMigrationService {
constructor(
@InjectRepository(FieldMetadataEntity, 'core')
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationFactory: WorkspaceMigrationFactory,
) {}
public async createTableMigration(
createdObjectMetadata: ObjectMetadataEntity,
) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdObjectMetadata.nameSingular}`),
createdObjectMetadata.workspaceId,
[
{
name: computeObjectTargetTable(createdObjectMetadata),
action: WorkspaceMigrationTableActionType.CREATE,
} satisfies WorkspaceMigrationTableAction,
],
);
}
public async createColumnsMigrations(
createdObjectMetadata: ObjectMetadataEntity,
fieldMetadataCollection: FieldMetadataEntity[],
) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(
`create-${createdObjectMetadata.nameSingular}-fields`,
),
createdObjectMetadata.workspaceId,
[
{
name: computeObjectTargetTable(createdObjectMetadata),
action: WorkspaceMigrationTableActionType.ALTER,
columns: fieldMetadataCollection.flatMap((fieldMetadata) =>
this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
fieldMetadata,
),
),
},
],
);
}
public async createRelationMigrations(
createdObjectMetadata: Pick<
ObjectMetadataItemWithFieldMaps,
'nameSingular' | 'workspaceId' | 'isCustom'
>,
relatedObjectMetadataCollection: Pick<
ObjectMetadataItemWithFieldMaps,
'nameSingular' | 'isCustom'
>[],
) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(
`create-${createdObjectMetadata.nameSingular}-relations`,
),
createdObjectMetadata.workspaceId,
buildMigrationsForCustomObjectRelations(
createdObjectMetadata,
relatedObjectMetadataCollection,
),
);
}
public async createRenameTableMigration(
existingObjectMetadata: Pick<
ObjectMetadataEntity,
'nameSingular' | 'isCustom'
>,
objectMetadataForUpdate: Pick<
ObjectMetadataEntity,
'nameSingular' | 'isCustom'
>,
workspaceId: string,
) {
const newTargetTableName = computeObjectTargetTable(
objectMetadataForUpdate,
);
const existingTargetTableName = computeObjectTargetTable(
existingObjectMetadata,
);
this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`rename-${existingObjectMetadata.nameSingular}`),
workspaceId,
[
{
name: existingTargetTableName,
newName: newTargetTableName,
action: WorkspaceMigrationTableActionType.ALTER,
},
],
);
}
public async updateRelationMigrations(
currentObjectMetadata: Pick<ObjectMetadataEntity, 'nameSingular'>,
alteredObjectMetadata: Pick<ObjectMetadataEntity, 'nameSingular'>,
relationMetadataCollection: {
targetObjectMetadata: ObjectMetadataEntity;
targetFieldMetadata: FieldMetadataEntity;
sourceFieldMetadata: FieldMetadataEntity;
}[],
workspaceId: string,
) {
for (const { targetObjectMetadata } of relationMetadataCollection) {
const targetTableName = computeObjectTargetTable(targetObjectMetadata);
const columnName = `${currentObjectMetadata.nameSingular}Id`;
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(
`rename-${currentObjectMetadata.nameSingular}-to-${alteredObjectMetadata.nameSingular}-in-${targetObjectMetadata.nameSingular}`,
),
workspaceId,
[
{
name: targetTableName,
action: WorkspaceMigrationTableActionType.ALTER,
columns: [
{
action: WorkspaceMigrationColumnActionType.ALTER,
currentColumnDefinition: {
columnName,
columnType: 'uuid',
isNullable: true,
defaultValue: null,
},
alteredColumnDefinition: {
columnName: `${alteredObjectMetadata.nameSingular}Id`,
columnType: 'uuid',
isNullable: true,
defaultValue: null,
},
},
],
},
],
);
}
}
public async createUpdateForeignKeysMigrations(
existingObjectMetadata: ObjectMetadataEntity,
updatedObjectMetadata: ObjectMetadataEntity,
relationsAndForeignKeysMetadata: {
relatedObjectMetadata: ObjectMetadataEntity;
foreignKeyFieldMetadata: FieldMetadataEntity;
}[],
workspaceId: string,
) {
for (const {
relatedObjectMetadata,
foreignKeyFieldMetadata,
} of relationsAndForeignKeysMetadata) {
const relatedObjectTableName = computeObjectTargetTable(
relatedObjectMetadata,
);
const columnName = `${existingObjectMetadata.nameSingular}Id`;
const columnType = fieldMetadataTypeToColumnType(
foreignKeyFieldMetadata.type,
);
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(
`rename-${existingObjectMetadata.nameSingular}-to-${updatedObjectMetadata.nameSingular}-in-${relatedObjectMetadata.nameSingular}`,
),
workspaceId,
[
{
name: relatedObjectTableName,
action: WorkspaceMigrationTableActionType.ALTER,
columns: [
{
action: WorkspaceMigrationColumnActionType.ALTER,
currentColumnDefinition: {
columnName,
columnType,
isNullable: true,
defaultValue: null,
},
alteredColumnDefinition: {
columnName: `${updatedObjectMetadata.nameSingular}Id`,
columnType,
isNullable: true,
defaultValue: null,
},
},
],
},
],
);
}
}
public async deleteAllRelationsAndDropTable(
objectMetadata: ObjectMetadataEntity,
workspaceId: string,
) {
const relationFields = objectMetadata.fields.filter((field) =>
isFieldMetadataInterfaceOfType(field, FieldMetadataType.RELATION),
) as FieldMetadataEntity<FieldMetadataType.RELATION>[];
const relationFieldsToDelete = [
...relationFields,
...(relationFields.map(
(relation) => relation.relationTargetFieldMetadata,
) as FieldMetadataEntity<FieldMetadataType.RELATION>[]),
];
await this.fieldMetadataRepository.delete(
relationFieldsToDelete.map((relation) => relation.id),
);
for (const relationToDelete of relationFieldsToDelete) {
if (
relationToDelete.settings?.relationType === RelationType.ONE_TO_MANY
) {
continue;
}
const joinColumnName = relationToDelete.settings?.joinColumnName;
if (!joinColumnName) {
throw new Error(
`Join column name is not set for relation field ${relationToDelete.name}`,
);
}
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(
`delete-${RELATION_MIGRATION_PRIORITY_PREFIX}-${relationToDelete.name}`,
),
workspaceId,
[
{
name: computeTableName(
relationToDelete.object.nameSingular,
relationToDelete.object.isCustom,
),
action: WorkspaceMigrationTableActionType.ALTER,
columns: [
{
action: WorkspaceMigrationColumnActionType.DROP,
columnName: joinColumnName,
} satisfies WorkspaceMigrationColumnDrop,
],
},
],
);
}
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`delete-${objectMetadata.nameSingular}`),
workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
action: WorkspaceMigrationTableActionType.DROP,
},
],
);
}
public async recomputeEnumNames(
updatedObjectMetadata: Pick<
ObjectMetadataItemWithFieldMaps,
'nameSingular' | 'isCustom' | 'id' | 'fieldsById'
>,
workspaceId: string,
) {
const enumFieldMetadataTypes = [
FieldMetadataType.SELECT,
FieldMetadataType.MULTI_SELECT,
FieldMetadataType.RATING,
FieldMetadataType.ACTOR,
];
const fieldMetadataToUpdate = Object.values(
updatedObjectMetadata.fieldsById,
).filter((field) => enumFieldMetadataTypes.includes(field.type));
for (const fieldMetadata of fieldMetadataToUpdate) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`update-${fieldMetadata.name}-enum-name`),
workspaceId,
[
{
name: computeTableName(
updatedObjectMetadata.nameSingular,
updatedObjectMetadata.isCustom,
),
action: WorkspaceMigrationTableActionType.ALTER,
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.ALTER,
fieldMetadata,
fieldMetadata,
),
},
],
);
}
}
}