From 40f43a4076559ba8ce9f5cf2bf45811a8e7e869b Mon Sep 17 00:00:00 2001 From: Weiko Date: Tue, 4 Feb 2025 11:18:57 +0100 Subject: [PATCH] add createMany fields to fieldMetadataService to batch field creation (#9957) ## Context Not exposed in the API yet, this new method allows us to reduce the time to create multiple fields at once, mostly during seeding. This allows us to batch transactions and avoid recomputing the cache everytime. With this change, we recompute the cache 7 times instead of 35 during seeding. We could do the same for objects. --- .../data-seed-dev-workspace.command.ts | 17 +- .../field-metadata/field-metadata.service.ts | 472 +++++++++++------- .../src/engine/seeder/seeder.service.ts | 8 +- 3 files changed, 297 insertions(+), 200 deletions(-) diff --git a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts index 34cc58d12..dc435894a 100644 --- a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts +++ b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts @@ -265,13 +265,12 @@ export class DataSeedWorkspaceCommand extends CommandRunner { workspaceId, ); - for (const customField of DEV_SEED_COMPANY_CUSTOM_FIELDS) { - // TODO: Use createMany once implemented for better performances - await this.fieldMetadataService.createOne({ + await this.fieldMetadataService.createMany( + DEV_SEED_COMPANY_CUSTOM_FIELDS.map((customField) => ({ ...customField, isCustom: true, - }); - } + })), + ); } async seedPeopleCustomFields( @@ -289,11 +288,11 @@ export class DataSeedWorkspaceCommand extends CommandRunner { workspaceId, ); - for (const customField of DEV_SEED_PERSON_CUSTOM_FIELDS) { - await this.fieldMetadataService.createOne({ + await this.fieldMetadataService.createMany( + DEV_SEED_PERSON_CUSTOM_FIELDS.map((customField) => ({ ...customField, isCustom: true, - }); - } + })), + ); } } diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index b15c0492c..c1d180947 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -5,7 +5,7 @@ import { i18n } from '@lingui/core'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; import isEmpty from 'lodash.isempty'; import { APP_LOCALES, FieldMetadataType, isDefined } from 'twenty-shared'; -import { DataSource, FindOneOptions, Repository } from 'typeorm'; +import { DataSource, FindOneOptions, In, Repository } from 'typeorm'; import { v4 as uuidV4, v4 } from 'uuid'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; @@ -95,195 +95,16 @@ export class FieldMetadataService extends TypeOrmQueryService { - const queryRunner = this.metadataDataSource.createQueryRunner(); + const [createdFieldMetadata] = await this.createMany([fieldMetadataInput]); - await queryRunner.connect(); - await queryRunner.startTransaction(); - - try { - const fieldMetadataRepository = - queryRunner.manager.getRepository( - FieldMetadataEntity, - ); - - const [objectMetadata] = await this.objectMetadataRepository.find({ - where: { - id: fieldMetadataInput.objectMetadataId, - workspaceId: fieldMetadataInput.workspaceId, - }, - relations: ['fields'], - order: {}, - }); - - if (!objectMetadata) { - throw new FieldMetadataException( - 'Object metadata does not exist', - FieldMetadataExceptionCode.OBJECT_METADATA_NOT_FOUND, - ); - } - - if (!fieldMetadataInput.isRemoteCreation) { - assertMutationNotOnRemoteObject(objectMetadata); - } - - // Double check in case the service is directly called - if (isEnumFieldMetadataType(fieldMetadataInput.type)) { - if ( - !fieldMetadataInput.options && - fieldMetadataInput.type !== FieldMetadataType.RATING - ) { - throw new FieldMetadataException( - 'Options are required for enum fields', - FieldMetadataExceptionCode.INVALID_FIELD_INPUT, - ); - } - } - - // Generate options for rating fields - if (fieldMetadataInput.type === FieldMetadataType.RATING) { - fieldMetadataInput.options = generateRatingOptions(); - } - - const fieldMetadataForCreate = { - id: v4(), - createdAt: new Date(), - updatedAt: new Date(), - ...fieldMetadataInput, - isNullable: generateNullable( - fieldMetadataInput.type, - fieldMetadataInput.isNullable, - fieldMetadataInput.isRemoteCreation, - ), - defaultValue: - fieldMetadataInput.defaultValue ?? - generateDefaultValue(fieldMetadataInput.type), - options: fieldMetadataInput.options - ? fieldMetadataInput.options.map((option) => ({ - ...option, - id: uuidV4(), - })) - : undefined, - isActive: true, - isCustom: true, - }; - - this.validateFieldMetadata( - fieldMetadataForCreate.type, - fieldMetadataForCreate, - objectMetadata, - ); - - if (fieldMetadataForCreate.isLabelSyncedWithName === true) { - validateNameAndLabelAreSyncOrThrow( - fieldMetadataForCreate.label, - fieldMetadataForCreate.name, - ); - } - - const createdFieldMetadata = await fieldMetadataRepository.save( - fieldMetadataForCreate, - ); - - if (!fieldMetadataInput.isRemoteCreation) { - await this.workspaceMigrationService.createCustomMigration( - generateMigrationName(`create-${createdFieldMetadata.name}`), - fieldMetadataInput.workspaceId, - [ - { - name: computeObjectTargetTable(objectMetadata), - action: WorkspaceMigrationTableActionType.ALTER, - columns: this.workspaceMigrationFactory.createColumnActions( - WorkspaceMigrationColumnActionType.CREATE, - createdFieldMetadata, - ), - } satisfies WorkspaceMigrationTableAction, - ], - ); - - await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( - fieldMetadataInput.workspaceId, - ); - } - - // TODO: Move viewField creation to a cdc scheduler - const dataSourceMetadata = - await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( - fieldMetadataInput.workspaceId, - ); - - const workspaceDataSource = - await this.typeORMService.connectToDataSource(dataSourceMetadata); - - const workspaceQueryRunner = workspaceDataSource?.createQueryRunner(); - - if (!workspaceQueryRunner) { - throw new FieldMetadataException( - 'Could not create workspace query runner', - FieldMetadataExceptionCode.INTERNAL_SERVER_ERROR, - ); - } - - await workspaceQueryRunner.connect(); - await workspaceQueryRunner.startTransaction(); - - try { - // TODO: use typeorm repository - const view = await workspaceQueryRunner?.query( - `SELECT id FROM ${dataSourceMetadata.schema}."view" - WHERE "objectMetadataId" = '${createdFieldMetadata.objectMetadataId}'`, - ); - - if (!isEmpty(view)) { - const existingViewFields = (await workspaceQueryRunner?.query( - `SELECT * FROM ${dataSourceMetadata.schema}."viewField" - WHERE "viewId" = '${view[0].id}'`, - )) as ViewFieldWorkspaceEntity[]; - - const createdFieldIsAlreadyInView = existingViewFields.some( - (existingViewField) => - existingViewField.fieldMetadataId === createdFieldMetadata.id, - ); - - if (!createdFieldIsAlreadyInView) { - const lastPosition = existingViewFields - .map((viewField) => viewField.position) - .reduce((acc, position) => { - if (position > acc) { - return position; - } - - return acc; - }, -1); - - await workspaceQueryRunner?.query( - `INSERT INTO ${dataSourceMetadata.schema}."viewField" - ("fieldMetadataId", "position", "isVisible", "size", "viewId") - VALUES ('${createdFieldMetadata.id}', '${lastPosition + 1}', true, 180, '${ - view[0].id - }')`, - ); - } - } - await workspaceQueryRunner.commitTransaction(); - } catch (error) { - await workspaceQueryRunner.rollbackTransaction(); - throw error; - } finally { - await workspaceQueryRunner.release(); - } - - await queryRunner.commitTransaction(); - - return createdFieldMetadata; - } catch (error) { - await queryRunner.rollbackTransaction(); - throw error; - } finally { - await queryRunner.release(); - await this.workspaceMetadataVersionService.incrementMetadataVersion( - fieldMetadataInput.workspaceId, + if (!createdFieldMetadata) { + throw new FieldMetadataException( + 'Failed to create field metadata', + FieldMetadataExceptionCode.INTERNAL_SERVER_ERROR, ); } + + return createdFieldMetadata; } override async updateOne( @@ -799,4 +620,281 @@ export class FieldMetadataService extends TypeOrmQueryService ({ + ...option, + id: uuidV4(), + })) + : undefined, + isActive: true, + isCustom: true, + }; + } + + private groupFieldInputsByObjectId( + fieldMetadataInputs: CreateFieldInput[], + ): Record { + return fieldMetadataInputs.reduce( + (acc, input) => { + if (!acc[input.objectMetadataId]) { + acc[input.objectMetadataId] = []; + } + acc[input.objectMetadataId].push(input); + + return acc; + }, + {} as Record, + ); + } + + private async validateAndCreateFieldMetadata( + fieldMetadataInput: CreateFieldInput, + objectMetadata: ObjectMetadataEntity, + fieldMetadataRepository: Repository, + ): Promise { + if (!fieldMetadataInput.isRemoteCreation) { + assertMutationNotOnRemoteObject(objectMetadata); + } + + if (isEnumFieldMetadataType(fieldMetadataInput.type)) { + if ( + !fieldMetadataInput.options && + fieldMetadataInput.type !== FieldMetadataType.RATING + ) { + throw new FieldMetadataException( + 'Options are required for enum fields', + FieldMetadataExceptionCode.INVALID_FIELD_INPUT, + ); + } + } + + if (fieldMetadataInput.type === FieldMetadataType.RATING) { + fieldMetadataInput.options = generateRatingOptions(); + } + + const fieldMetadataForCreate = + this.prepareCustomFieldMetadata(fieldMetadataInput); + + this.validateFieldMetadata( + fieldMetadataForCreate.type, + fieldMetadataForCreate, + objectMetadata, + ); + + if (fieldMetadataForCreate.isLabelSyncedWithName === true) { + validateNameAndLabelAreSyncOrThrow( + fieldMetadataForCreate.label, + fieldMetadataForCreate.name, + ); + } + + return await fieldMetadataRepository.save(fieldMetadataForCreate); + } + + private async createMigrationActions( + createdFieldMetadata: FieldMetadataEntity, + objectMetadata: ObjectMetadataEntity, + isRemoteCreation: boolean, + ): Promise { + if (isRemoteCreation) { + return null; + } + + return { + name: computeObjectTargetTable(objectMetadata), + action: WorkspaceMigrationTableActionType.ALTER, + columns: this.workspaceMigrationFactory.createColumnActions( + WorkspaceMigrationColumnActionType.CREATE, + createdFieldMetadata, + ), + }; + } + + async createMany( + fieldMetadataInputs: CreateFieldInput[], + ): Promise { + if (!fieldMetadataInputs.length) { + return []; + } + + const workspaceId = fieldMetadataInputs[0].workspaceId; + const queryRunner = this.metadataDataSource.createQueryRunner(); + + await queryRunner.connect(); + await queryRunner.startTransaction(); + + try { + const fieldMetadataRepository = + queryRunner.manager.getRepository( + FieldMetadataEntity, + ); + + const inputsByObjectId = + this.groupFieldInputsByObjectId(fieldMetadataInputs); + const objectMetadataIds = Object.keys(inputsByObjectId); + + const objectMetadatas = await this.objectMetadataRepository.find({ + where: { + id: In(objectMetadataIds), + workspaceId, + }, + relations: ['fields'], + }); + + const objectMetadataMap = objectMetadatas.reduce( + (acc, obj) => ({ ...acc, [obj.id]: obj }), + {} as Record, + ); + + const createdFieldMetadatas: FieldMetadataEntity[] = []; + const migrationActions: WorkspaceMigrationTableAction[] = []; + + for (const objectMetadataId of objectMetadataIds) { + const objectMetadata = objectMetadataMap[objectMetadataId]; + + if (!objectMetadata) { + throw new FieldMetadataException( + 'Object metadata does not exist', + FieldMetadataExceptionCode.OBJECT_METADATA_NOT_FOUND, + ); + } + + const inputs = inputsByObjectId[objectMetadataId]; + + for (const fieldMetadataInput of inputs) { + const createdFieldMetadata = + await this.validateAndCreateFieldMetadata( + fieldMetadataInput, + objectMetadata, + fieldMetadataRepository, + ); + + createdFieldMetadatas.push(createdFieldMetadata); + + const migrationAction = await this.createMigrationActions( + createdFieldMetadata, + objectMetadata, + fieldMetadataInput.isRemoteCreation ?? false, + ); + + if (migrationAction) { + migrationActions.push(migrationAction); + } + } + } + + if (migrationActions.length > 0) { + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`create-multiple-fields`), + workspaceId, + migrationActions, + ); + + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + } + + await this.createViewAndViewFields(createdFieldMetadatas, workspaceId); + + await queryRunner.commitTransaction(); + + return createdFieldMetadatas; + } catch (error) { + await queryRunner.rollbackTransaction(); + throw error; + } finally { + await queryRunner.release(); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); + } + } + + private async createViewAndViewFields( + createdFieldMetadatas: FieldMetadataEntity[], + workspaceId: string, + ) { + const dataSourceMetadata = + await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail( + workspaceId, + ); + + const workspaceDataSource = + await this.typeORMService.connectToDataSource(dataSourceMetadata); + + const workspaceQueryRunner = workspaceDataSource?.createQueryRunner(); + + if (!workspaceQueryRunner) { + throw new FieldMetadataException( + 'Could not create workspace query runner', + FieldMetadataExceptionCode.INTERNAL_SERVER_ERROR, + ); + } + + await workspaceQueryRunner.connect(); + await workspaceQueryRunner.startTransaction(); + + try { + for (const createdFieldMetadata of createdFieldMetadatas) { + const view = await workspaceQueryRunner?.query( + `SELECT id FROM ${dataSourceMetadata.schema}."view" + WHERE "objectMetadataId" = '${createdFieldMetadata.objectMetadataId}'`, + ); + + if (!isEmpty(view)) { + const existingViewFields = (await workspaceQueryRunner?.query( + `SELECT * FROM ${dataSourceMetadata.schema}."viewField" + WHERE "viewId" = '${view[0].id}'`, + )) as ViewFieldWorkspaceEntity[]; + + const createdFieldIsAlreadyInView = existingViewFields.some( + (existingViewField) => + existingViewField.fieldMetadataId === createdFieldMetadata.id, + ); + + if (!createdFieldIsAlreadyInView) { + const lastPosition = existingViewFields + .map((viewField) => viewField.position) + .reduce((acc, position) => { + if (position > acc) { + return position; + } + + return acc; + }, -1); + + await workspaceQueryRunner?.query( + `INSERT INTO ${dataSourceMetadata.schema}."viewField" + ("fieldMetadataId", "position", "isVisible", "size", "viewId") + VALUES ('${createdFieldMetadata.id}', '${ + lastPosition + 1 + }', true, 180, '${view[0].id}')`, + ); + } + } + } + await workspaceQueryRunner.commitTransaction(); + } catch (error) { + await workspaceQueryRunner.rollbackTransaction(); + throw error; + } finally { + await workspaceQueryRunner.release(); + } + } } diff --git a/packages/twenty-server/src/engine/seeder/seeder.service.ts b/packages/twenty-server/src/engine/seeder/seeder.service.ts index 75cc0aae1..8a7e43fa0 100644 --- a/packages/twenty-server/src/engine/seeder/seeder.service.ts +++ b/packages/twenty-server/src/engine/seeder/seeder.service.ts @@ -37,13 +37,13 @@ export class SeederService { throw new Error("Object metadata couldn't be created"); } - for (const fieldMetadataSeed of objectMetadataSeed.fields) { - await this.fieldMetadataService.createOne({ + await this.fieldMetadataService.createMany( + objectMetadataSeed.fields.map((fieldMetadataSeed) => ({ ...fieldMetadataSeed, objectMetadataId: createdObjectMetadata.id, workspaceId, - }); - } + })), + ); const objectMetadataAfterFieldCreation = await this.objectMetadataService.findOneWithinWorkspace(workspaceId, {