admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add createMany fields to fieldMetadataService to batch field creation (#9957)

Browse Source 
## Context
Not exposed in the API yet, this new method allows us to reduce the time
to create multiple fields at once, mostly during seeding. This allows us
to batch transactions and avoid recomputing the cache everytime.

With this change, we recompute the cache 7 times instead of 35 during
seeding. We could do the same for objects.
This commit is contained in:
Weiko
2025-02-04 11:18:57 +01:00
committed by GitHub
parent 0e5c2cff42
commit 40f43a4076
3 changed files with 297 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts
View File

@ -265,13 +265,12 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
workspaceId,
);
for (const customField of DEV_SEED_COMPANY_CUSTOM_FIELDS) {
// TODO: Use createMany once implemented for better performances
await this.fieldMetadataService.createOne({
await this.fieldMetadataService.createMany(
DEV_SEED_COMPANY_CUSTOM_FIELDS.map((customField) => ({
...customField,
isCustom: true,
});
}
})),
);
}
async seedPeopleCustomFields(
@ -289,11 +288,11 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
workspaceId,
);
for (const customField of DEV_SEED_PERSON_CUSTOM_FIELDS) {
await this.fieldMetadataService.createOne({
await this.fieldMetadataService.createMany(
DEV_SEED_PERSON_CUSTOM_FIELDS.map((customField) => ({
...customField,
isCustom: true,
});
}
})),
);
}
}

472
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
View File

@ -5,7 +5,7 @@ import { i18n } from '@lingui/core';
import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
import isEmpty from 'lodash.isempty';
import { APP_LOCALES, FieldMetadataType, isDefined } from 'twenty-shared';
import { DataSource, FindOneOptions, Repository } from 'typeorm';
import { DataSource, FindOneOptions, In, Repository } from 'typeorm';
import { v4 as uuidV4, v4 } from 'uuid';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
@ -95,195 +95,16 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
override async createOne(
fieldMetadataInput: CreateFieldInput,
): Promise<FieldMetadataEntity> {
const queryRunner = this.metadataDataSource.createQueryRunner();
const [createdFieldMetadata] = await this.createMany([fieldMetadataInput]);
await queryRunner.connect();
await queryRunner.startTransaction();
try {
const fieldMetadataRepository =
queryRunner.manager.getRepository<FieldMetadataEntity>(
FieldMetadataEntity,
);
const [objectMetadata] = await this.objectMetadataRepository.find({
where: {
id: fieldMetadataInput.objectMetadataId,
workspaceId: fieldMetadataInput.workspaceId,
},
relations: ['fields'],
order: {},
});
if (!objectMetadata) {
throw new FieldMetadataException(
'Object metadata does not exist',
FieldMetadataExceptionCode.OBJECT_METADATA_NOT_FOUND,
);
}
if (!fieldMetadataInput.isRemoteCreation) {
assertMutationNotOnRemoteObject(objectMetadata);
}
// Double check in case the service is directly called
if (isEnumFieldMetadataType(fieldMetadataInput.type)) {
if (
!fieldMetadataInput.options &&
fieldMetadataInput.type !== FieldMetadataType.RATING
) {
throw new FieldMetadataException(
'Options are required for enum fields',
FieldMetadataExceptionCode.INVALID_FIELD_INPUT,
);
}
}
// Generate options for rating fields
if (fieldMetadataInput.type === FieldMetadataType.RATING) {
fieldMetadataInput.options = generateRatingOptions();
}
const fieldMetadataForCreate = {
id: v4(),
createdAt: new Date(),
updatedAt: new Date(),
...fieldMetadataInput,
isNullable: generateNullable(
fieldMetadataInput.type,
fieldMetadataInput.isNullable,
fieldMetadataInput.isRemoteCreation,
),
defaultValue:
fieldMetadataInput.defaultValue ??
generateDefaultValue(fieldMetadataInput.type),
options: fieldMetadataInput.options
? fieldMetadataInput.options.map((option) => ({
...option,
id: uuidV4(),
}))
: undefined,
isActive: true,
isCustom: true,
};
this.validateFieldMetadata<CreateFieldInput>(
fieldMetadataForCreate.type,
fieldMetadataForCreate,
objectMetadata,
);
if (fieldMetadataForCreate.isLabelSyncedWithName === true) {
validateNameAndLabelAreSyncOrThrow(
fieldMetadataForCreate.label,
fieldMetadataForCreate.name,
);
}
const createdFieldMetadata = await fieldMetadataRepository.save(
fieldMetadataForCreate,
);
if (!fieldMetadataInput.isRemoteCreation) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdFieldMetadata.name}`),
fieldMetadataInput.workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
action: WorkspaceMigrationTableActionType.ALTER,
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
createdFieldMetadata,
),
} satisfies WorkspaceMigrationTableAction,
],
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
fieldMetadataInput.workspaceId,
);
}
// TODO: Move viewField creation to a cdc scheduler
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
fieldMetadataInput.workspaceId,
);
const workspaceDataSource =
await this.typeORMService.connectToDataSource(dataSourceMetadata);
const workspaceQueryRunner = workspaceDataSource?.createQueryRunner();
if (!workspaceQueryRunner) {
throw new FieldMetadataException(
'Could not create workspace query runner',
FieldMetadataExceptionCode.INTERNAL_SERVER_ERROR,
);
}
await workspaceQueryRunner.connect();
await workspaceQueryRunner.startTransaction();
try {
// TODO: use typeorm repository
const view = await workspaceQueryRunner?.query(
`SELECT id FROM ${dataSourceMetadata.schema}."view"
WHERE "objectMetadataId" = '${createdFieldMetadata.objectMetadataId}'`,
);
if (!isEmpty(view)) {
const existingViewFields = (await workspaceQueryRunner?.query(
`SELECT * FROM ${dataSourceMetadata.schema}."viewField"
WHERE "viewId" = '${view[0].id}'`,
)) as ViewFieldWorkspaceEntity[];
const createdFieldIsAlreadyInView = existingViewFields.some(
(existingViewField) =>
existingViewField.fieldMetadataId === createdFieldMetadata.id,
);
if (!createdFieldIsAlreadyInView) {
const lastPosition = existingViewFields
.map((viewField) => viewField.position)
.reduce((acc, position) => {
if (position > acc) {
return position;
}
return acc;
}, -1);
await workspaceQueryRunner?.query(
`INSERT INTO ${dataSourceMetadata.schema}."viewField"
("fieldMetadataId", "position", "isVisible", "size", "viewId")
VALUES ('${createdFieldMetadata.id}', '${lastPosition + 1}', true, 180, '${
view[0].id
}')`,
);
}
}
await workspaceQueryRunner.commitTransaction();
} catch (error) {
await workspaceQueryRunner.rollbackTransaction();
throw error;
} finally {
await workspaceQueryRunner.release();
}
await queryRunner.commitTransaction();
return createdFieldMetadata;
} catch (error) {
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
await this.workspaceMetadataVersionService.incrementMetadataVersion(
fieldMetadataInput.workspaceId,
if (!createdFieldMetadata) {
throw new FieldMetadataException(
'Failed to create field metadata',
FieldMetadataExceptionCode.INTERNAL_SERVER_ERROR,
);
}
return createdFieldMetadata;
}
override async updateOne(
@ -799,4 +620,281 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
return translatedMessage;
}
private prepareCustomFieldMetadata(fieldMetadataInput: CreateFieldInput) {
return {
id: v4(),
createdAt: new Date(),
updatedAt: new Date(),
...fieldMetadataInput,
isNullable: generateNullable(
fieldMetadataInput.type,
fieldMetadataInput.isNullable,
fieldMetadataInput.isRemoteCreation,
),
defaultValue:
fieldMetadataInput.defaultValue ??
generateDefaultValue(fieldMetadataInput.type),
options: fieldMetadataInput.options
? fieldMetadataInput.options.map((option) => ({
...option,
id: uuidV4(),
}))
: undefined,
isActive: true,
isCustom: true,
};
}
private groupFieldInputsByObjectId(
fieldMetadataInputs: CreateFieldInput[],
): Record<string, CreateFieldInput[]> {
return fieldMetadataInputs.reduce(
(acc, input) => {
if (!acc[input.objectMetadataId]) {
acc[input.objectMetadataId] = [];
}
acc[input.objectMetadataId].push(input);
return acc;
},
{} as Record<string, CreateFieldInput[]>,
);
}
private async validateAndCreateFieldMetadata(
fieldMetadataInput: CreateFieldInput,
objectMetadata: ObjectMetadataEntity,
fieldMetadataRepository: Repository<FieldMetadataEntity>,
): Promise<FieldMetadataEntity> {
if (!fieldMetadataInput.isRemoteCreation) {
assertMutationNotOnRemoteObject(objectMetadata);
}
if (isEnumFieldMetadataType(fieldMetadataInput.type)) {
if (
!fieldMetadataInput.options &&
fieldMetadataInput.type !== FieldMetadataType.RATING
) {
throw new FieldMetadataException(
'Options are required for enum fields',
FieldMetadataExceptionCode.INVALID_FIELD_INPUT,
);
}
}
if (fieldMetadataInput.type === FieldMetadataType.RATING) {
fieldMetadataInput.options = generateRatingOptions();
}
const fieldMetadataForCreate =
this.prepareCustomFieldMetadata(fieldMetadataInput);
this.validateFieldMetadata<CreateFieldInput>(
fieldMetadataForCreate.type,
fieldMetadataForCreate,
objectMetadata,
);
if (fieldMetadataForCreate.isLabelSyncedWithName === true) {
validateNameAndLabelAreSyncOrThrow(
fieldMetadataForCreate.label,
fieldMetadataForCreate.name,
);
}
return await fieldMetadataRepository.save(fieldMetadataForCreate);
}
private async createMigrationActions(
createdFieldMetadata: FieldMetadataEntity,
objectMetadata: ObjectMetadataEntity,
isRemoteCreation: boolean,
): Promise<WorkspaceMigrationTableAction | null> {
if (isRemoteCreation) {
return null;
}
return {
name: computeObjectTargetTable(objectMetadata),
action: WorkspaceMigrationTableActionType.ALTER,
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
createdFieldMetadata,
),
};
}
async createMany(
fieldMetadataInputs: CreateFieldInput[],
): Promise<FieldMetadataEntity[]> {
if (!fieldMetadataInputs.length) {
return [];
}
const workspaceId = fieldMetadataInputs[0].workspaceId;
const queryRunner = this.metadataDataSource.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
try {
const fieldMetadataRepository =
queryRunner.manager.getRepository<FieldMetadataEntity>(
FieldMetadataEntity,
);
const inputsByObjectId =
this.groupFieldInputsByObjectId(fieldMetadataInputs);
const objectMetadataIds = Object.keys(inputsByObjectId);
const objectMetadatas = await this.objectMetadataRepository.find({
where: {
id: In(objectMetadataIds),
workspaceId,
},
relations: ['fields'],
});
const objectMetadataMap = objectMetadatas.reduce(
(acc, obj) => ({ ...acc, [obj.id]: obj }),
{} as Record<string, ObjectMetadataEntity>,
);
const createdFieldMetadatas: FieldMetadataEntity[] = [];
const migrationActions: WorkspaceMigrationTableAction[] = [];
for (const objectMetadataId of objectMetadataIds) {
const objectMetadata = objectMetadataMap[objectMetadataId];
if (!objectMetadata) {
throw new FieldMetadataException(
'Object metadata does not exist',
FieldMetadataExceptionCode.OBJECT_METADATA_NOT_FOUND,
);
}
const inputs = inputsByObjectId[objectMetadataId];
for (const fieldMetadataInput of inputs) {
const createdFieldMetadata =
await this.validateAndCreateFieldMetadata(
fieldMetadataInput,
objectMetadata,
fieldMetadataRepository,
);
createdFieldMetadatas.push(createdFieldMetadata);
const migrationAction = await this.createMigrationActions(
createdFieldMetadata,
objectMetadata,
fieldMetadataInput.isRemoteCreation ?? false,
);
if (migrationAction) {
migrationActions.push(migrationAction);
}
}
}
if (migrationActions.length > 0) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-multiple-fields`),
workspaceId,
migrationActions,
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId,
);
}
await this.createViewAndViewFields(createdFieldMetadatas, workspaceId);
await queryRunner.commitTransaction();
return createdFieldMetadatas;
} catch (error) {
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
}
}
private async createViewAndViewFields(
createdFieldMetadatas: FieldMetadataEntity[],
workspaceId: string,
) {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
workspaceId,
);
const workspaceDataSource =
await this.typeORMService.connectToDataSource(dataSourceMetadata);
const workspaceQueryRunner = workspaceDataSource?.createQueryRunner();
if (!workspaceQueryRunner) {
throw new FieldMetadataException(
'Could not create workspace query runner',
FieldMetadataExceptionCode.INTERNAL_SERVER_ERROR,
);
}
await workspaceQueryRunner.connect();
await workspaceQueryRunner.startTransaction();
try {
for (const createdFieldMetadata of createdFieldMetadatas) {
const view = await workspaceQueryRunner?.query(
`SELECT id FROM ${dataSourceMetadata.schema}."view"
WHERE "objectMetadataId" = '${createdFieldMetadata.objectMetadataId}'`,
);
if (!isEmpty(view)) {
const existingViewFields = (await workspaceQueryRunner?.query(
`SELECT * FROM ${dataSourceMetadata.schema}."viewField"
WHERE "viewId" = '${view[0].id}'`,
)) as ViewFieldWorkspaceEntity[];
const createdFieldIsAlreadyInView = existingViewFields.some(
(existingViewField) =>
existingViewField.fieldMetadataId === createdFieldMetadata.id,
);
if (!createdFieldIsAlreadyInView) {
const lastPosition = existingViewFields
.map((viewField) => viewField.position)
.reduce((acc, position) => {
if (position > acc) {
return position;
}
return acc;
}, -1);
await workspaceQueryRunner?.query(
`INSERT INTO ${dataSourceMetadata.schema}."viewField"
("fieldMetadataId", "position", "isVisible", "size", "viewId")
VALUES ('${createdFieldMetadata.id}', '${
lastPosition + 1
}', true, 180, '${view[0].id}')`,
);
}
}
}
await workspaceQueryRunner.commitTransaction();
} catch (error) {
await workspaceQueryRunner.rollbackTransaction();
throw error;
} finally {
await workspaceQueryRunner.release();
}
}
}

8
packages/twenty-server/src/engine/seeder/seeder.service.ts
View File

@ -37,13 +37,13 @@ export class SeederService {
throw new Error("Object metadata couldn't be created");
}
for (const fieldMetadataSeed of objectMetadataSeed.fields) {
await this.fieldMetadataService.createOne({
await this.fieldMetadataService.createMany(
objectMetadataSeed.fields.map((fieldMetadataSeed) => ({
...fieldMetadataSeed,
objectMetadataId: createdObjectMetadata.id,
workspaceId,
});
}
})),
);
const objectMetadataAfterFieldCreation =
await this.objectMetadataService.findOneWithinWorkspace(workspaceId, {