mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor migration runner within transaction (#12941)

Browse Source 
Modifying the data-model can sometimes fail in the middle of your
operation, due to the way we handle both metadata update and schema
migration separately, a field can be created while the associated column
creation failed (same for object/table and such). This is also an issue
because WorkspaceMigrations are then stored as FAILED can never really
recovered by themselves so the schema is broken and we can't update the
models anymore.
This PR adds a executeMigrationFromPendingMigrationsWithinTransaction
method where we can (and must) pass a queryRunner executing a
transaction, which should come from the metadata services so that if
anything during metadata update OR schema update fails, it rolls back
everything (this also mean a workspaceMigration should never stay in a
failed state now).
This also fixes some issues with migration not running in the correct
order due to having the same timestamp and having to do some weird logic
to fix that.

This is a first step and fix before working on a much more reliable
solution in the upcoming weeks where we will refactor the way we
interact with the data model.

---------

Co-authored-by: Charles Bochet <charlesBochet@users.noreply.github.com>
This commit is contained in:
Weiko
2025-07-02 19:21:26 +02:00
committed by GitHub
parent 8cbb1aa71a
commit 41becaaea4
14 changed files with 726 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
View File

@ -158,6 +158,11 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
await queryRunner.startTransaction();
try {
const fieldMetadataRepository =
queryRunner.manager.getRepository<FieldMetadataEntity>(
FieldMetadataEntity,
);
if (
!isDefined(
objectMetadataItemWithFieldMaps.labelIdentifierFieldMetadataId,
@ -224,10 +229,9 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
);
}
// We're running field update under a transaction, so we can rollback if migration fails
await this.fieldMetadataRepository.update(id, fieldMetadataForUpdate);
await fieldMetadataRepository.update(id, fieldMetadataForUpdate);
const [updatedFieldMetadata] = await this.fieldMetadataRepository.find({
const [updatedFieldMetadata] = await fieldMetadataRepository.find({
where: { id },
});
@ -238,6 +242,36 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
);
}
if (
isDefined(fieldMetadataInput.name) ||
isDefined(updatableFieldInput.options) ||
isDefined(updatableFieldInput.defaultValue)
) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`update-${updatedFieldMetadata.name}`),
fieldMetadataInput.workspaceId,
[
{
name: computeObjectTargetTable(objectMetadataItemWithFieldMaps),
action: WorkspaceMigrationTableActionType.ALTER,
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.ALTER,
existingFieldMetadata,
updatedFieldMetadata,
),
} satisfies WorkspaceMigrationTableAction,
],
queryRunner,
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrationsWithinTransaction(
updatedFieldMetadata.workspaceId,
queryRunner,
);
}
await queryRunner.commitTransaction();
if (fieldMetadataInput.isActive === false) {
const viewsRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
@ -266,44 +300,19 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
);
}
if (
isDefined(fieldMetadataInput.name) ||
isDefined(updatableFieldInput.options) ||
isDefined(updatableFieldInput.defaultValue)
) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`update-${updatedFieldMetadata.name}`),
fieldMetadataInput.workspaceId,
[
{
name: computeObjectTargetTable(objectMetadataItemWithFieldMaps),
action: WorkspaceMigrationTableActionType.ALTER,
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.ALTER,
existingFieldMetadata,
updatedFieldMetadata,
),
} satisfies WorkspaceMigrationTableAction,
],
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
updatedFieldMetadata.workspaceId,
);
}
await queryRunner.commitTransaction();
return updatedFieldMetadata;
} catch (error) {
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
await this.workspaceMetadataVersionService.incrementMetadataVersion(
fieldMetadataInput.workspaceId,
);
return updatedFieldMetadata;
} catch (error) {
if (queryRunner.isTransactionActive) {
await queryRunner.rollbackTransaction();
}
throw error;
} finally {
await queryRunner.release();
}
}
@ -314,7 +323,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
const queryRunner = this.coreDataSource.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction(); // transaction not safe as a different queryRunner is used within workspaceMigrationRunnerService
await queryRunner.startTransaction();
try {
const fieldMetadataRepository =
@ -357,11 +366,6 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
);
}
await this.viewService.resetKanbanAggregateOperationByFieldMetadataId({
workspaceId,
fieldMetadataId: fieldMetadata.id,
});
if (fieldMetadata.type === FieldMetadataType.RELATION) {
const isManyToOneRelation =
(fieldMetadata as FieldMetadataEntity<FieldMetadataType.RELATION>)
@ -402,6 +406,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
],
} satisfies WorkspaceMigrationTableAction,
],
queryRunner,
);
} else if (isCompositeFieldMetadataType(fieldMetadata.type)) {
await fieldMetadataRepository.delete(fieldMetadata.id);
@ -433,6 +438,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
}),
} satisfies WorkspaceMigrationTableAction,
],
queryRunner,
);
} else {
await fieldMetadataRepository.delete(fieldMetadata.id);
@ -451,24 +457,35 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
],
} satisfies WorkspaceMigrationTableAction,
],
queryRunner,
);
}
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrationsWithinTransaction(
workspaceId,
queryRunner,
);
await queryRunner.commitTransaction();
return fieldMetadata;
} catch (error) {
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
await this.viewService.resetKanbanAggregateOperationByFieldMetadataId({
workspaceId,
fieldMetadataId: fieldMetadata.id,
});
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return fieldMetadata;
} catch (error) {
if (queryRunner.isTransactionActive) {
await queryRunner.rollbackTransaction();
}
throw error;
} finally {
await queryRunner.release();
}
}
@ -943,26 +960,32 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
generateMigrationName(`create-multiple-fields`),
workspaceId,
migrationActions,
queryRunner,
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrationsWithinTransaction(
workspaceId,
queryRunner,
);
}
await this.createViewAndViewFields(createdFieldMetadatas, workspaceId);
await queryRunner.commitTransaction();
return createdFieldMetadatas;
} catch (error) {
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
await this.createViewAndViewFields(createdFieldMetadatas, workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return createdFieldMetadatas;
} catch (error) {
if (queryRunner.isTransactionActive) {
await queryRunner.rollbackTransaction();
}
throw error;
} finally {
await queryRunner.release();
}
}