From fba63d9cb7d269208fafcebe34b2cb893d336138 Mon Sep 17 00:00:00 2001 From: Etienne <45695613+etiennejouan@users.noreply.github.com> Date: Fri, 28 Feb 2025 14:46:34 +0100 Subject: [PATCH] migrate rich text v1 workspace + move relation migration to 0.44 (#10582) Adapt from MigrateRichTextFieldCommand --- .../commands/database-command.module.ts | 2 + ...migrate-rich-text-content-patch.command.ts | 297 ++++++++++++++++++ .../0-43/0-43-upgrade-version.module.ts | 6 +- ...te-relations-to-field-metadata.command.ts} | 2 +- .../0-44/0-44-upgrade-version.module.ts | 31 ++ 5 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-rich-text-content-patch.command.ts rename packages/twenty-server/src/database/commands/upgrade-version/{0-43/0-43-migrate-relations-to-field-metadata.command.ts => 0-44/0-44-migrate-relations-to-field-metadata.command.ts} (99%) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-upgrade-version.module.ts diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index 402895f95..5c7101c62 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -9,6 +9,7 @@ import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-wo import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; import { UpgradeTo0_42CommandModule } from 'src/database/commands/upgrade-version/0-42/0-42-upgrade-version.module'; import { UpgradeTo0_43CommandModule } from 'src/database/commands/upgrade-version/0-43/0-43-upgrade-version.module'; +import { UpgradeTo0_44CommandModule } from 'src/database/commands/upgrade-version/0-44/0-44-upgrade-version.module'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity'; import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; @@ -51,6 +52,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp WorkspaceMetadataVersionModule, UpgradeTo0_42CommandModule, UpgradeTo0_43CommandModule, + UpgradeTo0_44CommandModule, FeatureFlagModule, ], providers: [ diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-rich-text-content-patch.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-rich-text-content-patch.command.ts new file mode 100644 index 000000000..95339b9a8 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-rich-text-content-patch.command.ts @@ -0,0 +1,297 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import { ServerBlockNoteEditor } from '@blocknote/server-util'; +import chalk from 'chalk'; +import { FieldMetadataType } from 'twenty-shared'; +import { Repository } from 'typeorm'; + +import { isCommandLogger } from 'src/database/commands/logger'; +import { MigrationCommand } from 'src/database/commands/migration-command/decorators/migration-command.decorator'; +import { + MaintainedWorkspacesMigrationCommandOptions, + MaintainedWorkspacesMigrationCommandRunner, +} from 'src/database/commands/migration-command/maintained-workspaces-migration-command.runner'; +import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; +import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { computeTableName } from 'src/engine/utils/compute-table-name.util'; +import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; + +type MigrateRichTextContentArgs = { + richTextFieldsWithObjectMetadata: RichTextFieldsWithObjectMetadata[]; + workspaceId: string; +}; + +type RichTextFieldsWithObjectMetadata = { + richTextField: FieldMetadataEntity; + objectMetadata: ObjectMetadataEntity | null; +}; + +type ProcessWorkspaceArgs = { + workspaceId: string; + index: number; + total: number; +}; + +type ProcessRichTextFieldsArgs = { + richTextFields: FieldMetadataEntity[]; + workspaceId: string; +}; + +@MigrationCommand({ + name: 'migrate-rich-text-content-patch', + description: 'Migrate RICH_TEXT content from v1 to v2', + version: '0.43', +}) +export class MigrateRichTextContentPatchCommand extends MaintainedWorkspacesMigrationCommandRunner { + private options: MaintainedWorkspacesMigrationCommandOptions; + + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, + @InjectRepository(FeatureFlag, 'core') + protected readonly featureFlagRepository: Repository, + private readonly workspaceDataSourceService: WorkspaceDataSourceService, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + async runMigrationCommandOnMaintainedWorkspaces( + _passedParam: string[], + options: MaintainedWorkspacesMigrationCommandOptions, + workspaceIds: string[], + ): Promise { + this.logger.log( + 'Running command to migrate RICH_TEXT contents from v1 to v2', + ); + + this.options = options; + if (isCommandLogger(this.logger)) { + this.logger.setVerbose(options.verbose ?? false); + } + + for (const [index, workspaceId] of workspaceIds.entries()) { + try { + await this.processWorkspace({ + workspaceId, + index, + total: workspaceIds.length, + }); + } catch (error) { + this.logger.log( + chalk.red(`Error in workspace ${workspaceId}: ${error}`), + ); + } + + await this.twentyORMGlobalManager.destroyDataSourceForWorkspace( + workspaceId, + ); + } + + this.logger.log(chalk.green('Command completed!')); + } + + private async processWorkspace({ + index, + total, + workspaceId, + }: ProcessWorkspaceArgs): Promise { + try { + this.logger.log( + `Running command for workspace ${workspaceId} ${index + 1}/${total}`, + ); + + if (await this.hasRichTextV2FeatureFlag(workspaceId)) { + throw new Error( + 'Rich text v2 feature flag is enabled, skipping migration', + ); + } + + const richTextFields = await this.fieldMetadataRepository.find({ + where: { + workspaceId, + type: FieldMetadataType.RICH_TEXT, + }, + }); + + if (!richTextFields.length) { + this.logger.log( + chalk.yellow('No RICH_TEXT fields found in this workspace'), + ); + + return; + } + + this.logger.log(`Found ${richTextFields.length} RICH_TEXT fields`); + + const richTextFieldsWithObjectMetadata = + await this.getRichTextFieldsWithObjectMetadata({ + richTextFields, + workspaceId, + }); + + await this.migrateToNewRichTextFieldsColumn({ + richTextFieldsWithObjectMetadata, + workspaceId, + }); + + this.logger.log( + chalk.green(`Command completed for workspace ${workspaceId}`), + ); + } catch (error) { + this.logger.log(chalk.red(`Error in workspace ${workspaceId}: ${error}`)); + } + } + + private async hasRichTextV2FeatureFlag( + workspaceId: string, + ): Promise { + return await this.featureFlagRepository.exists({ + where: { + workspaceId, + key: FeatureFlagKey.IsRichTextV2Enabled, + value: true, + }, + }); + } + + private async getRichTextFieldsWithObjectMetadata({ + richTextFields, + workspaceId, + }: ProcessRichTextFieldsArgs): Promise { + const richTextFieldsWithObjectMetadata: RichTextFieldsWithObjectMetadata[] = + []; + + for (const richTextField of richTextFields) { + const objectMetadata = await this.objectMetadataRepository.findOne({ + where: { id: richTextField.objectMetadataId }, + relations: { + fields: true, + }, + }); + + if (objectMetadata === null) { + this.logger.warn( + `Object metadata not found for rich text field ${richTextField.name} in workspace ${workspaceId}`, + ); + } + + richTextFieldsWithObjectMetadata.push({ + richTextField, + objectMetadata, + }); + } + + return richTextFieldsWithObjectMetadata; + } + + private jsonParseOrSilentlyFail(input: string): null | unknown { + try { + return JSON.parse(input); + } catch (e) { + return null; + } + } + + private async getMarkdownFieldValue({ + blocknoteFieldValue, + serverBlockNoteEditor, + }: { + blocknoteFieldValue: string | null; + serverBlockNoteEditor: ServerBlockNoteEditor; + }): Promise { + const blocknoteFieldValueIsDefined = + blocknoteFieldValue !== null && + blocknoteFieldValue !== undefined && + blocknoteFieldValue !== '{}'; + + if (!blocknoteFieldValueIsDefined) { + return null; + } + + const jsonParsedblocknoteFieldValue = + this.jsonParseOrSilentlyFail(blocknoteFieldValue); + + if (jsonParsedblocknoteFieldValue === null) { + return null; + } + + if (!Array.isArray(jsonParsedblocknoteFieldValue)) { + this.logger.warn( + `blocknoteFieldValue is defined and is not an array got ${blocknoteFieldValue}`, + ); + + return null; + } + + let markdown: string | null = null; + + try { + markdown = await serverBlockNoteEditor.blocksToMarkdownLossy( + jsonParsedblocknoteFieldValue, + ); + } catch (error) { + this.logger.warn( + `Error converting blocknote to markdown for ${blocknoteFieldValue}`, + ); + } + + return markdown; + } + + private async migrateToNewRichTextFieldsColumn({ + richTextFieldsWithObjectMetadata, + workspaceId, + }: MigrateRichTextContentArgs) { + const serverBlockNoteEditor = ServerBlockNoteEditor.create(); + + for (const { + richTextField, + objectMetadata, + } of richTextFieldsWithObjectMetadata) { + if (objectMetadata === null) { + this.logger.log( + `Object metadata not found for rich text field ${richTextField.name} in workspace ${workspaceId}`, + ); + continue; + } + + const schemaName = + this.workspaceDataSourceService.getSchemaName(workspaceId); + + const workspaceDataSource = + await this.twentyORMGlobalManager.getDataSourceForWorkspace( + workspaceId, + ); + + const rows = await workspaceDataSource.query( + `SELECT id, "${richTextField.name}" FROM "${schemaName}"."${computeTableName(objectMetadata.nameSingular, objectMetadata.isCustom)}" WHERE "${richTextField.name}" IS NOT NULL`, + ); + + this.logger.log(`Generating markdown for ${rows.length} records`); + + for (const row of rows) { + const blocknoteFieldValue = row[richTextField.name]; + const markdownFieldValue = await this.getMarkdownFieldValue({ + blocknoteFieldValue, + serverBlockNoteEditor, + }); + + if (!this.options.dryRun) { + await workspaceDataSource.query( + `UPDATE "${schemaName}"."${computeTableName(objectMetadata.nameSingular, objectMetadata.isCustom)}" SET "${richTextField.name}V2Blocknote" = $1, "${richTextField.name}V2Markdown" = $2 WHERE id = $3`, + [blocknoteFieldValue, markdownFieldValue, row.id], + ); + } + } + } + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-upgrade-version.module.ts index 4793d89cf..c3578ec22 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-upgrade-version.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-upgrade-version.module.ts @@ -5,7 +5,7 @@ import { MigrationCommandModule } from 'src/database/commands/migration-command/ import { StandardizationOfActorCompositeContextTypeCommand } from 'src/database/commands/upgrade-version/0-42/0-42-standardization-of-actor-composite-context-type'; import { AddTasksAssignedToMeViewCommand } from 'src/database/commands/upgrade-version/0-43/0-43-add-tasks-assigned-to-me-view.command'; import { MigrateIsSearchableForCustomObjectMetadataCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-is-searchable-for-custom-object-metadata.command'; -import { MigrateRelationsToFieldMetadataCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-relations-to-field-metadata.command'; +import { MigrateRichTextContentPatchCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-rich-text-content-patch.command'; import { MigrateSearchVectorOnNoteAndTaskEntitiesCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-search-vector-on-note-and-task-entities.command'; import { UpdateDefaultViewRecordOpeningOnWorkflowObjectsCommand } from 'src/database/commands/upgrade-version/0-43/0-43-update-default-view-record-opening-on-workflow-objects.command'; import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; @@ -15,6 +15,7 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat import { SearchModule } from 'src/engine/metadata-modules/search/search.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @Module({ @@ -30,6 +31,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor WorkspaceMigrationRunnerModule, WorkspaceMigrationModule, WorkspaceMetadataVersionModule, + WorkspaceDataSourceModule, ], providers: [ AddTasksAssignedToMeViewCommand, @@ -37,7 +39,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor MigrateIsSearchableForCustomObjectMetadataCommand, UpdateDefaultViewRecordOpeningOnWorkflowObjectsCommand, StandardizationOfActorCompositeContextTypeCommand, - MigrateRelationsToFieldMetadataCommand, + MigrateRichTextContentPatchCommand, ], }), ], diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-relations-to-field-metadata.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-migrate-relations-to-field-metadata.command.ts similarity index 99% rename from packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-relations-to-field-metadata.command.ts rename to packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-migrate-relations-to-field-metadata.command.ts index d9b2104be..74bb74658 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-relations-to-field-metadata.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-migrate-relations-to-field-metadata.command.ts @@ -24,7 +24,7 @@ import { isFieldMetadataOfType } from 'src/engine/utils/is-field-metadata-of-typ @MigrationCommand({ name: 'migrate-relations-to-field-metadata', description: 'Migrate relations to field metadata', - version: '0.43', + version: '0.44', }) export class MigrateRelationsToFieldMetadataCommand extends MaintainedWorkspacesMigrationCommandRunner { constructor( diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-upgrade-version.module.ts new file mode 100644 index 000000000..fff8cf08f --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-upgrade-version.module.ts @@ -0,0 +1,31 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { MigrationCommandModule } from 'src/database/commands/migration-command/migration-command.module'; +import { MigrateRelationsToFieldMetadataCommand } from 'src/database/commands/upgrade-version/0-44/0-44-migrate-relations-to-field-metadata.command'; +import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; + +@Module({ + imports: [ + MigrationCommandModule.register('0.44', { + imports: [ + TypeOrmModule.forFeature([Workspace, FeatureFlag], 'core'), + TypeOrmModule.forFeature( + [ObjectMetadataEntity, FieldMetadataEntity], + 'metadata', + ), + WorkspaceMigrationRunnerModule, + WorkspaceMigrationModule, + WorkspaceMetadataVersionModule, + ], + providers: [MigrateRelationsToFieldMetadataCommand], + }), + ], +}) +export class UpgradeTo0_44CommandModule {}