migrate rich text v1 workspace + move relation migration to 0.44 (#10582)

Adapt from MigrateRichTextFieldCommand

packages/twenty-server/src/database/commands/database-command.module.ts

@@ -9,6 +9,7 @@ import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-wo
 import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
 import { UpgradeTo0_42CommandModule } from 'src/database/commands/upgrade-version/0-42/0-42-upgrade-version.module';
 import { UpgradeTo0_43CommandModule } from 'src/database/commands/upgrade-version/0-43/0-43-upgrade-version.module';
+import { UpgradeTo0_44CommandModule } from 'src/database/commands/upgrade-version/0-44/0-44-upgrade-version.module';
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
 import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
@@ -51,6 +52,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
     WorkspaceMetadataVersionModule,
     UpgradeTo0_42CommandModule,
     UpgradeTo0_43CommandModule,
+    UpgradeTo0_44CommandModule,
     FeatureFlagModule,
   ],
   providers: [

packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-migrate-rich-text-content-patch.command.ts

@@ -0,0 +1,297 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { ServerBlockNoteEditor } from '@blocknote/server-util';
+import chalk from 'chalk';
+import { FieldMetadataType } from 'twenty-shared';
+import { Repository } from 'typeorm';
+
+import { isCommandLogger } from 'src/database/commands/logger';
+import { MigrationCommand } from 'src/database/commands/migration-command/decorators/migration-command.decorator';
+import {
+  MaintainedWorkspacesMigrationCommandOptions,
+  MaintainedWorkspacesMigrationCommandRunner,
+} from 'src/database/commands/migration-command/maintained-workspaces-migration-command.runner';
+import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
+import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { computeTableName } from 'src/engine/utils/compute-table-name.util';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
+
+type MigrateRichTextContentArgs = {
+  richTextFieldsWithObjectMetadata: RichTextFieldsWithObjectMetadata[];
+  workspaceId: string;
+};
+
+type RichTextFieldsWithObjectMetadata = {
+  richTextField: FieldMetadataEntity;
+  objectMetadata: ObjectMetadataEntity | null;
+};
+
+type ProcessWorkspaceArgs = {
+  workspaceId: string;
+  index: number;
+  total: number;
+};
+
+type ProcessRichTextFieldsArgs = {
+  richTextFields: FieldMetadataEntity[];
+  workspaceId: string;
+};
+
+@MigrationCommand({
+  name: 'migrate-rich-text-content-patch',
+  description: 'Migrate RICH_TEXT content from v1 to v2',
+  version: '0.43',
+})
+export class MigrateRichTextContentPatchCommand extends MaintainedWorkspacesMigrationCommandRunner<MaintainedWorkspacesMigrationCommandOptions> {
+  private options: MaintainedWorkspacesMigrationCommandOptions;
+
+  constructor(
+    @InjectRepository(Workspace, 'core')
+    protected readonly workspaceRepository: Repository<Workspace>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    @InjectRepository(FieldMetadataEntity, 'metadata')
+    private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
+    @InjectRepository(ObjectMetadataEntity, 'metadata')
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
+    @InjectRepository(FeatureFlag, 'core')
+    protected readonly featureFlagRepository: Repository<FeatureFlag>,
+    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  async runMigrationCommandOnMaintainedWorkspaces(
+    _passedParam: string[],
+    options: MaintainedWorkspacesMigrationCommandOptions,
+    workspaceIds: string[],
+  ): Promise<void> {
+    this.logger.log(
+      'Running command to migrate RICH_TEXT contents from v1 to v2',
+    );
+
+    this.options = options;
+    if (isCommandLogger(this.logger)) {
+      this.logger.setVerbose(options.verbose ?? false);
+    }
+
+    for (const [index, workspaceId] of workspaceIds.entries()) {
+      try {
+        await this.processWorkspace({
+          workspaceId,
+          index,
+          total: workspaceIds.length,
+        });
+      } catch (error) {
+        this.logger.log(
+          chalk.red(`Error in workspace ${workspaceId}: ${error}`),
+        );
+      }
+
+      await this.twentyORMGlobalManager.destroyDataSourceForWorkspace(
+        workspaceId,
+      );
+    }
+
+    this.logger.log(chalk.green('Command completed!'));
+  }
+
+  private async processWorkspace({
+    index,
+    total,
+    workspaceId,
+  }: ProcessWorkspaceArgs): Promise<void> {
+    try {
+      this.logger.log(
+        `Running command for workspace ${workspaceId} ${index + 1}/${total}`,
+      );
+
+      if (await this.hasRichTextV2FeatureFlag(workspaceId)) {
+        throw new Error(
+          'Rich text v2 feature flag is enabled, skipping migration',
+        );
+      }
+
+      const richTextFields = await this.fieldMetadataRepository.find({
+        where: {
+          workspaceId,
+          type: FieldMetadataType.RICH_TEXT,
+        },
+      });
+
+      if (!richTextFields.length) {
+        this.logger.log(
+          chalk.yellow('No RICH_TEXT fields found in this workspace'),
+        );
+
+        return;
+      }
+
+      this.logger.log(`Found ${richTextFields.length} RICH_TEXT fields`);
+
+      const richTextFieldsWithObjectMetadata =
+        await this.getRichTextFieldsWithObjectMetadata({
+          richTextFields,
+          workspaceId,
+        });
+
+      await this.migrateToNewRichTextFieldsColumn({
+        richTextFieldsWithObjectMetadata,
+        workspaceId,
+      });
+
+      this.logger.log(
+        chalk.green(`Command completed for workspace ${workspaceId}`),
+      );
+    } catch (error) {
+      this.logger.log(chalk.red(`Error in workspace ${workspaceId}: ${error}`));
+    }
+  }
+
+  private async hasRichTextV2FeatureFlag(
+    workspaceId: string,
+  ): Promise<boolean> {
+    return await this.featureFlagRepository.exists({
+      where: {
+        workspaceId,
+        key: FeatureFlagKey.IsRichTextV2Enabled,
+        value: true,
+      },
+    });
+  }
+
+  private async getRichTextFieldsWithObjectMetadata({
+    richTextFields,
+    workspaceId,
+  }: ProcessRichTextFieldsArgs): Promise<RichTextFieldsWithObjectMetadata[]> {
+    const richTextFieldsWithObjectMetadata: RichTextFieldsWithObjectMetadata[] =
+      [];
+
+    for (const richTextField of richTextFields) {
+      const objectMetadata = await this.objectMetadataRepository.findOne({
+        where: { id: richTextField.objectMetadataId },
+        relations: {
+          fields: true,
+        },
+      });
+
+      if (objectMetadata === null) {
+        this.logger.warn(
+          `Object metadata not found for rich text field ${richTextField.name} in workspace ${workspaceId}`,
+        );
+      }
+
+      richTextFieldsWithObjectMetadata.push({
+        richTextField,
+        objectMetadata,
+      });
+    }
+
+    return richTextFieldsWithObjectMetadata;
+  }
+
+  private jsonParseOrSilentlyFail(input: string): null | unknown {
+    try {
+      return JSON.parse(input);
+    } catch (e) {
+      return null;
+    }
+  }
+
+  private async getMarkdownFieldValue({
+    blocknoteFieldValue,
+    serverBlockNoteEditor,
+  }: {
+    blocknoteFieldValue: string | null;
+    serverBlockNoteEditor: ServerBlockNoteEditor;
+  }): Promise<string | null> {
+    const blocknoteFieldValueIsDefined =
+      blocknoteFieldValue !== null &&
+      blocknoteFieldValue !== undefined &&
+      blocknoteFieldValue !== '{}';
+
+    if (!blocknoteFieldValueIsDefined) {
+      return null;
+    }
+
+    const jsonParsedblocknoteFieldValue =
+      this.jsonParseOrSilentlyFail(blocknoteFieldValue);
+
+    if (jsonParsedblocknoteFieldValue === null) {
+      return null;
+    }
+
+    if (!Array.isArray(jsonParsedblocknoteFieldValue)) {
+      this.logger.warn(
+        `blocknoteFieldValue is defined and is not an array got ${blocknoteFieldValue}`,
+      );
+
+      return null;
+    }
+
+    let markdown: string | null = null;
+
+    try {
+      markdown = await serverBlockNoteEditor.blocksToMarkdownLossy(
+        jsonParsedblocknoteFieldValue,
+      );
+    } catch (error) {
+      this.logger.warn(
+        `Error converting blocknote to markdown for ${blocknoteFieldValue}`,
+      );
+    }
+
+    return markdown;
+  }
+
+  private async migrateToNewRichTextFieldsColumn({
+    richTextFieldsWithObjectMetadata,
+    workspaceId,
+  }: MigrateRichTextContentArgs) {
+    const serverBlockNoteEditor = ServerBlockNoteEditor.create();
+
+    for (const {
+      richTextField,
+      objectMetadata,
+    } of richTextFieldsWithObjectMetadata) {
+      if (objectMetadata === null) {
+        this.logger.log(
+          `Object metadata not found for rich text field ${richTextField.name} in workspace ${workspaceId}`,
+        );
+        continue;
+      }
+
+      const schemaName =
+        this.workspaceDataSourceService.getSchemaName(workspaceId);
+
+      const workspaceDataSource =
+        await this.twentyORMGlobalManager.getDataSourceForWorkspace(
+          workspaceId,
+        );
+
+      const rows = await workspaceDataSource.query(
+        `SELECT id, "${richTextField.name}" FROM "${schemaName}"."${computeTableName(objectMetadata.nameSingular, objectMetadata.isCustom)}" WHERE "${richTextField.name}" IS NOT NULL`,
+      );
+
+      this.logger.log(`Generating markdown for ${rows.length} records`);
+
+      for (const row of rows) {
+        const blocknoteFieldValue = row[richTextField.name];
+        const markdownFieldValue = await this.getMarkdownFieldValue({
+          blocknoteFieldValue,
+          serverBlockNoteEditor,
+        });
+
+        if (!this.options.dryRun) {
+          await workspaceDataSource.query(
+            `UPDATE "${schemaName}"."${computeTableName(objectMetadata.nameSingular, objectMetadata.isCustom)}" SET "${richTextField.name}V2Blocknote" = $1, "${richTextField.name}V2Markdown" = $2 WHERE id = $3`,
+            [blocknoteFieldValue, markdownFieldValue, row.id],
+          );
+        }
+      }
+    }
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version/0-43/0-43-upgrade-version.module.ts

@@ -5,7 +5,7 @@ import { MigrationCommandModule } from 'src/database/commands/migration-command/
 import { StandardizationOfActorCompositeContextTypeCommand } from 'src/database/commands/upgrade-version/0-42/0-42-standardization-of-actor-composite-context-type';
 import { AddTasksAssignedToMeViewCommand } from 'src/database/commands/upgrade-version/0-43/0-43-add-tasks-assigned-to-me-view.command';
 import { MigrateIsSearchableForCustomObjectMetadataCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-is-searchable-for-custom-object-metadata.command';
-import { MigrateRelationsToFieldMetadataCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-relations-to-field-metadata.command';
+import { MigrateRichTextContentPatchCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-rich-text-content-patch.command';
 import { MigrateSearchVectorOnNoteAndTaskEntitiesCommand } from 'src/database/commands/upgrade-version/0-43/0-43-migrate-search-vector-on-note-and-task-entities.command';
 import { UpdateDefaultViewRecordOpeningOnWorkflowObjectsCommand } from 'src/database/commands/upgrade-version/0-43/0-43-update-default-view-record-opening-on-workflow-objects.command';
 import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
@@ -15,6 +15,7 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
 import { SearchModule } from 'src/engine/metadata-modules/search/search.module';
 import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
 import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
 import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
 
 @Module({
@@ -30,6 +31,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor
         WorkspaceMigrationRunnerModule,
         WorkspaceMigrationModule,
         WorkspaceMetadataVersionModule,
+        WorkspaceDataSourceModule,
       ],
       providers: [
         AddTasksAssignedToMeViewCommand,
@@ -37,7 +39,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor
         MigrateIsSearchableForCustomObjectMetadataCommand,
         UpdateDefaultViewRecordOpeningOnWorkflowObjectsCommand,
         StandardizationOfActorCompositeContextTypeCommand,
-        MigrateRelationsToFieldMetadataCommand,
+        MigrateRichTextContentPatchCommand,
       ],
     }),
   ],

packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-migrate-relations-to-field-metadata.command.ts

@@ -24,7 +24,7 @@ import { isFieldMetadataOfType } from 'src/engine/utils/is-field-metadata-of-typ
 @MigrationCommand({
   name: 'migrate-relations-to-field-metadata',
   description: 'Migrate relations to field metadata',
-  version: '0.43',
+  version: '0.44',
 })
 export class MigrateRelationsToFieldMetadataCommand extends MaintainedWorkspacesMigrationCommandRunner {
   constructor(

packages/twenty-server/src/database/commands/upgrade-version/0-44/0-44-upgrade-version.module.ts

@@ -0,0 +1,31 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { MigrationCommandModule } from 'src/database/commands/migration-command/migration-command.module';
+import { MigrateRelationsToFieldMetadataCommand } from 'src/database/commands/upgrade-version/0-44/0-44-migrate-relations-to-field-metadata.command';
+import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
+import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
+import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
+
+@Module({
+  imports: [
+    MigrationCommandModule.register('0.44', {
+      imports: [
+        TypeOrmModule.forFeature([Workspace, FeatureFlag], 'core'),
+        TypeOrmModule.forFeature(
+          [ObjectMetadataEntity, FieldMetadataEntity],
+          'metadata',
+        ),
+        WorkspaceMigrationRunnerModule,
+        WorkspaceMigrationModule,
+        WorkspaceMetadataVersionModule,
+      ],
+      providers: [MigrateRelationsToFieldMetadataCommand],
+    }),
+  ],
+})
+export class UpgradeTo0_44CommandModule {}