[BUGFIX][PROD] RICH_TEXT_V2 command handle {} body col value (#10324)

# Introduction
Encountered in issue in production where we have a lot of records that
has RICH_TEXT_FIELD set to `{}`

```sh
[Nest] 20106  - 02/19/2025, 12:43:08 PM     LOG [MigrateRichTextFieldCommand] Generating markdown for 1 records
[Nest] 20106  - 02/19/2025, 12:43:09 PM     LOG [MigrateRichTextFieldCommand] Error in workspace 3b8e6458-5fc1-4e63-8563-008ccddaa6db: TypeError: o is not iterable
```

## Fix
While reading `fieldValue` definition also strictly check if it's `{}` +
checking after JSON parse if it's an iterable to pass to the
`serverBlockNoteEditor` in order to be 100 bullet proof for prod
migration command

## Refactor Dry run
Implemented dry run

## Refactor to Idempotency
Made the script idempotent in order to avoid issues with re-running
commands

## Error repro
- In local checkout on v0.41.5 run `yarn && npx nx reset && npx nx
start`
- Create record manually in db that has a RICH_TEXT body to `{}`
- Checkout to main, `yarn && npx nx reset && npx nx build twenty-server
&& yarn command:prod upgrade-0.42:migrate-rich-text-field -d`

packages/twenty-server/src/database/commands/upgrade-version/0-42/0-42-migrate-rich-text-field.command.ts

@@ -2,15 +2,13 @@ import { InjectRepository } from '@nestjs/typeorm';
 
 import { ServerBlockNoteEditor } from '@blocknote/server-util';
 import chalk from 'chalk';
-import { Command } from 'nest-commander';
+import { Command, Option } from 'nest-commander';
-import { FieldMetadataType } from 'twenty-shared';
+import { FieldMetadataType, isDefined } from 'twenty-shared';
 import { Repository } from 'typeorm';
 
-import {
+import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
-  ActiveWorkspacesCommandOptions,
-  ActiveWorkspacesCommandRunner,
-} from 'src/database/commands/active-workspaces.command';
 import { isCommandLogger } from 'src/database/commands/logger';
+import { Upgrade042CommandOptions } from 'src/database/commands/upgrade-version/0-42/0-42-upgrade-version.command';
 import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
 import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -35,11 +33,33 @@ import {
   TASK_STANDARD_FIELD_IDS,
 } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
 
+type MigrateRichTextContentArgs = {
+  richTextFieldsWithHasCreatedColumns: RichTextFieldWithHasCreatedColumnsAndObjectMetadata[];
+  workspaceId: string;
+};
+
+type RichTextFieldWithHasCreatedColumnsAndObjectMetadata = {
+  richTextField: FieldMetadataEntity;
+  hasCreatedColumns: boolean;
+  objectMetadata: ObjectMetadataEntity | null;
+};
+
+type ProcessWorkspaceArgs = {
+  workspaceId: string;
+  index: number;
+  total: number;
+};
+
+type ProcessRichTextFieldsArgs = {
+  richTextFields: FieldMetadataEntity[];
+  workspaceId: string;
+};
 @Command({
   name: 'upgrade-0.42:migrate-rich-text-field',
   description: 'Migrate RICH_TEXT fields to new composite structure',
 })
 export class MigrateRichTextFieldCommand extends ActiveWorkspacesCommandRunner {
+  private options: Upgrade042CommandOptions;
   constructor(
     @InjectRepository(Workspace, 'core')
     protected readonly workspaceRepository: Repository<Workspace>,
@@ -58,22 +78,39 @@ export class MigrateRichTextFieldCommand extends ActiveWorkspacesCommandRunner {
     super(workspaceRepository);
   }
 
+  @Option({
+    flags: '-f, --force [boolean]',
+    description:
+      'Force RICH_TEXT_FIELD value update even if column migration has already be run',
+    required: false,
+  })
+  parseForceValue(val?: boolean): boolean {
+    return val ?? false;
+  }
+
   async executeActiveWorkspacesCommand(
     _passedParam: string[],
-    options: ActiveWorkspacesCommandOptions,
+    options: Upgrade042CommandOptions,
     workspaceIds: string[],
   ): Promise<void> {
     this.logger.log(
       'Running command to migrate RICH_TEXT fields to new composite structure',
     );
-
+    if (options.force) {
+      this.logger.warn('Running in force mode');
+    }
+    this.options = options;
     if (isCommandLogger(this.logger)) {
       this.logger.setVerbose(options.verbose ?? false);
     }
 
     try {
       for (const [index, workspaceId] of workspaceIds.entries()) {
-        await this.processWorkspace(workspaceId, index, workspaceIds.length);
+        await this.processWorkspace({
+          workspaceId,
+          index,
+          total: workspaceIds.length,
+        });
       }
 
       this.logger.log(chalk.green('Command completed!'));
@@ -82,11 +119,11 @@ export class MigrateRichTextFieldCommand extends ActiveWorkspacesCommandRunner {
     }
   }
 
-  private async processWorkspace(
+  private async processWorkspace({
-    workspaceId: string,
+    index,
-    index: number,
+    total,
-    total: number,
+    workspaceId,
-  ): Promise<void> {
+  }: ProcessWorkspaceArgs): Promise<void> {
     try {
       this.logger.log(
         `Running command for workspace ${workspaceId} ${index + 1}/${total}`,
@@ -109,22 +146,28 @@ export class MigrateRichTextFieldCommand extends ActiveWorkspacesCommandRunner {
 
       this.logger.log(`Found ${richTextFields.length} RICH_TEXT fields`);
 
-      for (const richTextField of richTextFields) {
+      const richTextFieldsWithHasCreatedColumns =
-        await this.processRichTextField(richTextField, workspaceId);
+        await this.createIfMissingNewRichTextFieldsColumn({
-      }
+          richTextFields,
+          workspaceId,
+        });
 
-      await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
+      await this.migrateToNewRichTextFieldsColumn({
+        richTextFieldsWithHasCreatedColumns,
         workspaceId,
-      );
+      });
-
-      await this.workspaceMetadataVersionService.incrementMetadataVersion(
-        workspaceId,
-      );
-
-      await this.migrateRichTextContent(richTextFields, workspaceId);
 
       await this.enableRichTextV2FeatureFlag(workspaceId);
 
+      if (!this.options.dryRun) {
+        await this.workspaceMetadataVersionService.incrementMetadataVersion(
+          workspaceId,
+        );
+      }
+
+      await this.twentyORMGlobalManager.destroyDataSourceForWorkspace(
+        workspaceId,
+      );
       this.logger.log(
         chalk.green(`Command completed for workspace ${workspaceId}`),
       );
@@ -136,101 +179,241 @@ export class MigrateRichTextFieldCommand extends ActiveWorkspacesCommandRunner {
   private async enableRichTextV2FeatureFlag(
     workspaceId: string,
   ): Promise<void> {
-    await this.featureFlagRepository.upsert(
+    if (!this.options.dryRun) {
-      {
+      await this.featureFlagRepository.upsert(
-        workspaceId,
-        key: FeatureFlagKey.IsRichTextV2Enabled,
-        value: true,
-      },
-      {
-        conflictPaths: ['workspaceId', 'key'],
-        skipUpdateIfNoValuesChanged: true,
-      },
-    );
-  }
-
-  private async processRichTextField(
-    richTextField: FieldMetadataEntity,
-    workspaceId: string,
-  ) {
-    let standardId: string | null = null;
-
-    if (richTextField.standardId === TASK_STANDARD_FIELD_IDS.body) {
-      standardId = TASK_STANDARD_FIELD_IDS.bodyV2;
-    } else if (richTextField.standardId === NOTE_STANDARD_FIELD_IDS.body) {
-      standardId = NOTE_STANDARD_FIELD_IDS.bodyV2;
-    }
-
-    if (standardId === null && richTextField.isCustom === false) {
-      throw new Error(
-        `RICH_TEXT does not belong to a Task or a Note standard objects: ${richTextField.id}`,
-      );
-    }
-
-    const newRichTextField: Partial<FieldMetadataEntity> = {
-      ...richTextField,
-      name: `${richTextField.name}V2`,
-      id: undefined,
-      type: FieldMetadataType.RICH_TEXT_V2,
-      defaultValue: null,
-      standardId,
-    };
-
-    await this.fieldMetadataRepository.insert(newRichTextField);
-
-    const objectMetadata = await this.objectMetadataRepository.findOne({
-      where: { id: richTextField.objectMetadataId },
-    });
-
-    if (objectMetadata === null) {
-      this.logger.log(
-        `Object metadata not found for rich text field ${richTextField.name} in workspace ${workspaceId}`,
-      );
-
-      return;
-    }
-
-    await this.workspaceMigrationService.createCustomMigration(
-      generateMigrationName(
-        `migrate-rich-text-field-${objectMetadata.nameSingular}-${richTextField.name}`,
-      ),
-      workspaceId,
-      [
         {
-          name: computeObjectTargetTable(objectMetadata),
+          workspaceId,
-          action: WorkspaceMigrationTableActionType.ALTER,
+          key: FeatureFlagKey.IsRichTextV2Enabled,
-          columns: [
+          value: true,
-            {
+        },
-              action: WorkspaceMigrationColumnActionType.CREATE,
+        {
-              columnName: `${richTextField.name}V2Blocknote`,
+          conflictPaths: ['workspaceId', 'key'],
-              columnType: 'text',
+          skipUpdateIfNoValuesChanged: true,
-              isNullable: true,
+        },
-              defaultValue: null,
+      );
-            } satisfies WorkspaceMigrationColumnCreate,
+    }
-            {
-              action: WorkspaceMigrationColumnActionType.CREATE,
-              columnName: `${richTextField.name}V2Markdown`,
-              columnType: 'text',
-              isNullable: true,
-              defaultValue: null,
-            } satisfies WorkspaceMigrationColumnCreate,
-          ],
-        } satisfies WorkspaceMigrationTableAction,
-      ],
-    );
   }
 
-  private async migrateRichTextContent(
+  private buildRichTextFieldStandardId(richTextField: FieldMetadataEntity) {
-    richTextFields: FieldMetadataEntity[],
+    switch (true) {
-    workspaceId: string,
+      case richTextField.standardId === TASK_STANDARD_FIELD_IDS.body: {
-  ) {
+        return TASK_STANDARD_FIELD_IDS.bodyV2;
-    const serverBlockNoteEditor = ServerBlockNoteEditor.create();
+      }
+      case richTextField.standardId === NOTE_STANDARD_FIELD_IDS.body: {
+        return NOTE_STANDARD_FIELD_IDS.bodyV2;
+      }
+      case richTextField.isCustom: {
+        return null;
+      }
+      default: {
+        throw new Error(
+          `RICH_TEXT does not belong to a Task or a Note standard objects: ${richTextField.id}`,
+        );
+      }
+    }
+  }
+
+  private async createMarkdownBlockNoteV2Columns({
+    richTextField,
+    workspaceId,
+    objectMetadata,
+    fieldMetadataAlreadyExisting,
+  }: {
+    objectMetadata: ObjectMetadataEntity;
+    richTextField: FieldMetadataEntity;
+    workspaceId: string;
+    fieldMetadataAlreadyExisting: boolean;
+  }) {
+    const columnsToCreate: WorkspaceMigrationColumnCreate[] = [
+      {
+        action: WorkspaceMigrationColumnActionType.CREATE,
+        columnName: `${richTextField.name}V2Blocknote`,
+        columnType: 'text',
+        isNullable: true,
+        defaultValue: null,
+      },
+      {
+        action: WorkspaceMigrationColumnActionType.CREATE,
+        columnName: `${richTextField.name}V2Markdown`,
+        columnType: 'text',
+        isNullable: true,
+        defaultValue: null,
+      },
+    ] as const;
+
+    const shouldForceCreateColumns =
+      this.options.force && fieldMetadataAlreadyExisting;
+
+    if (shouldForceCreateColumns) {
+      this.logger.warn(
+        `Force creating V2 columns for workspaceId: ${workspaceId} objectMetadaId: ${objectMetadata.id}`,
+      );
+    }
+    const shouldCreateColumns =
+      !fieldMetadataAlreadyExisting || shouldForceCreateColumns;
+
+    if (!this.options.dryRun && shouldCreateColumns) {
+      await this.workspaceMigrationService.createCustomMigration(
+        generateMigrationName(
+          `migrate-rich-text-field-${objectMetadata.nameSingular}-${richTextField.name}`,
+        ),
+        workspaceId,
+        [
+          {
+            name: computeObjectTargetTable(objectMetadata),
+            action: WorkspaceMigrationTableActionType.ALTER,
+            columns: columnsToCreate,
+          } satisfies WorkspaceMigrationTableAction,
+        ],
+      );
+    }
+
+    return shouldCreateColumns;
+  }
+
+  private async createIfMissingNewRichTextFieldsColumn({
+    richTextFields,
+    workspaceId,
+  }: ProcessRichTextFieldsArgs): Promise<
+    RichTextFieldWithHasCreatedColumnsAndObjectMetadata[]
+  > {
+    const richTextFieldsWithHasCreatedColumns: RichTextFieldWithHasCreatedColumnsAndObjectMetadata[] =
+      [];
 
     for (const richTextField of richTextFields) {
+      const standardId = this.buildRichTextFieldStandardId(richTextField);
+
+      const newRichTextField: Partial<FieldMetadataEntity> = {
+        ...richTextField,
+        name: `${richTextField.name}V2`,
+        id: undefined,
+        type: FieldMetadataType.RICH_TEXT_V2,
+        defaultValue: null,
+        standardId,
+        workspaceId,
+      };
+
+      const existingFieldMetadata =
+        await this.fieldMetadataRepository.findOneBy({
+          name: newRichTextField.name,
+          type: newRichTextField.type,
+          standardId: newRichTextField.standardId ?? undefined,
+          workspaceId,
+        });
+      const fieldMetadataAlreadyExisting = isDefined(existingFieldMetadata);
+
+      if (fieldMetadataAlreadyExisting) {
+        this.logger.warn(
+          `FieldMetadata already exists in fieldMetadataRepository name: ${newRichTextField.name} standardId: ${newRichTextField.standardId} type: ${newRichTextField.type} workspaceId: ${workspaceId}`,
+        );
+      }
+
+      if (!this.options.dryRun && !fieldMetadataAlreadyExisting) {
+        await this.fieldMetadataRepository.insert(newRichTextField);
+      }
+
       const objectMetadata = await this.objectMetadataRepository.findOne({
         where: { id: richTextField.objectMetadataId },
+        relations: {
+          fields: true,
+        },
       });
 
+      if (objectMetadata === null) {
+        this.logger.warn(
+          `Object metadata not found for rich text field ${richTextField.name} in workspace ${workspaceId}`,
+        );
+        richTextFieldsWithHasCreatedColumns.push({
+          hasCreatedColumns: false,
+          richTextField,
+          objectMetadata,
+        });
+        continue;
+      }
+
+      const hasCreatedColumns = await this.createMarkdownBlockNoteV2Columns({
+        objectMetadata,
+        richTextField,
+        workspaceId,
+        fieldMetadataAlreadyExisting,
+      });
+
+      richTextFieldsWithHasCreatedColumns.push({
+        hasCreatedColumns,
+        richTextField,
+        objectMetadata,
+      });
+    }
+
+    const hasAtLeastOnePendingMigration =
+      richTextFieldsWithHasCreatedColumns.some(
+        ({ hasCreatedColumns }) => hasCreatedColumns,
+      );
+
+    if (!this.options.dryRun && hasAtLeastOnePendingMigration) {
+      await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
+        workspaceId,
+      );
+    }
+
+    return richTextFieldsWithHasCreatedColumns;
+  }
+
+  private jsonParseOrSilentlyFail(input: string): null | unknown {
+    try {
+      return JSON.parse(input);
+    } catch (e) {
+      return null;
+    }
+  }
+
+  private async getMardownFieldValue({
+    blocknoteFieldValue,
+    serverBlockNoteEditor,
+  }: {
+    blocknoteFieldValue: string | null;
+    serverBlockNoteEditor: ServerBlockNoteEditor;
+  }): Promise<string | null> {
+    const blocknoteFieldValueIsDefined =
+      blocknoteFieldValue !== null &&
+      blocknoteFieldValue !== undefined &&
+      blocknoteFieldValue !== '{}';
+
+    if (!blocknoteFieldValueIsDefined) {
+      return null;
+    }
+
+    const jsonParsedblocknoteFieldValue =
+      this.jsonParseOrSilentlyFail(blocknoteFieldValue);
+
+    if (jsonParsedblocknoteFieldValue === null) {
+      return null;
+    }
+
+    if (!Array.isArray(jsonParsedblocknoteFieldValue)) {
+      this.logger.warn(
+        `blocknoteFieldValue is defined and is not an array got ${blocknoteFieldValue}`,
+      );
+
+      return null;
+    }
+
+    return await serverBlockNoteEditor.blocksToMarkdownLossy(
+      jsonParsedblocknoteFieldValue,
+    );
+  }
+
+  private async migrateToNewRichTextFieldsColumn({
+    richTextFieldsWithHasCreatedColumns,
+    workspaceId,
+  }: MigrateRichTextContentArgs) {
+    const serverBlockNoteEditor = ServerBlockNoteEditor.create();
+
+    for (const {
+      richTextField,
+      hasCreatedColumns,
+      objectMetadata,
+    } of richTextFieldsWithHasCreatedColumns) {
       if (objectMetadata === null) {
         this.logger.log(
           `Object metadata not found for rich text field ${richTextField.name} in workspace ${workspaceId}`,
@@ -254,16 +437,22 @@ export class MigrateRichTextFieldCommand extends ActiveWorkspacesCommandRunner {
 
       for (const row of rows) {
         const blocknoteFieldValue = row[richTextField.name];
-        const markdownFieldValue = blocknoteFieldValue
+        const markdownFieldValue = await this.getMardownFieldValue({
-          ? await serverBlockNoteEditor.blocksToMarkdownLossy(
+          blocknoteFieldValue,
-              JSON.parse(blocknoteFieldValue),
+          serverBlockNoteEditor,
-            )
+        });
-          : null;
 
-        await workspaceDataSource.query(
+        if (this.options.force) {
-          `UPDATE "${schemaName}"."${computeTableName(objectMetadata.nameSingular, objectMetadata.isCustom)}" SET "${richTextField.name}V2Blocknote" = $1, "${richTextField.name}V2Markdown" = $2 WHERE id = $3`,
+          this.logger.warn(
-          [blocknoteFieldValue, markdownFieldValue, row.id],
+            `Force udpate rowId: ${row.id} RICH_TEXT_FIELD ${richTextField.id} objectMetadata ${objectMetadata.id}`,
-        );
+          );
+        }
+        if (!this.options.dryRun && (hasCreatedColumns || this.options.force)) {
+          await workspaceDataSource.query(
+            `UPDATE "${schemaName}"."${computeTableName(objectMetadata.nameSingular, objectMetadata.isCustom)}" SET "${richTextField.name}V2Blocknote" = $1, "${richTextField.name}V2Markdown" = $2 WHERE id = $3`,
+            [blocknoteFieldValue, markdownFieldValue, row.id],
+          );
+        }
       }
     }
   }

packages/twenty-server/src/database/commands/upgrade-version/0-42/0-42-upgrade-version.command.ts

@@ -1,6 +1,6 @@
 import { InjectRepository } from '@nestjs/typeorm';
 
-import { Command } from 'nest-commander';
+import { Command, Option } from 'nest-commander';
 import { Repository } from 'typeorm';
 
 import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
@@ -11,6 +11,11 @@ import { MigrateRichTextFieldCommand } from 'src/database/commands/upgrade-versi
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
 
+type Upgrade042CommandCustomOptions = {
+  force: boolean;
+};
+export type Upgrade042CommandOptions = BaseCommandOptions &
+  Upgrade042CommandCustomOptions;
 @Command({
   name: 'upgrade-0.42',
   description: 'Upgrade to 0.42',
@@ -27,9 +32,19 @@ export class UpgradeTo0_42Command extends ActiveWorkspacesCommandRunner {
     super(workspaceRepository);
   }
 
+  @Option({
+    flags: '-f, --force [boolean]',
+    description:
+      'Force RICH_TEXT_FIELD value update even if column migration has already be run',
+    required: false,
+  })
+  parseForceValue(val?: boolean): boolean {
+    return val ?? false;
+  }
+
   async executeActiveWorkspacesCommand(
     passedParam: string[],
-    options: BaseCommandOptions,
+    options: Upgrade042CommandOptions,
     workspaceIds: string[],
   ): Promise<void> {
     this.logger.log('Running command to upgrade to 0.42');