Create view migration script (#13356)

Create view migration command to copy views from the workspace schema to
the core schema.
Closes https://github.com/twentyhq/core-team-issues/issues/1247

packages/twenty-server/src/database/commands/database-command.module.ts

@@ -1,14 +1,24 @@
 import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
 
 import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.command';
 import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
 import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
+import { MigrateViewsToCoreCommand } from 'src/database/commands/views-migration/migrate-views-to-core.command';
 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
+import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
 import { FileModule } from 'src/engine/core-modules/file/file.module';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
 import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
+import { ViewField } from 'src/engine/metadata-modules/view/view-field.entity';
+import { ViewFilterGroup } from 'src/engine/metadata-modules/view/view-filter-group.entity';
+import { ViewFilter } from 'src/engine/metadata-modules/view/view-filter.entity';
+import { ViewGroup } from 'src/engine/metadata-modules/view/view-group.entity';
+import { ViewSort } from 'src/engine/metadata-modules/view/view-sort.entity';
+import { View } from 'src/engine/metadata-modules/view/view.entity';
 import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
 import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module';
 import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
@@ -22,6 +32,19 @@ import { DataSeedWorkspaceCommand } from './data-seed-dev-workspace.command';
   imports: [
     UpgradeVersionCommandModule,
 
+    TypeOrmModule.forFeature(
+      [
+        Workspace,
+        View,
+        ViewField,
+        ViewFilter,
+        ViewSort,
+        ViewGroup,
+        ViewFilterGroup,
+      ],
+      'core',
+    ),
+
     // Cron command dependencies
     MessagingImportManagerModule,
     CalendarEventImportManagerModule,
@@ -37,9 +60,11 @@ import { DataSeedWorkspaceCommand } from './data-seed-dev-workspace.command';
     DataSourceModule,
     WorkspaceCacheStorageModule,
     ApiKeyModule,
+    FeatureFlagModule,
   ],
   providers: [
     DataSeedWorkspaceCommand,
+    MigrateViewsToCoreCommand,
     ConfirmationQuestion,
     CronRegisterAllCommand,
   ],

packages/twenty-server/src/database/commands/views-migration/migrate-views-to-core.command.ts

@@ -0,0 +1,445 @@
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { DataSource, QueryRunner, Repository } from 'typeorm';
+
+import {
+  ActiveOrSuspendedWorkspacesMigrationCommandRunner,
+  RunOnWorkspaceArgs,
+} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
+import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ViewFilterGroupLogicalOperator } from 'src/engine/metadata-modules/view/enums/view-filter-group-logical-operator';
+import { ViewOpenRecordIn } from 'src/engine/metadata-modules/view/enums/view-open-record-in';
+import { ViewSortDirection } from 'src/engine/metadata-modules/view/enums/view-sort-direction';
+import { ViewField } from 'src/engine/metadata-modules/view/view-field.entity';
+import { ViewFilterGroup } from 'src/engine/metadata-modules/view/view-filter-group.entity';
+import { ViewFilter } from 'src/engine/metadata-modules/view/view-filter.entity';
+import { ViewGroup } from 'src/engine/metadata-modules/view/view-group.entity';
+import { ViewSort } from 'src/engine/metadata-modules/view/view-sort.entity';
+import { View } from 'src/engine/metadata-modules/view/view.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { ViewFieldWorkspaceEntity } from 'src/modules/view/standard-objects/view-field.workspace-entity';
+import { ViewFilterGroupWorkspaceEntity } from 'src/modules/view/standard-objects/view-filter-group.workspace-entity';
+import { ViewFilterWorkspaceEntity } from 'src/modules/view/standard-objects/view-filter.workspace-entity';
+import { ViewGroupWorkspaceEntity } from 'src/modules/view/standard-objects/view-group.workspace-entity';
+import { ViewSortWorkspaceEntity } from 'src/modules/view/standard-objects/view-sort.workspace-entity';
+import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity';
+
+@Command({
+  name: 'migrate:views-to-core',
+  description:
+    'Migrate views from workspace schemas to core schema and enable IS_CORE_VIEW_SYNCING_ENABLED feature flag',
+})
+export class MigrateViewsToCoreCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(Workspace, 'core')
+    protected readonly workspaceRepository: Repository<Workspace>,
+    @InjectRepository(View, 'core')
+    private readonly coreViewRepository: Repository<View>,
+    @InjectRepository(ViewField, 'core')
+    private readonly coreViewFieldRepository: Repository<ViewField>,
+    @InjectRepository(ViewFilter, 'core')
+    private readonly coreViewFilterRepository: Repository<ViewFilter>,
+    @InjectRepository(ViewSort, 'core')
+    private readonly coreViewSortRepository: Repository<ViewSort>,
+    @InjectRepository(ViewGroup, 'core')
+    private readonly coreViewGroupRepository: Repository<ViewGroup>,
+    @InjectRepository(ViewFilterGroup, 'core')
+    private readonly coreViewFilterGroupRepository: Repository<ViewFilterGroup>,
+    private readonly featureFlagService: FeatureFlagService,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    @InjectDataSource('core')
+    private readonly coreDataSource: DataSource,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  override async runOnWorkspace({
+    index,
+    total,
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    this.logger.log(
+      `Migrating views to core schema for workspace ${workspaceId} ${index + 1}/${total}`,
+    );
+
+    const queryRunner = this.coreDataSource.createQueryRunner();
+
+    await queryRunner.connect();
+
+    try {
+      const featureFlags =
+        await this.featureFlagService.getWorkspaceFeatureFlagsMap(workspaceId);
+
+      if (featureFlags?.IS_CORE_VIEW_SYNCING_ENABLED) {
+        this.logger.log(
+          `Workspace ${workspaceId} already has IS_CORE_VIEW_SYNCING_ENABLED feature flag, skipping migration`,
+        );
+
+        return;
+      }
+
+      await queryRunner.startTransaction();
+
+      try {
+        await this.migrateViews(
+          workspaceId,
+          options.dryRun ?? false,
+          queryRunner,
+        );
+
+        if (options.dryRun) {
+          this.logger.log(
+            `DRY RUN: Would enable IS_CORE_VIEW_SYNCING_ENABLED feature flag for workspace ${workspaceId}`,
+          );
+        } else {
+          await queryRunner.commitTransaction();
+          await this.enableCoreViewSyncingFeatureFlag(workspaceId);
+          this.logger.log(
+            `Successfully migrated views to core schema for workspace ${workspaceId}`,
+          );
+        }
+      } catch (error) {
+        if (queryRunner.isTransactionActive) {
+          await queryRunner.rollbackTransaction();
+          this.logger.error(
+            `Transaction rolled back for workspace ${workspaceId} due to error: ${error.message}`,
+          );
+        }
+
+        throw error;
+      }
+    } catch (error) {
+      this.logger.error(
+        `Failed to migrate views to core schema for workspace ${workspaceId}: ${error.message}`,
+      );
+
+      throw error;
+    } finally {
+      await queryRunner.release();
+    }
+  }
+
+  private async migrateViews(
+    workspaceId: string,
+    dryRun: boolean,
+    queryRunner: QueryRunner,
+  ): Promise<void> {
+    const workspaceViewRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<ViewWorkspaceEntity>(
+        workspaceId,
+        'view',
+        { shouldBypassPermissionChecks: true },
+      );
+
+    const workspaceViews = await workspaceViewRepository.find({
+      relations: [
+        'viewFields',
+        'viewFilters',
+        'viewSorts',
+        'viewGroups',
+        'viewFilterGroups',
+      ],
+      withDeleted: true,
+    });
+
+    if (workspaceViews.length === 0) {
+      this.logger.log(`No views to migrate for workspace ${workspaceId}`);
+
+      return;
+    }
+
+    this.logger.log(
+      `${dryRun ? 'DRY RUN: ' : ''}Found ${workspaceViews.length} views to migrate for workspace ${workspaceId}`,
+    );
+
+    if (dryRun) {
+      for (const view of workspaceViews) {
+        const deletedStatus = view.deletedAt ? ' (DELETED)' : '';
+
+        this.logger.log(
+          `DRY RUN: Would migrate view ${view.id} (${view.name})${deletedStatus} with ${view.viewFields.length} fields, ${view.viewFilters.length} filters, ${view.viewSorts.length} sorts, ${view.viewGroups.length} groups, ${view.viewFilterGroups.length} filter groups`,
+        );
+      }
+
+      return;
+    }
+
+    const viewRepository = queryRunner.manager.getRepository(View);
+
+    const existingCoreViews = await viewRepository.find({
+      where: { workspaceId },
+      select: ['id'],
+      withDeleted: true,
+    });
+
+    const existingViewIds = new Set(existingCoreViews.map((view) => view.id));
+
+    for (const workspaceView of workspaceViews) {
+      if (existingViewIds.has(workspaceView.id)) {
+        this.logger.warn(
+          `View ${workspaceView.id} already exists in core schema for workspace ${workspaceId}, skipping`,
+        );
+
+        continue;
+      }
+
+      await this.migrateViewEntity(workspaceView, workspaceId, queryRunner);
+
+      if (workspaceView.viewFields?.length > 0) {
+        await this.migrateViewFields(
+          workspaceView.viewFields,
+          workspaceId,
+          queryRunner,
+        );
+      }
+
+      if (workspaceView.viewFilters?.length > 0) {
+        await this.migrateViewFilters(
+          workspaceView.viewFilters,
+          workspaceId,
+          queryRunner,
+        );
+      }
+
+      if (workspaceView.viewSorts?.length > 0) {
+        await this.migrateViewSorts(
+          workspaceView.viewSorts,
+          workspaceId,
+          queryRunner,
+        );
+      }
+
+      if (workspaceView.viewGroups?.length > 0) {
+        await this.migrateViewGroups(
+          workspaceView.viewGroups,
+          workspaceId,
+          queryRunner,
+        );
+      }
+
+      if (workspaceView.viewFilterGroups?.length > 0) {
+        await this.migrateViewFilterGroups(
+          workspaceView.viewFilterGroups,
+          workspaceId,
+          queryRunner,
+        );
+      }
+
+      const deletedStatus = workspaceView.deletedAt ? ' (DELETED)' : '';
+
+      this.logger.log(
+        `Migrated view ${workspaceView.id} (${workspaceView.name})${deletedStatus} to core schema`,
+      );
+    }
+  }
+
+  private async migrateViewEntity(
+    workspaceView: ViewWorkspaceEntity,
+    workspaceId: string,
+    queryRunner: QueryRunner,
+  ): Promise<void> {
+    const coreView = {
+      id: workspaceView.id,
+      name: workspaceView.name,
+      objectMetadataId: workspaceView.objectMetadataId,
+      type: workspaceView.type,
+      key: workspaceView.key,
+      icon: workspaceView.icon,
+      position: workspaceView.position,
+      isCompact: workspaceView.isCompact,
+      openRecordIn:
+        workspaceView.openRecordIn === 'SIDE_PANEL'
+          ? ViewOpenRecordIn.SIDE_PANEL
+          : ViewOpenRecordIn.RECORD_PAGE,
+      kanbanAggregateOperation: workspaceView.kanbanAggregateOperation,
+      kanbanAggregateOperationFieldMetadataId:
+        workspaceView.kanbanAggregateOperationFieldMetadataId,
+      workspaceId,
+      createdAt: new Date(workspaceView.createdAt),
+      updatedAt: new Date(workspaceView.updatedAt),
+      deletedAt: workspaceView.deletedAt
+        ? new Date(workspaceView.deletedAt)
+        : null,
+    };
+
+    const repository = queryRunner.manager.getRepository(View);
+
+    await repository.insert(coreView);
+  }
+
+  private async migrateViewFields(
+    workspaceViewFields: ViewFieldWorkspaceEntity[],
+    workspaceId: string,
+    queryRunner: QueryRunner,
+  ): Promise<void> {
+    for (const field of workspaceViewFields) {
+      const coreViewField = {
+        id: field.id,
+        fieldMetadataId: field.fieldMetadataId,
+        viewId: field.viewId,
+        position: field.position,
+        isVisible: field.isVisible,
+        size: field.size,
+        workspaceId,
+        createdAt: new Date(field.createdAt),
+        updatedAt: new Date(field.updatedAt),
+        deletedAt: field.deletedAt ? new Date(field.deletedAt) : null,
+      };
+
+      const repository = queryRunner.manager.getRepository(ViewField);
+
+      await repository.insert(coreViewField);
+    }
+  }
+
+  private async migrateViewFilters(
+    workspaceViewFilters: ViewFilterWorkspaceEntity[],
+    workspaceId: string,
+    queryRunner: QueryRunner,
+  ): Promise<void> {
+    for (const filter of workspaceViewFilters) {
+      if (!filter.viewId) {
+        this.logger.warn(
+          `Skipping view filter ${filter.id} with null viewId for workspace ${workspaceId}`,
+        );
+        continue;
+      }
+
+      let parsedValue: JSON;
+
+      try {
+        parsedValue = JSON.parse(filter.value);
+      } catch {
+        throw new Error(
+          `Could not parse value to JSON for view filter ${filter.id} for workspace ${workspaceId}`,
+        );
+      }
+
+      const coreViewFilter = {
+        id: filter.id,
+        fieldMetadataId: filter.fieldMetadataId,
+        viewId: filter.viewId,
+        operand: filter.operand,
+        value: parsedValue,
+        displayValue: filter.displayValue,
+        viewFilterGroupId: filter.viewFilterGroupId,
+        workspaceId,
+        createdAt: new Date(filter.createdAt),
+        updatedAt: new Date(filter.updatedAt),
+        deletedAt: filter.deletedAt ? new Date(filter.deletedAt) : null,
+      };
+
+      const repository = queryRunner.manager.getRepository(ViewFilter);
+
+      await repository.insert(coreViewFilter);
+    }
+  }
+
+  private async migrateViewSorts(
+    workspaceViewSorts: ViewSortWorkspaceEntity[],
+    workspaceId: string,
+    queryRunner: QueryRunner,
+  ): Promise<void> {
+    for (const sort of workspaceViewSorts) {
+      if (!sort.viewId) {
+        this.logger.warn(
+          `Skipping view sort ${sort.id} with null viewId for workspace ${workspaceId}`,
+        );
+        continue;
+      }
+
+      const direction = sort.direction.toUpperCase() as ViewSortDirection;
+
+      const coreViewSort = {
+        id: sort.id,
+        fieldMetadataId: sort.fieldMetadataId,
+        viewId: sort.viewId,
+        direction: direction,
+        workspaceId,
+        createdAt: new Date(sort.createdAt),
+        updatedAt: new Date(sort.updatedAt),
+        deletedAt: sort.deletedAt ? new Date(sort.deletedAt) : null,
+      };
+
+      const repository = queryRunner.manager.getRepository(ViewSort);
+
+      await repository.insert(coreViewSort);
+    }
+  }
+
+  private async migrateViewGroups(
+    workspaceViewGroups: ViewGroupWorkspaceEntity[],
+    workspaceId: string,
+    queryRunner: QueryRunner,
+  ): Promise<void> {
+    for (const group of workspaceViewGroups) {
+      if (!group.viewId) {
+        this.logger.warn(
+          `Skipping view group ${group.id} with null viewId for workspace ${workspaceId}`,
+        );
+        continue;
+      }
+
+      const coreViewGroup = {
+        id: group.id,
+        fieldMetadataId: group.fieldMetadataId,
+        viewId: group.viewId,
+        fieldValue: group.fieldValue,
+        isVisible: group.isVisible,
+        position: group.position,
+        workspaceId,
+        createdAt: new Date(group.createdAt),
+        updatedAt: new Date(group.updatedAt),
+        deletedAt: group.deletedAt ? new Date(group.deletedAt) : null,
+      };
+
+      const repository = queryRunner.manager.getRepository(ViewGroup);
+
+      await repository.insert(coreViewGroup);
+    }
+  }
+
+  private async migrateViewFilterGroups(
+    workspaceViewFilterGroups: ViewFilterGroupWorkspaceEntity[],
+    workspaceId: string,
+    queryRunner: QueryRunner,
+  ): Promise<void> {
+    for (const filterGroup of workspaceViewFilterGroups) {
+      const coreViewFilterGroup = {
+        id: filterGroup.id,
+        viewId: filterGroup.viewId,
+        logicalOperator:
+          filterGroup.logicalOperator as ViewFilterGroupLogicalOperator,
+        parentViewFilterGroupId: filterGroup.parentViewFilterGroupId,
+        positionInViewFilterGroup: filterGroup.positionInViewFilterGroup,
+        workspaceId,
+        createdAt: new Date(filterGroup.createdAt),
+        updatedAt: new Date(filterGroup.updatedAt),
+        deletedAt: filterGroup.deletedAt
+          ? new Date(filterGroup.deletedAt)
+          : null,
+      };
+
+      const repository = queryRunner.manager.getRepository(ViewFilterGroup);
+
+      await repository.insert(coreViewFilterGroup);
+    }
+  }
+
+  private async enableCoreViewSyncingFeatureFlag(
+    workspaceId: string,
+  ): Promise<void> {
+    await this.featureFlagService.enableFeatureFlags(
+      [FeatureFlagKey.IS_CORE_VIEW_SYNCING_ENABLED],
+      workspaceId,
+    );
+
+    this.logger.log(
+      `Enabled IS_CORE_VIEW_SYNCING_ENABLED feature flag for workspace ${workspaceId}`,
+    );
+  }
+}

packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts

@@ -12,5 +12,6 @@ export enum FeatureFlagKey {
   IS_WORKSPACE_API_KEY_WEBHOOK_GRAPHQL_ENABLED = 'IS_WORKSPACE_API_KEY_WEBHOOK_GRAPHQL_ENABLED',
   IS_FIELDS_PERMISSIONS_ENABLED = 'IS_FIELDS_PERMISSIONS_ENABLED',
   IS_ANY_FIELD_SEARCH_ENABLED = 'IS_ANY_FIELD_SEARCH_ENABLED',
+  IS_CORE_VIEW_SYNCING_ENABLED = 'IS_CORE_VIEW_SYNCING_ENABLED',
   IS_TWO_FACTOR_AUTHENTICATION_ENABLED = 'IS_TWO_FACTOR_AUTHENTICATION_ENABLED',
 }

packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts

@@ -5,9 +5,9 @@ import { PlainObjectToDatabaseEntityTransformer } from 'typeorm/query-builder/tr
 
 import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface';
 
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
 import { validateOperationIsPermittedOrThrow } from 'src/engine/twenty-orm/repository/permissions.utils';
-import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 
 import { WorkspaceEntityManager } from './workspace-entity-manager';
 
@@ -109,6 +109,7 @@ describe('WorkspaceEntityManager', () => {
         IS_WORKSPACE_API_KEY_WEBHOOK_GRAPHQL_ENABLED: false,
         IS_FIELDS_PERMISSIONS_ENABLED: false,
         IS_ANY_FIELD_SEARCH_ENABLED: false,
+        IS_CORE_VIEW_SYNCING_ENABLED: false,
         IS_TWO_FACTOR_AUTHENTICATION_ENABLED: false,
       },
       eventEmitterService: {