diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index 09acbc456..f593e716a 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -740,6 +740,7 @@ export enum FeatureFlagKey { IS_AIRTABLE_INTEGRATION_ENABLED = 'IS_AIRTABLE_INTEGRATION_ENABLED', IS_AI_ENABLED = 'IS_AI_ENABLED', IS_ANY_FIELD_SEARCH_ENABLED = 'IS_ANY_FIELD_SEARCH_ENABLED', + IS_CORE_VIEW_SYNCING_ENABLED = 'IS_CORE_VIEW_SYNCING_ENABLED', IS_FIELDS_PERMISSIONS_ENABLED = 'IS_FIELDS_PERMISSIONS_ENABLED', IS_IMAP_SMTP_CALDAV_ENABLED = 'IS_IMAP_SMTP_CALDAV_ENABLED', IS_JSON_FILTER_ENABLED = 'IS_JSON_FILTER_ENABLED', diff --git a/packages/twenty-front/src/generated/graphql.ts b/packages/twenty-front/src/generated/graphql.ts index 4799279eb..95477b938 100644 --- a/packages/twenty-front/src/generated/graphql.ts +++ b/packages/twenty-front/src/generated/graphql.ts @@ -704,6 +704,7 @@ export enum FeatureFlagKey { IS_AIRTABLE_INTEGRATION_ENABLED = 'IS_AIRTABLE_INTEGRATION_ENABLED', IS_AI_ENABLED = 'IS_AI_ENABLED', IS_ANY_FIELD_SEARCH_ENABLED = 'IS_ANY_FIELD_SEARCH_ENABLED', + IS_CORE_VIEW_SYNCING_ENABLED = 'IS_CORE_VIEW_SYNCING_ENABLED', IS_FIELDS_PERMISSIONS_ENABLED = 'IS_FIELDS_PERMISSIONS_ENABLED', IS_IMAP_SMTP_CALDAV_ENABLED = 'IS_IMAP_SMTP_CALDAV_ENABLED', IS_JSON_FILTER_ENABLED = 'IS_JSON_FILTER_ENABLED', diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index 0303af2d5..5831315e2 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -1,14 +1,24 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.command'; import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module'; +import { MigrateViewsToCoreCommand } from 'src/database/commands/views-migration/migrate-views-to-core.command'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module'; +import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { FileModule } from 'src/engine/core-modules/file/file.module'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; +import { ViewField } from 'src/engine/metadata-modules/view/view-field.entity'; +import { ViewFilterGroup } from 'src/engine/metadata-modules/view/view-filter-group.entity'; +import { ViewFilter } from 'src/engine/metadata-modules/view/view-filter.entity'; +import { ViewGroup } from 'src/engine/metadata-modules/view/view-group.entity'; +import { ViewSort } from 'src/engine/metadata-modules/view/view-sort.entity'; +import { View } from 'src/engine/metadata-modules/view/view.entity'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { DevSeederModule } from 'src/engine/workspace-manager/dev-seeder/dev-seeder.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; @@ -22,6 +32,19 @@ import { DataSeedWorkspaceCommand } from './data-seed-dev-workspace.command'; imports: [ UpgradeVersionCommandModule, + TypeOrmModule.forFeature( + [ + Workspace, + View, + ViewField, + ViewFilter, + ViewSort, + ViewGroup, + ViewFilterGroup, + ], + 'core', + ), + // Cron command dependencies MessagingImportManagerModule, CalendarEventImportManagerModule, @@ -37,9 +60,11 @@ import { DataSeedWorkspaceCommand } from './data-seed-dev-workspace.command'; DataSourceModule, WorkspaceCacheStorageModule, ApiKeyModule, + FeatureFlagModule, ], providers: [ DataSeedWorkspaceCommand, + MigrateViewsToCoreCommand, ConfirmationQuestion, CronRegisterAllCommand, ], diff --git a/packages/twenty-server/src/database/commands/views-migration/migrate-views-to-core.command.ts b/packages/twenty-server/src/database/commands/views-migration/migrate-views-to-core.command.ts new file mode 100644 index 000000000..4e729bfcc --- /dev/null +++ b/packages/twenty-server/src/database/commands/views-migration/migrate-views-to-core.command.ts @@ -0,0 +1,445 @@ +import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { DataSource, QueryRunner, Repository } from 'typeorm'; + +import { + ActiveOrSuspendedWorkspacesMigrationCommandRunner, + RunOnWorkspaceArgs, +} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; +import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { ViewFilterGroupLogicalOperator } from 'src/engine/metadata-modules/view/enums/view-filter-group-logical-operator'; +import { ViewOpenRecordIn } from 'src/engine/metadata-modules/view/enums/view-open-record-in'; +import { ViewSortDirection } from 'src/engine/metadata-modules/view/enums/view-sort-direction'; +import { ViewField } from 'src/engine/metadata-modules/view/view-field.entity'; +import { ViewFilterGroup } from 'src/engine/metadata-modules/view/view-filter-group.entity'; +import { ViewFilter } from 'src/engine/metadata-modules/view/view-filter.entity'; +import { ViewGroup } from 'src/engine/metadata-modules/view/view-group.entity'; +import { ViewSort } from 'src/engine/metadata-modules/view/view-sort.entity'; +import { View } from 'src/engine/metadata-modules/view/view.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { ViewFieldWorkspaceEntity } from 'src/modules/view/standard-objects/view-field.workspace-entity'; +import { ViewFilterGroupWorkspaceEntity } from 'src/modules/view/standard-objects/view-filter-group.workspace-entity'; +import { ViewFilterWorkspaceEntity } from 'src/modules/view/standard-objects/view-filter.workspace-entity'; +import { ViewGroupWorkspaceEntity } from 'src/modules/view/standard-objects/view-group.workspace-entity'; +import { ViewSortWorkspaceEntity } from 'src/modules/view/standard-objects/view-sort.workspace-entity'; +import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; + +@Command({ + name: 'migrate:views-to-core', + description: + 'Migrate views from workspace schemas to core schema and enable IS_CORE_VIEW_SYNCING_ENABLED feature flag', +}) +export class MigrateViewsToCoreCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + @InjectRepository(View, 'core') + private readonly coreViewRepository: Repository, + @InjectRepository(ViewField, 'core') + private readonly coreViewFieldRepository: Repository, + @InjectRepository(ViewFilter, 'core') + private readonly coreViewFilterRepository: Repository, + @InjectRepository(ViewSort, 'core') + private readonly coreViewSortRepository: Repository, + @InjectRepository(ViewGroup, 'core') + private readonly coreViewGroupRepository: Repository, + @InjectRepository(ViewFilterGroup, 'core') + private readonly coreViewFilterGroupRepository: Repository, + private readonly featureFlagService: FeatureFlagService, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + @InjectDataSource('core') + private readonly coreDataSource: DataSource, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + override async runOnWorkspace({ + index, + total, + workspaceId, + options, + }: RunOnWorkspaceArgs): Promise { + this.logger.log( + `Migrating views to core schema for workspace ${workspaceId} ${index + 1}/${total}`, + ); + + const queryRunner = this.coreDataSource.createQueryRunner(); + + await queryRunner.connect(); + + try { + const featureFlags = + await this.featureFlagService.getWorkspaceFeatureFlagsMap(workspaceId); + + if (featureFlags?.IS_CORE_VIEW_SYNCING_ENABLED) { + this.logger.log( + `Workspace ${workspaceId} already has IS_CORE_VIEW_SYNCING_ENABLED feature flag, skipping migration`, + ); + + return; + } + + await queryRunner.startTransaction(); + + try { + await this.migrateViews( + workspaceId, + options.dryRun ?? false, + queryRunner, + ); + + if (options.dryRun) { + this.logger.log( + `DRY RUN: Would enable IS_CORE_VIEW_SYNCING_ENABLED feature flag for workspace ${workspaceId}`, + ); + } else { + await queryRunner.commitTransaction(); + await this.enableCoreViewSyncingFeatureFlag(workspaceId); + this.logger.log( + `Successfully migrated views to core schema for workspace ${workspaceId}`, + ); + } + } catch (error) { + if (queryRunner.isTransactionActive) { + await queryRunner.rollbackTransaction(); + this.logger.error( + `Transaction rolled back for workspace ${workspaceId} due to error: ${error.message}`, + ); + } + + throw error; + } + } catch (error) { + this.logger.error( + `Failed to migrate views to core schema for workspace ${workspaceId}: ${error.message}`, + ); + + throw error; + } finally { + await queryRunner.release(); + } + } + + private async migrateViews( + workspaceId: string, + dryRun: boolean, + queryRunner: QueryRunner, + ): Promise { + const workspaceViewRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'view', + { shouldBypassPermissionChecks: true }, + ); + + const workspaceViews = await workspaceViewRepository.find({ + relations: [ + 'viewFields', + 'viewFilters', + 'viewSorts', + 'viewGroups', + 'viewFilterGroups', + ], + withDeleted: true, + }); + + if (workspaceViews.length === 0) { + this.logger.log(`No views to migrate for workspace ${workspaceId}`); + + return; + } + + this.logger.log( + `${dryRun ? 'DRY RUN: ' : ''}Found ${workspaceViews.length} views to migrate for workspace ${workspaceId}`, + ); + + if (dryRun) { + for (const view of workspaceViews) { + const deletedStatus = view.deletedAt ? ' (DELETED)' : ''; + + this.logger.log( + `DRY RUN: Would migrate view ${view.id} (${view.name})${deletedStatus} with ${view.viewFields.length} fields, ${view.viewFilters.length} filters, ${view.viewSorts.length} sorts, ${view.viewGroups.length} groups, ${view.viewFilterGroups.length} filter groups`, + ); + } + + return; + } + + const viewRepository = queryRunner.manager.getRepository(View); + + const existingCoreViews = await viewRepository.find({ + where: { workspaceId }, + select: ['id'], + withDeleted: true, + }); + + const existingViewIds = new Set(existingCoreViews.map((view) => view.id)); + + for (const workspaceView of workspaceViews) { + if (existingViewIds.has(workspaceView.id)) { + this.logger.warn( + `View ${workspaceView.id} already exists in core schema for workspace ${workspaceId}, skipping`, + ); + + continue; + } + + await this.migrateViewEntity(workspaceView, workspaceId, queryRunner); + + if (workspaceView.viewFields?.length > 0) { + await this.migrateViewFields( + workspaceView.viewFields, + workspaceId, + queryRunner, + ); + } + + if (workspaceView.viewFilters?.length > 0) { + await this.migrateViewFilters( + workspaceView.viewFilters, + workspaceId, + queryRunner, + ); + } + + if (workspaceView.viewSorts?.length > 0) { + await this.migrateViewSorts( + workspaceView.viewSorts, + workspaceId, + queryRunner, + ); + } + + if (workspaceView.viewGroups?.length > 0) { + await this.migrateViewGroups( + workspaceView.viewGroups, + workspaceId, + queryRunner, + ); + } + + if (workspaceView.viewFilterGroups?.length > 0) { + await this.migrateViewFilterGroups( + workspaceView.viewFilterGroups, + workspaceId, + queryRunner, + ); + } + + const deletedStatus = workspaceView.deletedAt ? ' (DELETED)' : ''; + + this.logger.log( + `Migrated view ${workspaceView.id} (${workspaceView.name})${deletedStatus} to core schema`, + ); + } + } + + private async migrateViewEntity( + workspaceView: ViewWorkspaceEntity, + workspaceId: string, + queryRunner: QueryRunner, + ): Promise { + const coreView = { + id: workspaceView.id, + name: workspaceView.name, + objectMetadataId: workspaceView.objectMetadataId, + type: workspaceView.type, + key: workspaceView.key, + icon: workspaceView.icon, + position: workspaceView.position, + isCompact: workspaceView.isCompact, + openRecordIn: + workspaceView.openRecordIn === 'SIDE_PANEL' + ? ViewOpenRecordIn.SIDE_PANEL + : ViewOpenRecordIn.RECORD_PAGE, + kanbanAggregateOperation: workspaceView.kanbanAggregateOperation, + kanbanAggregateOperationFieldMetadataId: + workspaceView.kanbanAggregateOperationFieldMetadataId, + workspaceId, + createdAt: new Date(workspaceView.createdAt), + updatedAt: new Date(workspaceView.updatedAt), + deletedAt: workspaceView.deletedAt + ? new Date(workspaceView.deletedAt) + : null, + }; + + const repository = queryRunner.manager.getRepository(View); + + await repository.insert(coreView); + } + + private async migrateViewFields( + workspaceViewFields: ViewFieldWorkspaceEntity[], + workspaceId: string, + queryRunner: QueryRunner, + ): Promise { + for (const field of workspaceViewFields) { + const coreViewField = { + id: field.id, + fieldMetadataId: field.fieldMetadataId, + viewId: field.viewId, + position: field.position, + isVisible: field.isVisible, + size: field.size, + workspaceId, + createdAt: new Date(field.createdAt), + updatedAt: new Date(field.updatedAt), + deletedAt: field.deletedAt ? new Date(field.deletedAt) : null, + }; + + const repository = queryRunner.manager.getRepository(ViewField); + + await repository.insert(coreViewField); + } + } + + private async migrateViewFilters( + workspaceViewFilters: ViewFilterWorkspaceEntity[], + workspaceId: string, + queryRunner: QueryRunner, + ): Promise { + for (const filter of workspaceViewFilters) { + if (!filter.viewId) { + this.logger.warn( + `Skipping view filter ${filter.id} with null viewId for workspace ${workspaceId}`, + ); + continue; + } + + let parsedValue: JSON; + + try { + parsedValue = JSON.parse(filter.value); + } catch { + throw new Error( + `Could not parse value to JSON for view filter ${filter.id} for workspace ${workspaceId}`, + ); + } + + const coreViewFilter = { + id: filter.id, + fieldMetadataId: filter.fieldMetadataId, + viewId: filter.viewId, + operand: filter.operand, + value: parsedValue, + displayValue: filter.displayValue, + viewFilterGroupId: filter.viewFilterGroupId, + workspaceId, + createdAt: new Date(filter.createdAt), + updatedAt: new Date(filter.updatedAt), + deletedAt: filter.deletedAt ? new Date(filter.deletedAt) : null, + }; + + const repository = queryRunner.manager.getRepository(ViewFilter); + + await repository.insert(coreViewFilter); + } + } + + private async migrateViewSorts( + workspaceViewSorts: ViewSortWorkspaceEntity[], + workspaceId: string, + queryRunner: QueryRunner, + ): Promise { + for (const sort of workspaceViewSorts) { + if (!sort.viewId) { + this.logger.warn( + `Skipping view sort ${sort.id} with null viewId for workspace ${workspaceId}`, + ); + continue; + } + + const direction = sort.direction.toUpperCase() as ViewSortDirection; + + const coreViewSort = { + id: sort.id, + fieldMetadataId: sort.fieldMetadataId, + viewId: sort.viewId, + direction: direction, + workspaceId, + createdAt: new Date(sort.createdAt), + updatedAt: new Date(sort.updatedAt), + deletedAt: sort.deletedAt ? new Date(sort.deletedAt) : null, + }; + + const repository = queryRunner.manager.getRepository(ViewSort); + + await repository.insert(coreViewSort); + } + } + + private async migrateViewGroups( + workspaceViewGroups: ViewGroupWorkspaceEntity[], + workspaceId: string, + queryRunner: QueryRunner, + ): Promise { + for (const group of workspaceViewGroups) { + if (!group.viewId) { + this.logger.warn( + `Skipping view group ${group.id} with null viewId for workspace ${workspaceId}`, + ); + continue; + } + + const coreViewGroup = { + id: group.id, + fieldMetadataId: group.fieldMetadataId, + viewId: group.viewId, + fieldValue: group.fieldValue, + isVisible: group.isVisible, + position: group.position, + workspaceId, + createdAt: new Date(group.createdAt), + updatedAt: new Date(group.updatedAt), + deletedAt: group.deletedAt ? new Date(group.deletedAt) : null, + }; + + const repository = queryRunner.manager.getRepository(ViewGroup); + + await repository.insert(coreViewGroup); + } + } + + private async migrateViewFilterGroups( + workspaceViewFilterGroups: ViewFilterGroupWorkspaceEntity[], + workspaceId: string, + queryRunner: QueryRunner, + ): Promise { + for (const filterGroup of workspaceViewFilterGroups) { + const coreViewFilterGroup = { + id: filterGroup.id, + viewId: filterGroup.viewId, + logicalOperator: + filterGroup.logicalOperator as ViewFilterGroupLogicalOperator, + parentViewFilterGroupId: filterGroup.parentViewFilterGroupId, + positionInViewFilterGroup: filterGroup.positionInViewFilterGroup, + workspaceId, + createdAt: new Date(filterGroup.createdAt), + updatedAt: new Date(filterGroup.updatedAt), + deletedAt: filterGroup.deletedAt + ? new Date(filterGroup.deletedAt) + : null, + }; + + const repository = queryRunner.manager.getRepository(ViewFilterGroup); + + await repository.insert(coreViewFilterGroup); + } + } + + private async enableCoreViewSyncingFeatureFlag( + workspaceId: string, + ): Promise { + await this.featureFlagService.enableFeatureFlags( + [FeatureFlagKey.IS_CORE_VIEW_SYNCING_ENABLED], + workspaceId, + ); + + this.logger.log( + `Enabled IS_CORE_VIEW_SYNCING_ENABLED feature flag for workspace ${workspaceId}`, + ); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts index 8c021eb15..bb1a7aeaf 100644 --- a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts @@ -12,5 +12,6 @@ export enum FeatureFlagKey { IS_WORKSPACE_API_KEY_WEBHOOK_GRAPHQL_ENABLED = 'IS_WORKSPACE_API_KEY_WEBHOOK_GRAPHQL_ENABLED', IS_FIELDS_PERMISSIONS_ENABLED = 'IS_FIELDS_PERMISSIONS_ENABLED', IS_ANY_FIELD_SEARCH_ENABLED = 'IS_ANY_FIELD_SEARCH_ENABLED', + IS_CORE_VIEW_SYNCING_ENABLED = 'IS_CORE_VIEW_SYNCING_ENABLED', IS_TWO_FACTOR_AUTHENTICATION_ENABLED = 'IS_TWO_FACTOR_AUTHENTICATION_ENABLED', } diff --git a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts index 181545445..65296508d 100644 --- a/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts +++ b/packages/twenty-server/src/engine/twenty-orm/entity-manager/workspace-entity-manager.spec.ts @@ -5,9 +5,9 @@ import { PlainObjectToDatabaseEntityTransformer } from 'typeorm/query-builder/tr import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface'; +import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; import { validateOperationIsPermittedOrThrow } from 'src/engine/twenty-orm/repository/permissions.utils'; -import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { WorkspaceEntityManager } from './workspace-entity-manager'; @@ -109,6 +109,7 @@ describe('WorkspaceEntityManager', () => { IS_WORKSPACE_API_KEY_WEBHOOK_GRAPHQL_ENABLED: false, IS_FIELDS_PERMISSIONS_ENABLED: false, IS_ANY_FIELD_SEARCH_ENABLED: false, + IS_CORE_VIEW_SYNCING_ENABLED: false, IS_TWO_FACTOR_AUTHENTICATION_ENABLED: false, }, eventEmitterService: {