diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-deduplicate-indexed-fields.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-deduplicate-indexed-fields.command.ts new file mode 100644 index 000000000..357937fd6 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-deduplicate-indexed-fields.command.ts @@ -0,0 +1,296 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import { Command } from 'nest-commander'; +import { IsNull, Repository } from 'typeorm'; + +import { + ActiveOrSuspendedWorkspacesMigrationCommandRunner, + RunOnWorkspaceArgs, +} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; + +@Command({ + name: 'upgrade:0-55:deduplicate-indexed-fields', + description: 'Deduplicate fields where we want to setup the index back on', +}) +export class DeduplicateIndexedFieldsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + override async runOnWorkspace({ + workspaceId, + options, + }: RunOnWorkspaceArgs): Promise { + this.logger.log( + `Deduplicating indexed fields for workspace ${workspaceId}`, + ); + + // searchVector should not be problematic since you cannot duplicate a search vector from what I guess + // in company, domainName is indexed with decorator @WorkspaceIsUnique() + // in person, email is indexed with decorator @WorkspaceIsUnique() + // in message-channel-message-association, the object is indexed with decorator @WorkspaceIndex(['messageChannelId', 'messageId'] + // in view-field, the object is indexed with decorator @WorkspaceIndex(['fieldMetadataId', 'viewId'] + // in view-sort, the object is indexed with decorator @WorkspaceIndex(['viewId', 'fieldMetadataId'] + + // not needed since no unique constraint on this one: + // in oportunity, stage is indexed with decorator @WorkspaceFieldIndex() + + await this.enforceUniqueConstraintsForWorkspace( + workspaceId, + options.dryRun ?? false, + ); + } + + private async enforceUniqueConstraintsForWorkspace( + workspaceId: string, + dryRun: boolean, + ): Promise { + await this.enforceUniqueCompanyDomainName(workspaceId, dryRun); + + await this.enforceUniquePersonEmail(workspaceId, dryRun); + + await this.enforceUniqueMessageChannelMessageAssociation( + workspaceId, + dryRun, + ); + + await this.enforceUniqueViewField(workspaceId, dryRun); + + await this.enforceUniqueViewSort(workspaceId, dryRun); + } + + private async enforceUniqueCompanyDomainName( + workspaceId: string, + dryRun: boolean, + ): Promise { + const companyRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'company', + ); + + const duplicates = await companyRepository + .createQueryBuilder('company') + .select('company.domainNamePrimaryLinkUrl') + .addSelect('COUNT(*)', 'count') + .where('company.deletedAt IS NULL') + .andWhere('company.domainNamePrimaryLinkUrl IS NOT NULL') + .andWhere("company.domainNamePrimaryLinkUrl != ''") + .groupBy('company.domainNamePrimaryLinkUrl') + .having('COUNT(*) > 1') + .getRawMany(); + + for (const duplicate of duplicates) { + const { company_domainNamePrimaryLinkUrl } = duplicate; + const companies = await companyRepository.find({ + where: { + domainName: { + primaryLinkUrl: company_domainNamePrimaryLinkUrl, + }, + deletedAt: IsNull(), + }, + order: { createdAt: 'DESC' }, + }); + + for (let i = 1; i < companies.length; i++) { + const newdomainNamePrimaryLinkUrl = `${company_domainNamePrimaryLinkUrl}${i}`; + + if (!dryRun) { + await companyRepository.update(companies[i].id, { + domainNamePrimaryLinkUrl: newdomainNamePrimaryLinkUrl, + }); + } + this.logger.log( + `Updated company ${companies[i].id} domainName from ${company_domainNamePrimaryLinkUrl} to ${newdomainNamePrimaryLinkUrl}`, + ); + } + } + } + + private async enforceUniquePersonEmail( + workspaceId: string, + dryRun: boolean, + ): Promise { + const personRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'person', + ); + + const duplicates = await personRepository + .createQueryBuilder('person') + .select('person.emailsPrimaryEmail') + .addSelect('COUNT(*)', 'count') + .where('person.deletedAt IS NULL') + .andWhere('person.emailsPrimaryEmail IS NOT NULL') + .andWhere("person.emailsPrimaryEmail != ''") + .groupBy('person.emailsPrimaryEmail') + .having('COUNT(*) > 1') + .getRawMany(); + + for (const duplicate of duplicates) { + const { person_emailsPrimaryEmail } = duplicate; + const persons = await personRepository.find({ + where: { + emails: { + primaryEmail: person_emailsPrimaryEmail, + }, + deletedAt: IsNull(), + }, + order: { createdAt: 'DESC' }, + }); + + for (let i = 1; i < persons.length; i++) { + const newEmail = person_emailsPrimaryEmail?.includes('@') + ? `${person_emailsPrimaryEmail.split('@')[0]}+${i}@${person_emailsPrimaryEmail.split('@')[1]}` + : `${person_emailsPrimaryEmail}+${i}`; + + if (!dryRun) { + await personRepository.update(persons[i].id, { + emailsPrimaryEmail: newEmail, + }); + } + this.logger.log( + `Updated person ${persons[i].id} emailsPrimaryEmail from ${person_emailsPrimaryEmail} to ${newEmail}`, + ); + } + } + } + + private async enforceUniqueMessageChannelMessageAssociation( + workspaceId: string, + dryRun: boolean, + ): Promise { + const messageChannelMessageAssociationRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'messageChannelMessageAssociation', + ); + + const duplicates = await messageChannelMessageAssociationRepository + .createQueryBuilder('messageChannelMessageAssociation') + .select('messageChannelMessageAssociation.messageId') + .addSelect('messageChannelMessageAssociation.messageChannelId') + .addSelect('COUNT(*)', 'count') + .where('messageChannelMessageAssociation.deletedAt IS NULL') + .groupBy('messageChannelMessageAssociation.messageId') + .addGroupBy('messageChannelMessageAssociation.messageChannelId') + .having('COUNT(*) > 1') + .getRawMany(); + + for (const duplicate of duplicates) { + const { + messageChannelMessageAssociation_messageId, + messageChannelMessageAssociation_messageChannelId, + } = duplicate; + const messageChannelMessageAssociations = + await messageChannelMessageAssociationRepository.find({ + where: { + messageId: messageChannelMessageAssociation_messageId, + messageChannelId: messageChannelMessageAssociation_messageChannelId, + deletedAt: IsNull(), + }, + order: { createdAt: 'DESC' }, + }); + + for (let i = 1; i < messageChannelMessageAssociations.length; i++) { + if (!dryRun) { + await messageChannelMessageAssociationRepository.delete( + messageChannelMessageAssociations[i].id, + ); + } + this.logger.log( + `Deleted messageChannelMessageAssociation ${messageChannelMessageAssociations[i].id}`, + ); + } + } + } + + private async enforceUniqueViewField( + workspaceId: string, + dryRun: boolean, + ): Promise { + const viewFieldRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'viewField', + ); + + const duplicates = await viewFieldRepository + .createQueryBuilder('viewField') + .select('viewField.fieldMetadataId') + .addSelect('viewField.viewId') + .addSelect('COUNT(*)', 'count') + .where('viewField.deletedAt IS NULL') + .groupBy('viewField.fieldMetadataId') + .addGroupBy('viewField.viewId') + .having('COUNT(*) > 1') + .getRawMany(); + + for (const duplicate of duplicates) { + const { viewField_fieldMetadataId, viewField_viewId } = duplicate; + const viewFields = await viewFieldRepository.find({ + where: { + fieldMetadataId: viewField_fieldMetadataId, + viewId: viewField_viewId, + deletedAt: IsNull(), + }, + order: { createdAt: 'DESC' }, + }); + + for (let i = 1; i < viewFields.length; i++) { + if (!dryRun) { + await viewFieldRepository.delete(viewFields[i].id); + } + this.logger.log(`Deleted viewField ${viewFields[i].id}`); + } + } + } + + private async enforceUniqueViewSort( + workspaceId: string, + dryRun: boolean, + ): Promise { + const viewSortRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'viewSort', + ); + + const duplicates = await viewSortRepository + .createQueryBuilder('viewSort') + .select('viewSort.viewId') + .addSelect('viewSort.fieldMetadataId') + .addSelect('COUNT(*)', 'count') + .where('viewSort.deletedAt IS NULL') + .groupBy('viewSort.viewId') + .addGroupBy('viewSort.fieldMetadataId') + .having('COUNT(*) > 1') + .getRawMany(); + + for (const duplicate of duplicates) { + const { viewSort_viewId, viewSort_fieldMetadataId } = duplicate; + const viewSorts = await viewSortRepository.find({ + where: { + fieldMetadataId: viewSort_fieldMetadataId, + viewId: viewSort_viewId, + deletedAt: IsNull(), + }, + order: { createdAt: 'DESC' }, + }); + + for (let i = 1; i < viewSorts.length; i++) { + if (!dryRun) { + await viewSortRepository.delete(viewSorts[i].id); + } + this.logger.log(`Deleted viewSort ${viewSorts[i].id}`); + } + } + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module.ts new file mode 100644 index 000000000..e9387f37f --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module.ts @@ -0,0 +1,32 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { DeduplicateIndexedFieldsCommand } from 'src/database/commands/upgrade-version-command/0-55/0-55-deduplicate-indexed-fields.command'; +import { AppToken } from 'src/engine/core-modules/app-token/app-token.entity'; +import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; +import { User } from 'src/engine/core-modules/user/user.entity'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature( + [Workspace, AppToken, User, UserWorkspace], + 'core', + ), + TypeOrmModule.forFeature( + [FieldMetadataEntity, ObjectMetadataEntity], + 'metadata', + ), + WorkspaceDataSourceModule, + WorkspaceMigrationRunnerModule, + WorkspaceMetadataVersionModule, + ], + providers: [DeduplicateIndexedFieldsCommand], + exports: [DeduplicateIndexedFieldsCommand], +}) +export class V0_55_UpgradeVersionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts index fbdc421f9..f5c2b76a1 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts @@ -8,6 +8,7 @@ import { V0_51_UpgradeVersionCommandModule } from 'src/database/commands/upgrade import { V0_52_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-52/0-52-upgrade-version-command.module'; import { V0_53_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-53/0-53-upgrade-version-command.module'; import { V0_54_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-54/0-54-upgrade-version-command.module'; +import { V0_55_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module'; import { DatabaseMigrationService, UpgradeCommand, @@ -25,6 +26,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp V0_52_UpgradeVersionCommandModule, V0_53_UpgradeVersionCommandModule, V0_54_UpgradeVersionCommandModule, + V0_55_UpgradeVersionCommandModule, WorkspaceSyncMetadataModule, ], providers: [DatabaseMigrationService, UpgradeCommand], diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts index e1b67d4cb..5de0407e9 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; @@ -14,6 +14,8 @@ import { isGatedAndNotEnabled } from 'src/engine/workspace-manager/workspace-syn @Injectable() export class StandardIndexFactory { + private readonly logger = new Logger(StandardIndexFactory.name); + create( standardObjectMetadataDefinitions: (typeof BaseWorkspaceEntity)[], context: WorkspaceSyncContext, @@ -67,30 +69,46 @@ export class StandardIndexFactory { ); }); - return workspaceIndexMetadataArgsCollection.map( - (workspaceIndexMetadataArgs) => { - const objectMetadata = - originalStandardObjectMetadataMap[workspaceEntity.nameSingular]; + return ( + workspaceIndexMetadataArgsCollection + .map((workspaceIndexMetadataArgs) => { + const objectMetadata = + originalStandardObjectMetadataMap[workspaceEntity.nameSingular]; - if (!objectMetadata) { - throw new Error( - `Object metadata not found for ${workspaceEntity.nameSingular}`, + if (!objectMetadata) { + throw new Error( + `Object metadata not found for ${workspaceEntity.nameSingular}`, + ); + } + + const indexMetadata: PartialIndexMetadata = { + workspaceId: context.workspaceId, + objectMetadataId: objectMetadata.id, + name: workspaceIndexMetadataArgs.name, + columns: workspaceIndexMetadataArgs.columns, + isUnique: workspaceIndexMetadataArgs.isUnique, + isCustom: false, + indexWhereClause: workspaceIndexMetadataArgs.whereClause, + indexType: workspaceIndexMetadataArgs.type, + }; + + return indexMetadata; + }) + // TODO: remove this filter when we have a way to handle index on relations + .filter((workspaceIndexMetadataArgs) => { + const objectMetadata = + originalStandardObjectMetadataMap[workspaceEntity.nameSingular]; + + const hasAllFields = workspaceIndexMetadataArgs.columns.every( + (expectedField) => { + return objectMetadata.fields.some( + (field) => field.name === expectedField, + ); + }, ); - } - const indexMetadata: PartialIndexMetadata = { - workspaceId: context.workspaceId, - objectMetadataId: objectMetadata.id, - name: workspaceIndexMetadataArgs.name, - columns: workspaceIndexMetadataArgs.columns, - isUnique: workspaceIndexMetadataArgs.isUnique, - isCustom: false, - indexWhereClause: workspaceIndexMetadataArgs.whereClause, - indexType: workspaceIndexMetadataArgs.type, - }; - - return indexMetadata; - }, + return hasAllFields; + }) ); } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index 44d1e17e1..acf182818 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -133,17 +133,15 @@ export class WorkspaceSyncMetadataService { `Workspace relation migrations took ${workspaceRelationMigrationsEnd - workspaceRelationMigrationsStart}ms`, ); - const workspaceIndexMigrations: Partial[] = []; - // 4 - Sync standard indexes on standard objects const workspaceIndexMigrationsStart = performance.now(); - // workspaceIndexMigrations = - // await this.workspaceSyncIndexMetadataService.synchronize( - // context, - // manager, - // storage, - // ); + const workspaceIndexMigrations = + await this.workspaceSyncIndexMetadataService.synchronize( + context, + manager, + storage, + ); const workspaceIndexMigrationsEnd = performance.now(); this.logger.log(