Index-back-into-the-game (#12466)

# Indexes

### TLDR:
 Putting indexes back, except relation ones

### Details:
- Added index synchronization logic back (it was removed previously in
45d4845b26)
in the sync-metadata service.

- for unique inedexes, a command will create unicity again by handling
duplicates that were cretated since the
45d4845b26
was triggered

packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-deduplicate-indexed-fields.command.ts

@@ -0,0 +1,296 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { IsNull, Repository } from 'typeorm';
+
+import {
+  ActiveOrSuspendedWorkspacesMigrationCommandRunner,
+  RunOnWorkspaceArgs,
+} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+
+@Command({
+  name: 'upgrade:0-55:deduplicate-indexed-fields',
+  description: 'Deduplicate fields where we want to setup the index back on',
+})
+export class DeduplicateIndexedFieldsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(Workspace, 'core')
+    protected readonly workspaceRepository: Repository<Workspace>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+    options,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    this.logger.log(
+      `Deduplicating indexed fields for workspace ${workspaceId}`,
+    );
+
+    // searchVector should not be problematic since you cannot duplicate a search vector from what I guess
+    // in company, domainName is indexed with decorator @WorkspaceIsUnique()
+    // in person, email is indexed with decorator @WorkspaceIsUnique()
+    // in message-channel-message-association, the object is indexed with decorator @WorkspaceIndex(['messageChannelId', 'messageId']
+    // in view-field, the object is indexed with decorator @WorkspaceIndex(['fieldMetadataId', 'viewId']
+    // in view-sort, the object is indexed with decorator @WorkspaceIndex(['viewId', 'fieldMetadataId']
+
+    // not needed since no unique constraint on this one:
+    // in oportunity, stage is indexed with decorator @WorkspaceFieldIndex()
+
+    await this.enforceUniqueConstraintsForWorkspace(
+      workspaceId,
+      options.dryRun ?? false,
+    );
+  }
+
+  private async enforceUniqueConstraintsForWorkspace(
+    workspaceId: string,
+    dryRun: boolean,
+  ): Promise<void> {
+    await this.enforceUniqueCompanyDomainName(workspaceId, dryRun);
+
+    await this.enforceUniquePersonEmail(workspaceId, dryRun);
+
+    await this.enforceUniqueMessageChannelMessageAssociation(
+      workspaceId,
+      dryRun,
+    );
+
+    await this.enforceUniqueViewField(workspaceId, dryRun);
+
+    await this.enforceUniqueViewSort(workspaceId, dryRun);
+  }
+
+  private async enforceUniqueCompanyDomainName(
+    workspaceId: string,
+    dryRun: boolean,
+  ): Promise<void> {
+    const companyRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        workspaceId,
+        'company',
+      );
+
+    const duplicates = await companyRepository
+      .createQueryBuilder('company')
+      .select('company.domainNamePrimaryLinkUrl')
+      .addSelect('COUNT(*)', 'count')
+      .where('company.deletedAt IS NULL')
+      .andWhere('company.domainNamePrimaryLinkUrl IS NOT NULL')
+      .andWhere("company.domainNamePrimaryLinkUrl != ''")
+      .groupBy('company.domainNamePrimaryLinkUrl')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const { company_domainNamePrimaryLinkUrl } = duplicate;
+      const companies = await companyRepository.find({
+        where: {
+          domainName: {
+            primaryLinkUrl: company_domainNamePrimaryLinkUrl,
+          },
+          deletedAt: IsNull(),
+        },
+        order: { createdAt: 'DESC' },
+      });
+
+      for (let i = 1; i < companies.length; i++) {
+        const newdomainNamePrimaryLinkUrl = `${company_domainNamePrimaryLinkUrl}${i}`;
+
+        if (!dryRun) {
+          await companyRepository.update(companies[i].id, {
+            domainNamePrimaryLinkUrl: newdomainNamePrimaryLinkUrl,
+          });
+        }
+        this.logger.log(
+          `Updated company ${companies[i].id} domainName from ${company_domainNamePrimaryLinkUrl} to ${newdomainNamePrimaryLinkUrl}`,
+        );
+      }
+    }
+  }
+
+  private async enforceUniquePersonEmail(
+    workspaceId: string,
+    dryRun: boolean,
+  ): Promise<void> {
+    const personRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        workspaceId,
+        'person',
+      );
+
+    const duplicates = await personRepository
+      .createQueryBuilder('person')
+      .select('person.emailsPrimaryEmail')
+      .addSelect('COUNT(*)', 'count')
+      .where('person.deletedAt IS NULL')
+      .andWhere('person.emailsPrimaryEmail IS NOT NULL')
+      .andWhere("person.emailsPrimaryEmail != ''")
+      .groupBy('person.emailsPrimaryEmail')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const { person_emailsPrimaryEmail } = duplicate;
+      const persons = await personRepository.find({
+        where: {
+          emails: {
+            primaryEmail: person_emailsPrimaryEmail,
+          },
+          deletedAt: IsNull(),
+        },
+        order: { createdAt: 'DESC' },
+      });
+
+      for (let i = 1; i < persons.length; i++) {
+        const newEmail = person_emailsPrimaryEmail?.includes('@')
+          ? `${person_emailsPrimaryEmail.split('@')[0]}+${i}@${person_emailsPrimaryEmail.split('@')[1]}`
+          : `${person_emailsPrimaryEmail}+${i}`;
+
+        if (!dryRun) {
+          await personRepository.update(persons[i].id, {
+            emailsPrimaryEmail: newEmail,
+          });
+        }
+        this.logger.log(
+          `Updated person ${persons[i].id} emailsPrimaryEmail from ${person_emailsPrimaryEmail} to ${newEmail}`,
+        );
+      }
+    }
+  }
+
+  private async enforceUniqueMessageChannelMessageAssociation(
+    workspaceId: string,
+    dryRun: boolean,
+  ): Promise<void> {
+    const messageChannelMessageAssociationRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        workspaceId,
+        'messageChannelMessageAssociation',
+      );
+
+    const duplicates = await messageChannelMessageAssociationRepository
+      .createQueryBuilder('messageChannelMessageAssociation')
+      .select('messageChannelMessageAssociation.messageId')
+      .addSelect('messageChannelMessageAssociation.messageChannelId')
+      .addSelect('COUNT(*)', 'count')
+      .where('messageChannelMessageAssociation.deletedAt IS NULL')
+      .groupBy('messageChannelMessageAssociation.messageId')
+      .addGroupBy('messageChannelMessageAssociation.messageChannelId')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const {
+        messageChannelMessageAssociation_messageId,
+        messageChannelMessageAssociation_messageChannelId,
+      } = duplicate;
+      const messageChannelMessageAssociations =
+        await messageChannelMessageAssociationRepository.find({
+          where: {
+            messageId: messageChannelMessageAssociation_messageId,
+            messageChannelId: messageChannelMessageAssociation_messageChannelId,
+            deletedAt: IsNull(),
+          },
+          order: { createdAt: 'DESC' },
+        });
+
+      for (let i = 1; i < messageChannelMessageAssociations.length; i++) {
+        if (!dryRun) {
+          await messageChannelMessageAssociationRepository.delete(
+            messageChannelMessageAssociations[i].id,
+          );
+        }
+        this.logger.log(
+          `Deleted messageChannelMessageAssociation ${messageChannelMessageAssociations[i].id}`,
+        );
+      }
+    }
+  }
+
+  private async enforceUniqueViewField(
+    workspaceId: string,
+    dryRun: boolean,
+  ): Promise<void> {
+    const viewFieldRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        workspaceId,
+        'viewField',
+      );
+
+    const duplicates = await viewFieldRepository
+      .createQueryBuilder('viewField')
+      .select('viewField.fieldMetadataId')
+      .addSelect('viewField.viewId')
+      .addSelect('COUNT(*)', 'count')
+      .where('viewField.deletedAt IS NULL')
+      .groupBy('viewField.fieldMetadataId')
+      .addGroupBy('viewField.viewId')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const { viewField_fieldMetadataId, viewField_viewId } = duplicate;
+      const viewFields = await viewFieldRepository.find({
+        where: {
+          fieldMetadataId: viewField_fieldMetadataId,
+          viewId: viewField_viewId,
+          deletedAt: IsNull(),
+        },
+        order: { createdAt: 'DESC' },
+      });
+
+      for (let i = 1; i < viewFields.length; i++) {
+        if (!dryRun) {
+          await viewFieldRepository.delete(viewFields[i].id);
+        }
+        this.logger.log(`Deleted viewField ${viewFields[i].id}`);
+      }
+    }
+  }
+
+  private async enforceUniqueViewSort(
+    workspaceId: string,
+    dryRun: boolean,
+  ): Promise<void> {
+    const viewSortRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        workspaceId,
+        'viewSort',
+      );
+
+    const duplicates = await viewSortRepository
+      .createQueryBuilder('viewSort')
+      .select('viewSort.viewId')
+      .addSelect('viewSort.fieldMetadataId')
+      .addSelect('COUNT(*)', 'count')
+      .where('viewSort.deletedAt IS NULL')
+      .groupBy('viewSort.viewId')
+      .addGroupBy('viewSort.fieldMetadataId')
+      .having('COUNT(*) > 1')
+      .getRawMany();
+
+    for (const duplicate of duplicates) {
+      const { viewSort_viewId, viewSort_fieldMetadataId } = duplicate;
+      const viewSorts = await viewSortRepository.find({
+        where: {
+          fieldMetadataId: viewSort_fieldMetadataId,
+          viewId: viewSort_viewId,
+          deletedAt: IsNull(),
+        },
+        order: { createdAt: 'DESC' },
+      });
+
+      for (let i = 1; i < viewSorts.length; i++) {
+        if (!dryRun) {
+          await viewSortRepository.delete(viewSorts[i].id);
+        }
+        this.logger.log(`Deleted viewSort ${viewSorts[i].id}`);
+      }
+    }
+  }
+}

packages/twenty-server/src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module.ts

@@ -0,0 +1,32 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { DeduplicateIndexedFieldsCommand } from 'src/database/commands/upgrade-version-command/0-55/0-55-deduplicate-indexed-fields.command';
+import { AppToken } from 'src/engine/core-modules/app-token/app-token.entity';
+import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
+import { User } from 'src/engine/core-modules/user/user.entity';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature(
+      [Workspace, AppToken, User, UserWorkspace],
+      'core',
+    ),
+    TypeOrmModule.forFeature(
+      [FieldMetadataEntity, ObjectMetadataEntity],
+      'metadata',
+    ),
+    WorkspaceDataSourceModule,
+    WorkspaceMigrationRunnerModule,
+    WorkspaceMetadataVersionModule,
+  ],
+  providers: [DeduplicateIndexedFieldsCommand],
+  exports: [DeduplicateIndexedFieldsCommand],
+})
+export class V0_55_UpgradeVersionCommandModule {}

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade-version-command.module.ts

@@ -8,6 +8,7 @@ import { V0_51_UpgradeVersionCommandModule } from 'src/database/commands/upgrade
 import { V0_52_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-52/0-52-upgrade-version-command.module';
 import { V0_53_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-53/0-53-upgrade-version-command.module';
 import { V0_54_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-54/0-54-upgrade-version-command.module';
+import { V0_55_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module';
 import {
   DatabaseMigrationService,
   UpgradeCommand,
@@ -25,6 +26,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
     V0_52_UpgradeVersionCommandModule,
     V0_53_UpgradeVersionCommandModule,
     V0_54_UpgradeVersionCommandModule,
+    V0_55_UpgradeVersionCommandModule,
     WorkspaceSyncMetadataModule,
   ],
   providers: [DatabaseMigrationService, UpgradeCommand],

packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts

@@ -1,4 +1,4 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';
 
 import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface';
 import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
@@ -14,6 +14,8 @@ import { isGatedAndNotEnabled } from 'src/engine/workspace-manager/workspace-syn
 
 @Injectable()
 export class StandardIndexFactory {
+  private readonly logger = new Logger(StandardIndexFactory.name);
+
   create(
     standardObjectMetadataDefinitions: (typeof BaseWorkspaceEntity)[],
     context: WorkspaceSyncContext,
@@ -67,30 +69,46 @@ export class StandardIndexFactory {
         );
       });
 
-    return workspaceIndexMetadataArgsCollection.map(
+    return (
-      (workspaceIndexMetadataArgs) => {
+      workspaceIndexMetadataArgsCollection
-        const objectMetadata =
+        .map((workspaceIndexMetadataArgs) => {
-          originalStandardObjectMetadataMap[workspaceEntity.nameSingular];
+          const objectMetadata =
+            originalStandardObjectMetadataMap[workspaceEntity.nameSingular];
 
-        if (!objectMetadata) {
+          if (!objectMetadata) {
-          throw new Error(
+            throw new Error(
-            `Object metadata not found for ${workspaceEntity.nameSingular}`,
+              `Object metadata not found for ${workspaceEntity.nameSingular}`,
+            );
+          }
+
+          const indexMetadata: PartialIndexMetadata = {
+            workspaceId: context.workspaceId,
+            objectMetadataId: objectMetadata.id,
+            name: workspaceIndexMetadataArgs.name,
+            columns: workspaceIndexMetadataArgs.columns,
+            isUnique: workspaceIndexMetadataArgs.isUnique,
+            isCustom: false,
+            indexWhereClause: workspaceIndexMetadataArgs.whereClause,
+            indexType: workspaceIndexMetadataArgs.type,
+          };
+
+          return indexMetadata;
+        })
+        // TODO: remove this filter when we have a way to handle index on relations
+        .filter((workspaceIndexMetadataArgs) => {
+          const objectMetadata =
+            originalStandardObjectMetadataMap[workspaceEntity.nameSingular];
+
+          const hasAllFields = workspaceIndexMetadataArgs.columns.every(
+            (expectedField) => {
+              return objectMetadata.fields.some(
+                (field) => field.name === expectedField,
+              );
+            },
           );
-        }
 
-        const indexMetadata: PartialIndexMetadata = {
+          return hasAllFields;
-          workspaceId: context.workspaceId,
+        })
-          objectMetadataId: objectMetadata.id,
-          name: workspaceIndexMetadataArgs.name,
-          columns: workspaceIndexMetadataArgs.columns,
-          isUnique: workspaceIndexMetadataArgs.isUnique,
-          isCustom: false,
-          indexWhereClause: workspaceIndexMetadataArgs.whereClause,
-          indexType: workspaceIndexMetadataArgs.type,
-        };
-
-        return indexMetadata;
-      },
     );
   }
 

packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts

@@ -133,17 +133,15 @@ export class WorkspaceSyncMetadataService {
         `Workspace relation migrations took ${workspaceRelationMigrationsEnd - workspaceRelationMigrationsStart}ms`,
       );
 
-      const workspaceIndexMigrations: Partial<WorkspaceMigrationEntity>[] = [];
-
       // 4 - Sync standard indexes on standard objects
       const workspaceIndexMigrationsStart = performance.now();
 
-      // workspaceIndexMigrations =
+      const workspaceIndexMigrations =
-      //   await this.workspaceSyncIndexMetadataService.synchronize(
+        await this.workspaceSyncIndexMetadataService.synchronize(
-      //     context,
+          context,
-      //     manager,
+          manager,
-      //     storage,
+          storage,
-      //   );
+        );
       const workspaceIndexMigrationsEnd = performance.now();
 
       this.logger.log(