diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1718985664968-addIndexMetadataTable.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1718985664968-addIndexMetadataTable.ts new file mode 100644 index 000000000..c4d5afca9 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1718985664968-addIndexMetadataTable.ts @@ -0,0 +1,28 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddIndexMetadataTable1718985664968 implements MigrationInterface { + name = 'AddIndexMetadataTable1718985664968'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "metadata"."indexMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "workspaceId" character varying, "objectMetadataId" uuid NOT NULL, CONSTRAINT "PK_f73bb3c3678aee204e341f0ca4e" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `CREATE TABLE "metadata"."indexFieldMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "indexMetadataId" uuid NOT NULL, "fieldMetadataId" uuid NOT NULL, "order" integer NOT NULL, CONSTRAINT "PK_5928f67e43eff7d95aa79fd96fd" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" ADD CONSTRAINT "FK_051487e9b745cb175950130b63f" FOREIGN KEY ("objectMetadataId") REFERENCES "metadata"."objectMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_b20192c432612eb710801dd5664" FOREIGN KEY ("indexMetadataId") REFERENCES "metadata"."indexMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_be0950612a54b58c72bd62d629e" FOREIGN KEY ("fieldMetadataId") REFERENCES "metadata"."fieldMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "metadata"."indexFieldMetadata"`); + await queryRunner.query(`DROP TABLE "metadata"."indexMetadata"`); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts index dd24719ad..44cdf69d1 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts @@ -9,6 +9,7 @@ import { CreateDateColumn, UpdateDateColumn, Relation, + OneToMany, } from 'typeorm'; import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; @@ -18,6 +19,7 @@ import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadat import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; +import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity'; export enum FieldMetadataType { UUID = 'UUID', @@ -119,6 +121,16 @@ export class FieldMetadataEntity< ) toRelationMetadata: Relation; + @OneToMany( + () => IndexFieldMetadataEntity, + (indexFieldMetadata: IndexFieldMetadataEntity) => + indexFieldMetadata.fieldMetadata, + { + cascade: true, + }, + ) + indexFieldMetadatas: Relation; + @CreateDateColumn({ type: 'timestamptz' }) createdAt: Date; diff --git a/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts new file mode 100644 index 000000000..79ce331c7 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts @@ -0,0 +1,46 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + JoinColumn, + ManyToOne, + Relation, +} from 'typeorm'; + +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; + +@Entity('indexFieldMetadata') +export class IndexFieldMetadataEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: false }) + indexMetadataId: string; + + @ManyToOne( + () => IndexMetadataEntity, + (indexMetadata) => indexMetadata.indexFieldMetadatas, + { + onDelete: 'CASCADE', + }, + ) + @JoinColumn() + indexMetadata: Relation; + + @Column({ nullable: false }) + fieldMetadataId: string; + + @ManyToOne( + () => FieldMetadataEntity, + (fieldMetadata) => fieldMetadata.indexFieldMetadatas, + { + onDelete: 'CASCADE', + }, + ) + @JoinColumn() + fieldMetadata: Relation; + + @Column({ nullable: false }) + order: number; +} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.module.ts new file mode 100644 index 000000000..937851df1 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([IndexFieldMetadataEntity], 'metadata')], + providers: [], + exports: [], +}) +export class IndexFieldMetadataModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts new file mode 100644 index 000000000..d006a092e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts @@ -0,0 +1,43 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + JoinColumn, + ManyToOne, + Relation, + OneToMany, +} from 'typeorm'; + +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity'; + +@Entity('indexMetadata') +export class IndexMetadataEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: false }) + name: string; + + @Column({ nullable: true }) + workspaceId: string; + + @Column({ nullable: false, type: 'uuid' }) + objectMetadataId: string; + + @ManyToOne(() => ObjectMetadataEntity, (object) => object.indexes, { + onDelete: 'CASCADE', + }) + @JoinColumn() + objectMetadata: Relation; + + @OneToMany( + () => IndexFieldMetadataEntity, + (indexFieldMetadata: IndexFieldMetadataEntity) => + indexFieldMetadata.indexMetadata, + { + cascade: true, + }, + ) + indexFieldMetadatas: Relation; +} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.module.ts new file mode 100644 index 000000000..01cb4a3e6 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([IndexMetadataEntity], 'metadata')], + providers: [], + exports: [], +}) +export class IndexMetadataModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name.ts new file mode 100644 index 000000000..ebf84d17d --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name.ts @@ -0,0 +1,11 @@ +import { createHash } from 'crypto'; + +export const generateDeterministicIndexName = (columns: string[]): string => { + const hash = createHash('sha256'); + + columns.forEach((column) => { + hash.update(column); + }); + + return hash.digest('hex').slice(0, 27); +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts index d4cf45c14..cc64c231b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts @@ -15,6 +15,7 @@ import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metad import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; @Entity('objectMetadata') @Unique('IndexOnNameSingularAndWorkspaceIdUnique', [ @@ -82,6 +83,11 @@ export class ObjectMetadataEntity implements ObjectMetadataInterface { }) fields: Relation; + @OneToMany(() => FieldMetadataEntity, (field) => field.object, { + cascade: true, + }) + indexes: Relation; + @OneToMany( () => RelationMetadataEntity, (relation: RelationMetadataEntity) => relation.fromObjectMetadata, diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts index 8116e0b3d..12084f80f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts @@ -18,6 +18,11 @@ export enum WorkspaceMigrationColumnActionType { export type WorkspaceMigrationRenamedEnum = { from: string; to: string }; export type WorkspaceMigrationEnum = string | WorkspaceMigrationRenamedEnum; +export enum WorkspaceMigrationIndexActionType { + CREATE = 'CREATE', + DROP = 'DROP', +} + export interface WorkspaceMigrationColumnDefinition { columnName: string; columnType: string; @@ -27,6 +32,12 @@ export interface WorkspaceMigrationColumnDefinition { defaultValue?: any; } +export interface WorkspaceMigrationIndexAction { + action: WorkspaceMigrationIndexActionType; + name: string; + columns: string[]; +} + export interface WorkspaceMigrationColumnCreate extends WorkspaceMigrationColumnDefinition { action: WorkspaceMigrationColumnActionType.CREATE; @@ -105,6 +116,7 @@ export enum WorkspaceMigrationTableActionType { CREATE_FOREIGN_TABLE = 'create_foreign_table', DROP_FOREIGN_TABLE = 'drop_foreign_table', ALTER_FOREIGN_TABLE = 'alter_foreign_table', + ALTER_INDEXES = 'alter_indexes', } export type WorkspaceMigrationTableAction = { @@ -113,6 +125,7 @@ export type WorkspaceMigrationTableAction = { action: WorkspaceMigrationTableActionType; columns?: WorkspaceMigrationColumnAction[]; foreignTable?: WorkspaceMigrationForeignTable; + indexes?: WorkspaceMigrationIndexAction[]; }; @Entity('workspaceMigration') diff --git a/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts new file mode 100644 index 000000000..e74394c2c --- /dev/null +++ b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts @@ -0,0 +1,37 @@ +import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name'; +import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage'; + +export interface WorkspaceIndexOptions { + columns?: string[]; +} + +export function WorkspaceIndex(): PropertyDecorator; +export function WorkspaceIndex(columns: string[]): ClassDecorator; +export function WorkspaceIndex( + columns?: string[], +): PropertyDecorator | ClassDecorator { + return (target: any, propertyKey: string | symbol) => { + if (propertyKey === undefined && columns === undefined) { + throw new Error('Class level WorkspaceIndex should be used with columns'); + } + + // TODO: handle composite field metadata types + // TODO: handle relation field metadata types + + if (Array.isArray(columns) && columns.length > 0) { + metadataArgsStorage.addIndexes({ + name: `IDX_${generateDeterministicIndexName(columns)}`, + columns, + target: target, + }); + + return; + } + + metadataArgsStorage.addIndexes({ + name: `IDX_${generateDeterministicIndexName([propertyKey.toString()])}`, + columns: [propertyKey.toString()], + target: target.constructor, + }); + }; +} diff --git a/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface.ts b/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface.ts new file mode 100644 index 000000000..add4c89e7 --- /dev/null +++ b/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface.ts @@ -0,0 +1,17 @@ +export interface WorkspaceIndexMetadataArgs { + /** + * Class to which index is applied. + */ + // eslint-disable-next-line @typescript-eslint/ban-types + readonly target: Function; + + /* + * Index name. + */ + name: string; + + /* + * Index columns. + */ + columns: string[]; +} diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts index 9fb114be9..f228116b5 100644 --- a/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts +++ b/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts @@ -5,6 +5,7 @@ import { WorkspaceFieldMetadataArgs } from 'src/engine/twenty-orm/interfaces/wor import { WorkspaceEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-entity-metadata-args.interface'; import { WorkspaceRelationMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-relation-metadata-args.interface'; import { WorkspaceExtendedEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-extended-entity-metadata-args.interface'; +import { WorkspaceIndexMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface'; export class MetadataArgsStorage { private readonly entities: WorkspaceEntityMetadataArgs[] = []; @@ -13,6 +14,7 @@ export class MetadataArgsStorage { private readonly relations: WorkspaceRelationMetadataArgs[] = []; private readonly dynamicRelations: WorkspaceDynamicRelationMetadataArgs[] = []; + private readonly indexes: WorkspaceIndexMetadataArgs[] = []; addEntities(...entities: WorkspaceEntityMetadataArgs[]): void { this.entities.push(...entities); @@ -32,6 +34,10 @@ export class MetadataArgsStorage { this.relations.push(...relations); } + addIndexes(...indexes: WorkspaceIndexMetadataArgs[]): void { + this.indexes.push(...indexes); + } + addDynamicRelations( ...dynamicRelations: WorkspaceDynamicRelationMetadataArgs[] ): void { @@ -93,6 +99,16 @@ export class MetadataArgsStorage { return this.filterByTarget(this.relations, target); } + filterIndexes(target: Function | string): WorkspaceIndexMetadataArgs[]; + + filterIndexes(target: (Function | string)[]): WorkspaceIndexMetadataArgs[]; + + filterIndexes( + target: (Function | string) | (Function | string)[], + ): WorkspaceIndexMetadataArgs[] { + return this.filterByTarget(this.indexes, target); + } + filterDynamicRelations( target: Function | string, ): WorkspaceDynamicRelationMetadataArgs[]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts index 50f96d394..5d5e66ac5 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts @@ -1,3 +1,5 @@ +import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory'; + import { WorkspaceMigrationObjectFactory } from './workspace-migration-object.factory'; import { WorkspaceMigrationFieldFactory } from './workspace-migration-field.factory'; import { WorkspaceMigrationRelationFactory } from './workspace-migration-relation.factory'; @@ -6,4 +8,5 @@ export const workspaceMigrationBuilderFactories = [ WorkspaceMigrationObjectFactory, WorkspaceMigrationFieldFactory, WorkspaceMigrationRelationFactory, + WorkspaceMigrationIndexFactory, ]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory.ts new file mode 100644 index 000000000..0392041ed --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory.ts @@ -0,0 +1,156 @@ +import { Injectable } from '@nestjs/common'; + +import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface'; + +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { + WorkspaceMigrationEntity, + WorkspaceMigrationIndexActionType, + WorkspaceMigrationTableActionType, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util'; +import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; + +@Injectable() +export class WorkspaceMigrationIndexFactory { + constructor() {} + + async create( + originalObjectMetadataCollection: ObjectMetadataEntity[], + indexMetadataCollection: IndexMetadataEntity[], + action: WorkspaceMigrationBuilderAction, + ): Promise[]> { + const originalObjectMetadataMap = Object.fromEntries( + originalObjectMetadataCollection.map((obj) => [obj.id, obj]), + ); + + const indexMetadataByObjectMetadataMap = new Map< + ObjectMetadataEntity, + IndexMetadataEntity[] + >(); + + indexMetadataCollection.forEach((currentIndexMetadata) => { + const objectMetadata = + originalObjectMetadataMap[currentIndexMetadata.objectMetadataId]; + + if (!objectMetadata) { + throw new Error( + `Object metadata with id ${currentIndexMetadata.objectMetadataId} not found`, + ); + } + + if (!indexMetadataByObjectMetadataMap.has(objectMetadata)) { + indexMetadataByObjectMetadataMap.set(objectMetadata, []); + } + + indexMetadataByObjectMetadataMap + ?.get(objectMetadata) + ?.push(currentIndexMetadata); + }); + + switch (action) { + case WorkspaceMigrationBuilderAction.CREATE: + return this.createIndexMigration(indexMetadataByObjectMetadataMap); + case WorkspaceMigrationBuilderAction.DELETE: + return this.deleteIndexMigration(indexMetadataByObjectMetadataMap); + default: + return []; + } + } + + private async createIndexMigration( + indexMetadataByObjectMetadataMap: Map< + ObjectMetadataEntity, + IndexMetadataEntity[] + >, + ): Promise[]> { + const workspaceMigrations: Partial[] = []; + + for (const [ + objectMetadata, + indexMetadataCollection, + ] of indexMetadataByObjectMetadataMap) { + const targetTable = computeObjectTargetTable(objectMetadata); + + const fieldsById = Object.fromEntries( + objectMetadata.fields.map((field) => [field.id, field]), + ); + + const indexes = indexMetadataCollection.map((indexMetadata) => ({ + name: indexMetadata.name, + action: WorkspaceMigrationIndexActionType.CREATE, + columns: indexMetadata.indexFieldMetadatas + .sort((a, b) => a.order - b.order) + .map((indexFieldMetadata) => { + const fieldMetadata = + fieldsById[indexFieldMetadata.fieldMetadataId]; + + if (!fieldMetadata) { + throw new Error( + `Field metadata with id ${indexFieldMetadata.fieldMetadataId} not found in object metadata with id ${objectMetadata.id}`, + ); + } + + return fieldMetadata.name; + }), + })); + + workspaceMigrations.push({ + workspaceId: objectMetadata.workspaceId, + name: generateMigrationName( + `create-${objectMetadata.nameSingular}-indexes`, + ), + isCustom: false, + migrations: [ + { + name: targetTable, + action: WorkspaceMigrationTableActionType.ALTER_INDEXES, + indexes, + }, + ], + }); + } + + return workspaceMigrations; + } + + private async deleteIndexMigration( + indexMetadataByObjectMetadataMap: Map< + ObjectMetadataEntity, + IndexMetadataEntity[] + >, + ): Promise[]> { + const workspaceMigrations: Partial[] = []; + + for (const [ + objectMetadata, + indexMetadataCollection, + ] of indexMetadataByObjectMetadataMap) { + const targetTable = computeObjectTargetTable(objectMetadata); + + const indexes = indexMetadataCollection.map((indexMetadata) => ({ + name: indexMetadata.name, + action: WorkspaceMigrationIndexActionType.DROP, + columns: [], + })); + + workspaceMigrations.push({ + workspaceId: objectMetadata.workspaceId, + name: generateMigrationName( + `delete-${objectMetadata.nameSingular}-indexes`, + ), + isCustom: false, + migrations: [ + { + name: targetTable, + action: WorkspaceMigrationTableActionType.ALTER_INDEXES, + indexes, + }, + ], + }); + } + + return workspaceMigrations; + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index 560d9fff4..c239462a2 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -5,6 +5,7 @@ import { Table, TableColumn, TableForeignKey, + TableIndex, TableUnique, } from 'typeorm'; @@ -20,6 +21,8 @@ import { WorkspaceMigrationColumnDropRelation, WorkspaceMigrationTableActionType, WorkspaceMigrationForeignTable, + WorkspaceMigrationIndexAction, + WorkspaceMigrationIndexActionType, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service'; @@ -137,6 +140,7 @@ export class WorkspaceMigrationRunnerService { tableMigration.columns, ); } + break; } case WorkspaceMigrationTableActionType.DROP: @@ -163,6 +167,17 @@ export class WorkspaceMigrationRunnerService { tableMigration.columns, ); break; + + case WorkspaceMigrationTableActionType.ALTER_INDEXES: + if (tableMigration.indexes && tableMigration.indexes.length > 0) { + await this.handleIndexesChanges( + queryRunner, + schemaName, + tableMigration.newName ?? tableMigration.name, + tableMigration.indexes, + ); + } + break; default: throw new Error( `Migration table action ${tableMigration.action} not supported`, @@ -170,6 +185,32 @@ export class WorkspaceMigrationRunnerService { } } + private async handleIndexesChanges( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + indexes: WorkspaceMigrationIndexAction[], + ) { + for (const index of indexes) { + switch (index.action) { + case WorkspaceMigrationIndexActionType.CREATE: + await queryRunner.createIndex( + `${schemaName}.${tableName}`, + new TableIndex({ + name: index.name, + columnNames: index.columns, + }), + ); + break; + case WorkspaceMigrationIndexActionType.DROP: + await queryRunner.dropIndex(`${schemaName}.${tableName}`, index.name); + break; + default: + throw new Error(`Migration index action not supported`); + } + } + } + /** * Creates a table for a given schema and table name * diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts index b6d15d677..106a0d876 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts @@ -5,7 +5,6 @@ import { Command, CommandRunner, Option } from 'nest-commander'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service'; import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service'; -import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service'; @@ -28,7 +27,6 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { private readonly workspaceHealthService: WorkspaceHealthService, private readonly dataSourceService: DataSourceService, private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService, - private readonly workspaceService: WorkspaceService, ) { super(); } @@ -37,9 +35,8 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { _passedParam: string[], options: RunWorkspaceMigrationsOptions, ): Promise { - const workspaceIds = options.workspaceId - ? [options.workspaceId] - : await this.workspaceService.getWorkspaceIds(); + // TODO: re-implement load index from workspaceService, this is breaking the logger + const workspaceIds = options.workspaceId ? [options.workspaceId] : []; for (const workspaceId of workspaceIds) { try { @@ -105,7 +102,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { @Option({ flags: '-w, --workspace-id [workspace_id]', description: 'workspace id', - required: false, + required: true, }) parseWorkspaceId(value: string): string { return value; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts index 3c95f7bab..2f29fbf41 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts @@ -1,3 +1,5 @@ +import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator'; + import { WorkspaceFieldComparator } from './workspace-field.comparator'; import { WorkspaceObjectComparator } from './workspace-object.comparator'; import { WorkspaceRelationComparator } from './workspace-relation.comparator'; @@ -6,4 +8,5 @@ export const workspaceSyncMetadataComparators = [ WorkspaceFieldComparator, WorkspaceObjectComparator, WorkspaceRelationComparator, + WorkspaceIndexComparator, ]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator.ts new file mode 100644 index 000000000..5fbf2ad97 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator.ts @@ -0,0 +1,90 @@ +import { Injectable } from '@nestjs/common'; + +import diff from 'microdiff'; + +import { + IndexComparatorResult, + ComparatorAction, +} from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; + +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { transformMetadataForComparison } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util'; + +const propertiesToIgnore = ['createdAt', 'updatedAt', 'indexFieldMetadatas']; + +@Injectable() +export class WorkspaceIndexComparator { + constructor() {} + + compare( + originalIndexMetadataCollection: IndexMetadataEntity[], + standardIndexMetadataCollection: Partial[], + ): IndexComparatorResult[] { + const results: IndexComparatorResult[] = []; + + // Create a map of standard relations + const standardIndexMetadataMap = transformMetadataForComparison( + standardIndexMetadataCollection, + { + keyFactory(indexMetadata) { + return `${indexMetadata.name}`; + }, + }, + ); + + const originalIndexMetadataCollectionWithColumns = + originalIndexMetadataCollection.map((indexMetadata) => { + return { + ...indexMetadata, + columns: indexMetadata.indexFieldMetadatas.map( + (indexFieldMetadata) => indexFieldMetadata.fieldMetadata.name, + ), + indexFieldMetadatas: undefined, + }; + }); + + // Create a filtered map of original relations + // We filter out 'id' later because we need it to remove the relation from DB + const originalIndexMetadataMap = transformMetadataForComparison( + originalIndexMetadataCollectionWithColumns, + { + shouldIgnoreProperty: (property) => + propertiesToIgnore.includes(property), + keyFactory(indexMetadata) { + return `${indexMetadata.name}`; + }, + }, + ); + + // Compare indexes + const indexesDifferences = diff( + originalIndexMetadataMap, + standardIndexMetadataMap, + ); + + for (const difference of indexesDifferences) { + switch (difference.type) { + case 'CREATE': { + results.push({ + action: ComparatorAction.CREATE, + object: difference.value, + }); + break; + } + case 'REMOVE': { + if (difference.path[difference.path.length - 1] !== 'id') { + results.push({ + action: ComparatorAction.DELETE, + object: difference.oldValue, + }); + } + break; + } + default: + break; + } + } + + return results; + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts index 721854369..958bf346c 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts @@ -1,3 +1,5 @@ +import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory'; + import { FeatureFlagFactory } from './feature-flags.factory'; import { StandardFieldFactory } from './standard-field.factory'; import { StandardObjectFactory } from './standard-object.factory'; @@ -8,4 +10,5 @@ export const workspaceSyncMetadataFactories = [ StandardFieldFactory, StandardObjectFactory, StandardRelationFactory, + StandardIndexFactory, ]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts new file mode 100644 index 000000000..8ad2c89d1 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts @@ -0,0 +1,69 @@ +import { Injectable } from '@nestjs/common'; + +import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface'; +import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface'; + +import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage'; + +@Injectable() +export class StandardIndexFactory { + create( + standardObjectMetadataDefinitions: (typeof BaseWorkspaceEntity)[], + context: WorkspaceSyncContext, + originalObjectMetadataMap: Record, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Partial[] { + return standardObjectMetadataDefinitions.flatMap((standardObjectMetadata) => + this.createIndexMetadata( + standardObjectMetadata, + context, + originalObjectMetadataMap, + workspaceFeatureFlagsMap, + ), + ); + } + + private createIndexMetadata( + target: typeof BaseWorkspaceEntity, + context: WorkspaceSyncContext, + originalObjectMetadataMap: Record, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Partial[] { + const workspaceEntity = metadataArgsStorage.filterEntities(target); + + if (!workspaceEntity) { + throw new Error( + `Object metadata decorator not found, can't parse ${target.name}`, + ); + } + + const workspaceIndexMetadataArgsCollection = + metadataArgsStorage.filterIndexes(target); + + return workspaceIndexMetadataArgsCollection.map( + (workspaceIndexMetadataArgs) => { + const objectMetadata = + originalObjectMetadataMap[workspaceEntity.nameSingular]; + + if (!objectMetadata) { + throw new Error( + `Object metadata not found for ${workspaceEntity.nameSingular}`, + ); + } + + const indexMetadata: PartialIndexMetadata = { + workspaceId: context.workspaceId, + objectMetadataId: objectMetadata.id, + name: workspaceIndexMetadataArgs.name, + columns: workspaceIndexMetadataArgs.columns, + }; + + return indexMetadata; + }, + ); + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts index efc061bc7..ab40fdfa7 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts @@ -1,5 +1,6 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ComputedPartialFieldMetadata } from './partial-field-metadata.interface'; import { ComputedPartialWorkspaceEntity } from './partial-object-metadata.interface'; @@ -49,3 +50,8 @@ export type RelationComparatorResult = | ComparatorCreateResult> | ComparatorDeleteResult | ComparatorUpdateResult>; + +export type IndexComparatorResult = + | ComparatorCreateResult> + | ComparatorUpdateResult> + | ComparatorDeleteResult; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts new file mode 100644 index 000000000..7b174d6fb --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts @@ -0,0 +1,8 @@ +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; + +export type PartialIndexMetadata = Omit< + IndexMetadataEntity, + 'id' | 'objectMetadata' | 'indexFieldMetadatas' +> & { + columns: string[]; +}; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts index 5ba73cbcc..4e6c0e117 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts @@ -11,6 +11,7 @@ import { v4 as uuidV4 } from 'uuid'; import { DeepPartial } from 'typeorm/common/DeepPartial'; import { PartialFieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-field-metadata.interface'; +import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { @@ -22,6 +23,7 @@ import { FieldMetadataComplexOption } from 'src/engine/metadata-modules/field-me import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; import { FieldMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory'; import { ObjectMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-object.factory'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; @Injectable() export class WorkspaceMetadataUpdaterService { @@ -230,6 +232,69 @@ export class WorkspaceMetadataUpdaterService { }; } + async updateIndexMetadata( + manager: EntityManager, + storage: WorkspaceSyncStorage, + originalObjectMetadataCollection: ObjectMetadataEntity[], + ): Promise<{ + createdIndexMetadataCollection: IndexMetadataEntity[]; + }> { + const indexMetadataRepository = manager.getRepository(IndexMetadataEntity); + + const convertIndexMetadataForSaving = ( + indexMetadata: PartialIndexMetadata, + ) => { + const convertIndexFieldMetadataForSaving = ( + column: string, + order: number, + ) => { + const fieldMetadata = originalObjectMetadataCollection + .find((object) => object.id === indexMetadata.objectMetadataId) + ?.fields.find((field) => column === field.name); + + if (!fieldMetadata) { + throw new Error(` + Field metadata not found for column ${column} in object ${indexMetadata.objectMetadataId} + `); + } + + return { + fieldMetadataId: fieldMetadata.id, + order, + }; + }; + + return { + ...indexMetadata, + indexFieldMetadatas: indexMetadata.columns.map((column, index) => + convertIndexFieldMetadataForSaving(column, index), + ), + }; + }; + + /** + * Create index metadata + */ + const createdIndexMetadataCollection = await indexMetadataRepository.save( + storage.indexMetadataCreateCollection.map(convertIndexMetadataForSaving), + ); + + /** + * Delete index metadata + */ + if (storage.indexMetadataDeleteCollection.length > 0) { + await indexMetadataRepository.delete( + storage.indexMetadataDeleteCollection.map( + (indexMetadata) => indexMetadata.id, + ), + ); + } + + return { + createdIndexMetadataCollection, + }; + } + /** * Update entities in the database * @param manager EntityManager diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts index 4fe25b628..e590900bc 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts @@ -48,7 +48,7 @@ export class WorkspaceSyncFieldMetadataService { relations: ['dataSource', 'fields'], }); - // Filter out custom objects + // Filter out non-custom objects const customObjectMetadataCollection = originalObjectMetadataCollection.filter( (objectMetadata) => objectMetadata.isCustom, @@ -61,7 +61,7 @@ export class WorkspaceSyncFieldMetadataService { workspaceFeatureFlagsMap, ); - // Loop over all standard objects and compare them with the objects in DB + // Loop over all custom objects from the DB and compare their fields with standard fields for (const customObjectMetadata of customObjectMetadataCollection) { // Also, maybe it's better to refactor a bit and move generation part into a separate module ? const standardObjectMetadata = computeStandardObject( diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts new file mode 100644 index 000000000..4f7a30ef9 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts @@ -0,0 +1,119 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface'; +import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; +import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface'; + +import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; +import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { mapObjectMetadataByUniqueIdentifier } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/sync-metadata.util'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects'; +import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator'; +import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service'; +import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory'; + +@Injectable() +export class WorkspaceSyncIndexMetadataService { + private readonly logger = new Logger(WorkspaceSyncIndexMetadataService.name); + + constructor( + private readonly standardIndexFactory: StandardIndexFactory, + private readonly workspaceIndexComparator: WorkspaceIndexComparator, + private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService, + private readonly workspaceMigrationIndexFactory: WorkspaceMigrationIndexFactory, + ) {} + + async synchronize( + context: WorkspaceSyncContext, + manager: EntityManager, + storage: WorkspaceSyncStorage, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Promise[]> { + this.logger.log('Syncing index metadata'); + + const objectMetadataRepository = + manager.getRepository(ObjectMetadataEntity); + + // Retrieve object metadata collection from DB + const originalObjectMetadataCollection = + await objectMetadataRepository.find({ + where: { + workspaceId: context.workspaceId, + // We're only interested in standard fields + fields: { isCustom: false }, + isCustom: false, + }, + relations: ['dataSource', 'fields', 'indexes'], + }); + + // Create map of object metadata & field metadata by unique identifier + const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier( + originalObjectMetadataCollection, + // Relation are based on the singular name + (objectMetadata) => objectMetadata.nameSingular, + ); + + const indexMetadataRepository = manager.getRepository(IndexMetadataEntity); + + const originalIndexMetadataCollection = await indexMetadataRepository.find({ + where: { + workspaceId: context.workspaceId, + }, + relations: ['indexFieldMetadatas.fieldMetadata'], + }); + + // Generate index metadata from models + const standardIndexMetadataCollection = this.standardIndexFactory.create( + standardObjectMetadataDefinitions, + context, + originalObjectMetadataMap, + workspaceFeatureFlagsMap, + ); + + const indexComparatorResults = this.workspaceIndexComparator.compare( + originalIndexMetadataCollection, + standardIndexMetadataCollection, + ); + + for (const indexComparatorResult of indexComparatorResults) { + if (indexComparatorResult.action === ComparatorAction.CREATE) { + storage.addCreateIndexMetadata(indexComparatorResult.object); + } else if (indexComparatorResult.action === ComparatorAction.DELETE) { + storage.addDeleteIndexMetadata(indexComparatorResult.object); + } + } + + const metadataIndexUpdaterResult = + await this.workspaceMetadataUpdaterService.updateIndexMetadata( + manager, + storage, + originalObjectMetadataCollection, + ); + + // Create migrations + const createIndexWorkspaceMigrations = + await this.workspaceMigrationIndexFactory.create( + originalObjectMetadataCollection, + metadataIndexUpdaterResult.createdIndexMetadataCollection, + WorkspaceMigrationBuilderAction.CREATE, + ); + + const deleteIndexWorkspaceMigrations = + await this.workspaceMigrationIndexFactory.create( + originalObjectMetadataCollection, + storage.indexMetadataDeleteCollection, + WorkspaceMigrationBuilderAction.DELETE, + ); + + return [ + ...createIndexWorkspaceMigrations, + ...deleteIndexWorkspaceMigrations, + ]; + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts index 147389b18..7d809b3f8 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts @@ -112,6 +112,10 @@ export class WorkspaceSyncObjectMetadataService { /** * COMPARE FIELD METADATA + * NOTE: This should be moved to WorkspaceSyncFieldMetadataService for more clarity since + * this code only adds field metadata to the storage but it's actually used in the other service. + * NOTE2: WorkspaceSyncFieldMetadataService has been added for custom fields sync, it should be refactored to handle + * both custom and non-custom fields. */ const fieldComparatorResults = this.workspaceFieldComparator.compare( originalObjectMetadata, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts index 6d0a7a9d3..5cf0fd60a 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts @@ -4,6 +4,7 @@ import { ComputedPartialFieldMetadata } from 'src/engine/workspace-manager/works import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; export class WorkspaceSyncStorage { // Object metadata @@ -25,10 +26,17 @@ export class WorkspaceSyncStorage { // Relation metadata private readonly _relationMetadataCreateCollection: Partial[] = []; - private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] = - []; private readonly _relationMetadataUpdateCollection: Partial[] = []; + private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] = + []; + + // Index metadata + private readonly _indexMetadataCreateCollection: Partial[] = + []; + private readonly _indexMetadataUpdateCollection: Partial[] = + []; + private readonly _indexMetadataDeleteCollection: IndexMetadataEntity[] = []; constructor() {} @@ -68,6 +76,18 @@ export class WorkspaceSyncStorage { return this._relationMetadataDeleteCollection; } + get indexMetadataCreateCollection() { + return this._indexMetadataCreateCollection; + } + + get indexMetadataUpdateCollection() { + return this._indexMetadataUpdateCollection; + } + + get indexMetadataDeleteCollection() { + return this._indexMetadataDeleteCollection; + } + addCreateObjectMetadata(object: ComputedPartialWorkspaceEntity) { this._objectMetadataCreateCollection.push(object); } @@ -107,4 +127,16 @@ export class WorkspaceSyncStorage { addDeleteRelationMetadata(relation: RelationMetadataEntity) { this._relationMetadataDeleteCollection.push(relation); } + + addCreateIndexMetadata(index: Partial) { + this._indexMetadataCreateCollection.push(index); + } + + addUpdateIndexMetadata(index: Partial) { + this._indexMetadataUpdateCollection.push(index); + } + + addDeleteIndexMetadata(index: IndexMetadataEntity) { + this._indexMetadataDeleteCollection.push(index); + } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts index 7e8d046cb..951d9e74a 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts @@ -16,6 +16,7 @@ import { WorkspaceSyncRelationMetadataService } from 'src/engine/workspace-manag import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service'; import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module'; import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service'; @Module({ imports: [ @@ -41,6 +42,7 @@ import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspa WorkspaceSyncRelationMetadataService, WorkspaceSyncFieldMetadataService, WorkspaceSyncMetadataService, + WorkspaceSyncIndexMetadataService, ], exports: [...workspaceSyncMetadataFactories, WorkspaceSyncMetadataService], }) diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index 8b4db8d36..b233be4e3 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -13,6 +13,7 @@ import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/ import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service'; interface SynchronizeOptions { applyChanges?: boolean; @@ -31,6 +32,7 @@ export class WorkspaceSyncMetadataService { private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService, private readonly workspaceSyncFieldMetadataService: WorkspaceSyncFieldMetadataService, private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceSyncIndexMetadataService: WorkspaceSyncIndexMetadataService, ) {} /** @@ -97,11 +99,21 @@ export class WorkspaceSyncMetadataService { workspaceFeatureFlagsMap, ); + // 4 - Sync standard indexes on standard objects + const workspaceIndexMigrations = + await this.workspaceSyncIndexMetadataService.synchronize( + context, + manager, + storage, + workspaceFeatureFlagsMap, + ); + // Save workspace migrations into the database workspaceMigrations = await workspaceMigrationRepository.save([ ...workspaceObjectMigrations, ...workspaceFieldMigrations, ...workspaceRelationMigrations, + ...workspaceIndexMigrations, ]); // If we're running a dry run, rollback the transaction and do not execute migrations