From e13dc7a1fc47b638903cd6f67ca3db0c8be1f4a0 Mon Sep 17 00:00:00 2001 From: Weiko Date: Sat, 22 Jun 2024 12:39:57 +0200 Subject: [PATCH] [FlexibleSchema] Add IndexMetadata decorator (#5981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Context Our Flexible Schema engine dynamically generates entities/tables/APIs for us but was not flexible enough to build indexes in the DB. With more and more features involving heavy queries such as Messaging, we are now adding a new WorkspaceIndex() decorator for our standard objects (will come later for custom objects). This decorator will give enough information to the workspace sync metadata manager to generate the proper migrations that will create or drop indexes on demand. To be aligned with the rest of the engine, we are adding 2 new tables: IndexMetadata and IndexFieldMetadata, that will store the info of our indexes. ## Implementation ```typescript @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.person, namePlural: 'people', labelSingular: 'Person', labelPlural: 'People', description: 'A person', icon: 'IconUser', }) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceField({ standardId: PERSON_STANDARD_FIELD_IDS.email, type: FieldMetadataType.EMAIL, label: 'Email', description: 'Contact’s Email', icon: 'IconMail', }) @WorkspaceIndex() email: string; ``` By simply adding the WorkspaceIndex decorator, sync-metadata command will create a new index for that column. We can also add composite indexes, note that the order is important for PSQL. ```typescript @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.person, namePlural: 'people', labelSingular: 'Person', labelPlural: 'People', description: 'A person', icon: 'IconUser', }) @WorkspaceIndex(['phone', 'email']) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { ``` Currently composite fields and relation fields are not handled by @WorkspaceIndex() and you will need to use this notation instead ```typescript @WorkspaceIndex(['companyId', 'nameFirstName']) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { ``` Screenshot 2024-06-21 at 15 15 45 Next step: We might need to implement more complex index expressions, this is why we have an expression column in IndexMetadata. What I had in mind for the decorator, still open to discussion ```typescript @WorkspaceIndex(['nameFirstName', 'nameLastName'], { expression: "$1 || ' ' || $2"}) export class PersonWorkspaceEntity extends BaseWorkspaceEntity { ``` --------- Co-authored-by: Charles Bochet --- .../1718985664968-addIndexMetadataTable.ts | 28 ++++ .../field-metadata/field-metadata.entity.ts | 12 ++ .../index-field-metadata.entity.ts | 46 ++++++ .../index-field-metadata.module.ts | 11 ++ .../index-metadata/index-metadata.entity.ts | 43 +++++ .../index-metadata/index-metadata.module.ts | 11 ++ .../generate-deterministic-index-name.ts | 11 ++ .../object-metadata/object-metadata.entity.ts | 6 + .../workspace-migration.entity.ts | 13 ++ .../decorators/workspace-index.decorator.ts | 37 +++++ ...workspace-index-metadata-args.interface.ts | 17 ++ .../storage/metadata-args.storage.ts | 16 ++ .../factories/index.ts | 3 + .../workspace-migration-index.factory.ts | 156 ++++++++++++++++++ .../workspace-migration-runner.service.ts | 41 +++++ .../sync-workspace-metadata.command.ts | 9 +- .../comparators/index.ts | 3 + .../comparators/workspace-index.comparator.ts | 90 ++++++++++ .../factories/index.ts | 3 + .../factories/standard-index.factory.ts | 69 ++++++++ .../interfaces/comparator.interface.ts | 6 + .../partial-index-metadata.interface.ts | 8 + .../workspace-metadata-updater.service.ts | 65 ++++++++ .../workspace-sync-field-metadata.service.ts | 4 +- .../workspace-sync-index-metadata.service.ts | 119 +++++++++++++ .../workspace-sync-object-metadata.service.ts | 4 + .../storage/workspace-sync.storage.ts | 36 +++- .../workspace-sync-metadata.module.ts | 2 + .../workspace-sync-metadata.service.ts | 12 ++ 29 files changed, 871 insertions(+), 10 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/metadata/migrations/1718985664968-addIndexMetadataTable.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name.ts create mode 100644 packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts create mode 100644 packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface.ts create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory.ts create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator.ts create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1718985664968-addIndexMetadataTable.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1718985664968-addIndexMetadataTable.ts new file mode 100644 index 000000000..c4d5afca9 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1718985664968-addIndexMetadataTable.ts @@ -0,0 +1,28 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddIndexMetadataTable1718985664968 implements MigrationInterface { + name = 'AddIndexMetadataTable1718985664968'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "metadata"."indexMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "workspaceId" character varying, "objectMetadataId" uuid NOT NULL, CONSTRAINT "PK_f73bb3c3678aee204e341f0ca4e" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `CREATE TABLE "metadata"."indexFieldMetadata" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "indexMetadataId" uuid NOT NULL, "fieldMetadataId" uuid NOT NULL, "order" integer NOT NULL, CONSTRAINT "PK_5928f67e43eff7d95aa79fd96fd" PRIMARY KEY ("id"))`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" ADD CONSTRAINT "FK_051487e9b745cb175950130b63f" FOREIGN KEY ("objectMetadataId") REFERENCES "metadata"."objectMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_b20192c432612eb710801dd5664" FOREIGN KEY ("indexMetadataId") REFERENCES "metadata"."indexMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" ADD CONSTRAINT "FK_be0950612a54b58c72bd62d629e" FOREIGN KEY ("fieldMetadataId") REFERENCES "metadata"."fieldMetadata"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "metadata"."indexFieldMetadata"`); + await queryRunner.query(`DROP TABLE "metadata"."indexMetadata"`); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts index dd24719ad..44cdf69d1 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts @@ -9,6 +9,7 @@ import { CreateDateColumn, UpdateDateColumn, Relation, + OneToMany, } from 'typeorm'; import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; @@ -18,6 +19,7 @@ import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadat import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; +import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity'; export enum FieldMetadataType { UUID = 'UUID', @@ -119,6 +121,16 @@ export class FieldMetadataEntity< ) toRelationMetadata: Relation; + @OneToMany( + () => IndexFieldMetadataEntity, + (indexFieldMetadata: IndexFieldMetadataEntity) => + indexFieldMetadata.fieldMetadata, + { + cascade: true, + }, + ) + indexFieldMetadatas: Relation; + @CreateDateColumn({ type: 'timestamptz' }) createdAt: Date; diff --git a/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts new file mode 100644 index 000000000..79ce331c7 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts @@ -0,0 +1,46 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + JoinColumn, + ManyToOne, + Relation, +} from 'typeorm'; + +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; + +@Entity('indexFieldMetadata') +export class IndexFieldMetadataEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: false }) + indexMetadataId: string; + + @ManyToOne( + () => IndexMetadataEntity, + (indexMetadata) => indexMetadata.indexFieldMetadatas, + { + onDelete: 'CASCADE', + }, + ) + @JoinColumn() + indexMetadata: Relation; + + @Column({ nullable: false }) + fieldMetadataId: string; + + @ManyToOne( + () => FieldMetadataEntity, + (fieldMetadata) => fieldMetadata.indexFieldMetadatas, + { + onDelete: 'CASCADE', + }, + ) + @JoinColumn() + fieldMetadata: Relation; + + @Column({ nullable: false }) + order: number; +} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.module.ts new file mode 100644 index 000000000..937851df1 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([IndexFieldMetadataEntity], 'metadata')], + providers: [], + exports: [], +}) +export class IndexFieldMetadataModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts new file mode 100644 index 000000000..d006a092e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts @@ -0,0 +1,43 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + JoinColumn, + ManyToOne, + Relation, + OneToMany, +} from 'typeorm'; + +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity'; + +@Entity('indexMetadata') +export class IndexMetadataEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: false }) + name: string; + + @Column({ nullable: true }) + workspaceId: string; + + @Column({ nullable: false, type: 'uuid' }) + objectMetadataId: string; + + @ManyToOne(() => ObjectMetadataEntity, (object) => object.indexes, { + onDelete: 'CASCADE', + }) + @JoinColumn() + objectMetadata: Relation; + + @OneToMany( + () => IndexFieldMetadataEntity, + (indexFieldMetadata: IndexFieldMetadataEntity) => + indexFieldMetadata.indexMetadata, + { + cascade: true, + }, + ) + indexFieldMetadatas: Relation; +} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.module.ts new file mode 100644 index 000000000..01cb4a3e6 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([IndexMetadataEntity], 'metadata')], + providers: [], + exports: [], +}) +export class IndexMetadataModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name.ts new file mode 100644 index 000000000..ebf84d17d --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name.ts @@ -0,0 +1,11 @@ +import { createHash } from 'crypto'; + +export const generateDeterministicIndexName = (columns: string[]): string => { + const hash = createHash('sha256'); + + columns.forEach((column) => { + hash.update(column); + }); + + return hash.digest('hex').slice(0, 27); +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts index d4cf45c14..cc64c231b 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.entity.ts @@ -15,6 +15,7 @@ import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metad import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; @Entity('objectMetadata') @Unique('IndexOnNameSingularAndWorkspaceIdUnique', [ @@ -82,6 +83,11 @@ export class ObjectMetadataEntity implements ObjectMetadataInterface { }) fields: Relation; + @OneToMany(() => FieldMetadataEntity, (field) => field.object, { + cascade: true, + }) + indexes: Relation; + @OneToMany( () => RelationMetadataEntity, (relation: RelationMetadataEntity) => relation.fromObjectMetadata, diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts index 8116e0b3d..12084f80f 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts @@ -18,6 +18,11 @@ export enum WorkspaceMigrationColumnActionType { export type WorkspaceMigrationRenamedEnum = { from: string; to: string }; export type WorkspaceMigrationEnum = string | WorkspaceMigrationRenamedEnum; +export enum WorkspaceMigrationIndexActionType { + CREATE = 'CREATE', + DROP = 'DROP', +} + export interface WorkspaceMigrationColumnDefinition { columnName: string; columnType: string; @@ -27,6 +32,12 @@ export interface WorkspaceMigrationColumnDefinition { defaultValue?: any; } +export interface WorkspaceMigrationIndexAction { + action: WorkspaceMigrationIndexActionType; + name: string; + columns: string[]; +} + export interface WorkspaceMigrationColumnCreate extends WorkspaceMigrationColumnDefinition { action: WorkspaceMigrationColumnActionType.CREATE; @@ -105,6 +116,7 @@ export enum WorkspaceMigrationTableActionType { CREATE_FOREIGN_TABLE = 'create_foreign_table', DROP_FOREIGN_TABLE = 'drop_foreign_table', ALTER_FOREIGN_TABLE = 'alter_foreign_table', + ALTER_INDEXES = 'alter_indexes', } export type WorkspaceMigrationTableAction = { @@ -113,6 +125,7 @@ export type WorkspaceMigrationTableAction = { action: WorkspaceMigrationTableActionType; columns?: WorkspaceMigrationColumnAction[]; foreignTable?: WorkspaceMigrationForeignTable; + indexes?: WorkspaceMigrationIndexAction[]; }; @Entity('workspaceMigration') diff --git a/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts new file mode 100644 index 000000000..e74394c2c --- /dev/null +++ b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts @@ -0,0 +1,37 @@ +import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name'; +import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage'; + +export interface WorkspaceIndexOptions { + columns?: string[]; +} + +export function WorkspaceIndex(): PropertyDecorator; +export function WorkspaceIndex(columns: string[]): ClassDecorator; +export function WorkspaceIndex( + columns?: string[], +): PropertyDecorator | ClassDecorator { + return (target: any, propertyKey: string | symbol) => { + if (propertyKey === undefined && columns === undefined) { + throw new Error('Class level WorkspaceIndex should be used with columns'); + } + + // TODO: handle composite field metadata types + // TODO: handle relation field metadata types + + if (Array.isArray(columns) && columns.length > 0) { + metadataArgsStorage.addIndexes({ + name: `IDX_${generateDeterministicIndexName(columns)}`, + columns, + target: target, + }); + + return; + } + + metadataArgsStorage.addIndexes({ + name: `IDX_${generateDeterministicIndexName([propertyKey.toString()])}`, + columns: [propertyKey.toString()], + target: target.constructor, + }); + }; +} diff --git a/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface.ts b/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface.ts new file mode 100644 index 000000000..add4c89e7 --- /dev/null +++ b/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface.ts @@ -0,0 +1,17 @@ +export interface WorkspaceIndexMetadataArgs { + /** + * Class to which index is applied. + */ + // eslint-disable-next-line @typescript-eslint/ban-types + readonly target: Function; + + /* + * Index name. + */ + name: string; + + /* + * Index columns. + */ + columns: string[]; +} diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts index 9fb114be9..f228116b5 100644 --- a/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts +++ b/packages/twenty-server/src/engine/twenty-orm/storage/metadata-args.storage.ts @@ -5,6 +5,7 @@ import { WorkspaceFieldMetadataArgs } from 'src/engine/twenty-orm/interfaces/wor import { WorkspaceEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-entity-metadata-args.interface'; import { WorkspaceRelationMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-relation-metadata-args.interface'; import { WorkspaceExtendedEntityMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-extended-entity-metadata-args.interface'; +import { WorkspaceIndexMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-index-metadata-args.interface'; export class MetadataArgsStorage { private readonly entities: WorkspaceEntityMetadataArgs[] = []; @@ -13,6 +14,7 @@ export class MetadataArgsStorage { private readonly relations: WorkspaceRelationMetadataArgs[] = []; private readonly dynamicRelations: WorkspaceDynamicRelationMetadataArgs[] = []; + private readonly indexes: WorkspaceIndexMetadataArgs[] = []; addEntities(...entities: WorkspaceEntityMetadataArgs[]): void { this.entities.push(...entities); @@ -32,6 +34,10 @@ export class MetadataArgsStorage { this.relations.push(...relations); } + addIndexes(...indexes: WorkspaceIndexMetadataArgs[]): void { + this.indexes.push(...indexes); + } + addDynamicRelations( ...dynamicRelations: WorkspaceDynamicRelationMetadataArgs[] ): void { @@ -93,6 +99,16 @@ export class MetadataArgsStorage { return this.filterByTarget(this.relations, target); } + filterIndexes(target: Function | string): WorkspaceIndexMetadataArgs[]; + + filterIndexes(target: (Function | string)[]): WorkspaceIndexMetadataArgs[]; + + filterIndexes( + target: (Function | string) | (Function | string)[], + ): WorkspaceIndexMetadataArgs[] { + return this.filterByTarget(this.indexes, target); + } + filterDynamicRelations( target: Function | string, ): WorkspaceDynamicRelationMetadataArgs[]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts index 50f96d394..5d5e66ac5 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/index.ts @@ -1,3 +1,5 @@ +import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory'; + import { WorkspaceMigrationObjectFactory } from './workspace-migration-object.factory'; import { WorkspaceMigrationFieldFactory } from './workspace-migration-field.factory'; import { WorkspaceMigrationRelationFactory } from './workspace-migration-relation.factory'; @@ -6,4 +8,5 @@ export const workspaceMigrationBuilderFactories = [ WorkspaceMigrationObjectFactory, WorkspaceMigrationFieldFactory, WorkspaceMigrationRelationFactory, + WorkspaceMigrationIndexFactory, ]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory.ts new file mode 100644 index 000000000..0392041ed --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory.ts @@ -0,0 +1,156 @@ +import { Injectable } from '@nestjs/common'; + +import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface'; + +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { + WorkspaceMigrationEntity, + WorkspaceMigrationIndexActionType, + WorkspaceMigrationTableActionType, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util'; +import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; + +@Injectable() +export class WorkspaceMigrationIndexFactory { + constructor() {} + + async create( + originalObjectMetadataCollection: ObjectMetadataEntity[], + indexMetadataCollection: IndexMetadataEntity[], + action: WorkspaceMigrationBuilderAction, + ): Promise[]> { + const originalObjectMetadataMap = Object.fromEntries( + originalObjectMetadataCollection.map((obj) => [obj.id, obj]), + ); + + const indexMetadataByObjectMetadataMap = new Map< + ObjectMetadataEntity, + IndexMetadataEntity[] + >(); + + indexMetadataCollection.forEach((currentIndexMetadata) => { + const objectMetadata = + originalObjectMetadataMap[currentIndexMetadata.objectMetadataId]; + + if (!objectMetadata) { + throw new Error( + `Object metadata with id ${currentIndexMetadata.objectMetadataId} not found`, + ); + } + + if (!indexMetadataByObjectMetadataMap.has(objectMetadata)) { + indexMetadataByObjectMetadataMap.set(objectMetadata, []); + } + + indexMetadataByObjectMetadataMap + ?.get(objectMetadata) + ?.push(currentIndexMetadata); + }); + + switch (action) { + case WorkspaceMigrationBuilderAction.CREATE: + return this.createIndexMigration(indexMetadataByObjectMetadataMap); + case WorkspaceMigrationBuilderAction.DELETE: + return this.deleteIndexMigration(indexMetadataByObjectMetadataMap); + default: + return []; + } + } + + private async createIndexMigration( + indexMetadataByObjectMetadataMap: Map< + ObjectMetadataEntity, + IndexMetadataEntity[] + >, + ): Promise[]> { + const workspaceMigrations: Partial[] = []; + + for (const [ + objectMetadata, + indexMetadataCollection, + ] of indexMetadataByObjectMetadataMap) { + const targetTable = computeObjectTargetTable(objectMetadata); + + const fieldsById = Object.fromEntries( + objectMetadata.fields.map((field) => [field.id, field]), + ); + + const indexes = indexMetadataCollection.map((indexMetadata) => ({ + name: indexMetadata.name, + action: WorkspaceMigrationIndexActionType.CREATE, + columns: indexMetadata.indexFieldMetadatas + .sort((a, b) => a.order - b.order) + .map((indexFieldMetadata) => { + const fieldMetadata = + fieldsById[indexFieldMetadata.fieldMetadataId]; + + if (!fieldMetadata) { + throw new Error( + `Field metadata with id ${indexFieldMetadata.fieldMetadataId} not found in object metadata with id ${objectMetadata.id}`, + ); + } + + return fieldMetadata.name; + }), + })); + + workspaceMigrations.push({ + workspaceId: objectMetadata.workspaceId, + name: generateMigrationName( + `create-${objectMetadata.nameSingular}-indexes`, + ), + isCustom: false, + migrations: [ + { + name: targetTable, + action: WorkspaceMigrationTableActionType.ALTER_INDEXES, + indexes, + }, + ], + }); + } + + return workspaceMigrations; + } + + private async deleteIndexMigration( + indexMetadataByObjectMetadataMap: Map< + ObjectMetadataEntity, + IndexMetadataEntity[] + >, + ): Promise[]> { + const workspaceMigrations: Partial[] = []; + + for (const [ + objectMetadata, + indexMetadataCollection, + ] of indexMetadataByObjectMetadataMap) { + const targetTable = computeObjectTargetTable(objectMetadata); + + const indexes = indexMetadataCollection.map((indexMetadata) => ({ + name: indexMetadata.name, + action: WorkspaceMigrationIndexActionType.DROP, + columns: [], + })); + + workspaceMigrations.push({ + workspaceId: objectMetadata.workspaceId, + name: generateMigrationName( + `delete-${objectMetadata.nameSingular}-indexes`, + ), + isCustom: false, + migrations: [ + { + name: targetTable, + action: WorkspaceMigrationTableActionType.ALTER_INDEXES, + indexes, + }, + ], + }); + } + + return workspaceMigrations; + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index 560d9fff4..c239462a2 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -5,6 +5,7 @@ import { Table, TableColumn, TableForeignKey, + TableIndex, TableUnique, } from 'typeorm'; @@ -20,6 +21,8 @@ import { WorkspaceMigrationColumnDropRelation, WorkspaceMigrationTableActionType, WorkspaceMigrationForeignTable, + WorkspaceMigrationIndexAction, + WorkspaceMigrationIndexActionType, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service'; @@ -137,6 +140,7 @@ export class WorkspaceMigrationRunnerService { tableMigration.columns, ); } + break; } case WorkspaceMigrationTableActionType.DROP: @@ -163,6 +167,17 @@ export class WorkspaceMigrationRunnerService { tableMigration.columns, ); break; + + case WorkspaceMigrationTableActionType.ALTER_INDEXES: + if (tableMigration.indexes && tableMigration.indexes.length > 0) { + await this.handleIndexesChanges( + queryRunner, + schemaName, + tableMigration.newName ?? tableMigration.name, + tableMigration.indexes, + ); + } + break; default: throw new Error( `Migration table action ${tableMigration.action} not supported`, @@ -170,6 +185,32 @@ export class WorkspaceMigrationRunnerService { } } + private async handleIndexesChanges( + queryRunner: QueryRunner, + schemaName: string, + tableName: string, + indexes: WorkspaceMigrationIndexAction[], + ) { + for (const index of indexes) { + switch (index.action) { + case WorkspaceMigrationIndexActionType.CREATE: + await queryRunner.createIndex( + `${schemaName}.${tableName}`, + new TableIndex({ + name: index.name, + columnNames: index.columns, + }), + ); + break; + case WorkspaceMigrationIndexActionType.DROP: + await queryRunner.dropIndex(`${schemaName}.${tableName}`, index.name); + break; + default: + throw new Error(`Migration index action not supported`); + } + } + } + /** * Creates a table for a given schema and table name * diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts index b6d15d677..106a0d876 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command.ts @@ -5,7 +5,6 @@ import { Command, CommandRunner, Option } from 'nest-commander'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service'; import { WorkspaceHealthService } from 'src/engine/workspace-manager/workspace-health/workspace-health.service'; -import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { SyncWorkspaceLoggerService } from './services/sync-workspace-logger.service'; @@ -28,7 +27,6 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { private readonly workspaceHealthService: WorkspaceHealthService, private readonly dataSourceService: DataSourceService, private readonly syncWorkspaceLoggerService: SyncWorkspaceLoggerService, - private readonly workspaceService: WorkspaceService, ) { super(); } @@ -37,9 +35,8 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { _passedParam: string[], options: RunWorkspaceMigrationsOptions, ): Promise { - const workspaceIds = options.workspaceId - ? [options.workspaceId] - : await this.workspaceService.getWorkspaceIds(); + // TODO: re-implement load index from workspaceService, this is breaking the logger + const workspaceIds = options.workspaceId ? [options.workspaceId] : []; for (const workspaceId of workspaceIds) { try { @@ -105,7 +102,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { @Option({ flags: '-w, --workspace-id [workspace_id]', description: 'workspace id', - required: false, + required: true, }) parseWorkspaceId(value: string): string { return value; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts index 3c95f7bab..2f29fbf41 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/index.ts @@ -1,3 +1,5 @@ +import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator'; + import { WorkspaceFieldComparator } from './workspace-field.comparator'; import { WorkspaceObjectComparator } from './workspace-object.comparator'; import { WorkspaceRelationComparator } from './workspace-relation.comparator'; @@ -6,4 +8,5 @@ export const workspaceSyncMetadataComparators = [ WorkspaceFieldComparator, WorkspaceObjectComparator, WorkspaceRelationComparator, + WorkspaceIndexComparator, ]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator.ts new file mode 100644 index 000000000..5fbf2ad97 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator.ts @@ -0,0 +1,90 @@ +import { Injectable } from '@nestjs/common'; + +import diff from 'microdiff'; + +import { + IndexComparatorResult, + ComparatorAction, +} from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; + +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { transformMetadataForComparison } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/utils/transform-metadata-for-comparison.util'; + +const propertiesToIgnore = ['createdAt', 'updatedAt', 'indexFieldMetadatas']; + +@Injectable() +export class WorkspaceIndexComparator { + constructor() {} + + compare( + originalIndexMetadataCollection: IndexMetadataEntity[], + standardIndexMetadataCollection: Partial[], + ): IndexComparatorResult[] { + const results: IndexComparatorResult[] = []; + + // Create a map of standard relations + const standardIndexMetadataMap = transformMetadataForComparison( + standardIndexMetadataCollection, + { + keyFactory(indexMetadata) { + return `${indexMetadata.name}`; + }, + }, + ); + + const originalIndexMetadataCollectionWithColumns = + originalIndexMetadataCollection.map((indexMetadata) => { + return { + ...indexMetadata, + columns: indexMetadata.indexFieldMetadatas.map( + (indexFieldMetadata) => indexFieldMetadata.fieldMetadata.name, + ), + indexFieldMetadatas: undefined, + }; + }); + + // Create a filtered map of original relations + // We filter out 'id' later because we need it to remove the relation from DB + const originalIndexMetadataMap = transformMetadataForComparison( + originalIndexMetadataCollectionWithColumns, + { + shouldIgnoreProperty: (property) => + propertiesToIgnore.includes(property), + keyFactory(indexMetadata) { + return `${indexMetadata.name}`; + }, + }, + ); + + // Compare indexes + const indexesDifferences = diff( + originalIndexMetadataMap, + standardIndexMetadataMap, + ); + + for (const difference of indexesDifferences) { + switch (difference.type) { + case 'CREATE': { + results.push({ + action: ComparatorAction.CREATE, + object: difference.value, + }); + break; + } + case 'REMOVE': { + if (difference.path[difference.path.length - 1] !== 'id') { + results.push({ + action: ComparatorAction.DELETE, + object: difference.oldValue, + }); + } + break; + } + default: + break; + } + } + + return results; + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts index 721854369..958bf346c 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/index.ts @@ -1,3 +1,5 @@ +import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory'; + import { FeatureFlagFactory } from './feature-flags.factory'; import { StandardFieldFactory } from './standard-field.factory'; import { StandardObjectFactory } from './standard-object.factory'; @@ -8,4 +10,5 @@ export const workspaceSyncMetadataFactories = [ StandardFieldFactory, StandardObjectFactory, StandardRelationFactory, + StandardIndexFactory, ]; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts new file mode 100644 index 000000000..8ad2c89d1 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts @@ -0,0 +1,69 @@ +import { Injectable } from '@nestjs/common'; + +import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface'; +import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface'; + +import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage'; + +@Injectable() +export class StandardIndexFactory { + create( + standardObjectMetadataDefinitions: (typeof BaseWorkspaceEntity)[], + context: WorkspaceSyncContext, + originalObjectMetadataMap: Record, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Partial[] { + return standardObjectMetadataDefinitions.flatMap((standardObjectMetadata) => + this.createIndexMetadata( + standardObjectMetadata, + context, + originalObjectMetadataMap, + workspaceFeatureFlagsMap, + ), + ); + } + + private createIndexMetadata( + target: typeof BaseWorkspaceEntity, + context: WorkspaceSyncContext, + originalObjectMetadataMap: Record, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Partial[] { + const workspaceEntity = metadataArgsStorage.filterEntities(target); + + if (!workspaceEntity) { + throw new Error( + `Object metadata decorator not found, can't parse ${target.name}`, + ); + } + + const workspaceIndexMetadataArgsCollection = + metadataArgsStorage.filterIndexes(target); + + return workspaceIndexMetadataArgsCollection.map( + (workspaceIndexMetadataArgs) => { + const objectMetadata = + originalObjectMetadataMap[workspaceEntity.nameSingular]; + + if (!objectMetadata) { + throw new Error( + `Object metadata not found for ${workspaceEntity.nameSingular}`, + ); + } + + const indexMetadata: PartialIndexMetadata = { + workspaceId: context.workspaceId, + objectMetadataId: objectMetadata.id, + name: workspaceIndexMetadataArgs.name, + columns: workspaceIndexMetadataArgs.columns, + }; + + return indexMetadata; + }, + ); + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts index efc061bc7..ab40fdfa7 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface.ts @@ -1,5 +1,6 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ComputedPartialFieldMetadata } from './partial-field-metadata.interface'; import { ComputedPartialWorkspaceEntity } from './partial-object-metadata.interface'; @@ -49,3 +50,8 @@ export type RelationComparatorResult = | ComparatorCreateResult> | ComparatorDeleteResult | ComparatorUpdateResult>; + +export type IndexComparatorResult = + | ComparatorCreateResult> + | ComparatorUpdateResult> + | ComparatorDeleteResult; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts new file mode 100644 index 000000000..7b174d6fb --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts @@ -0,0 +1,8 @@ +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; + +export type PartialIndexMetadata = Omit< + IndexMetadataEntity, + 'id' | 'objectMetadata' | 'indexFieldMetadatas' +> & { + columns: string[]; +}; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts index 5ba73cbcc..4e6c0e117 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts @@ -11,6 +11,7 @@ import { v4 as uuidV4 } from 'uuid'; import { DeepPartial } from 'typeorm/common/DeepPartial'; import { PartialFieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-field-metadata.interface'; +import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { @@ -22,6 +23,7 @@ import { FieldMetadataComplexOption } from 'src/engine/metadata-modules/field-me import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; import { FieldMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory'; import { ObjectMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-object.factory'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; @Injectable() export class WorkspaceMetadataUpdaterService { @@ -230,6 +232,69 @@ export class WorkspaceMetadataUpdaterService { }; } + async updateIndexMetadata( + manager: EntityManager, + storage: WorkspaceSyncStorage, + originalObjectMetadataCollection: ObjectMetadataEntity[], + ): Promise<{ + createdIndexMetadataCollection: IndexMetadataEntity[]; + }> { + const indexMetadataRepository = manager.getRepository(IndexMetadataEntity); + + const convertIndexMetadataForSaving = ( + indexMetadata: PartialIndexMetadata, + ) => { + const convertIndexFieldMetadataForSaving = ( + column: string, + order: number, + ) => { + const fieldMetadata = originalObjectMetadataCollection + .find((object) => object.id === indexMetadata.objectMetadataId) + ?.fields.find((field) => column === field.name); + + if (!fieldMetadata) { + throw new Error(` + Field metadata not found for column ${column} in object ${indexMetadata.objectMetadataId} + `); + } + + return { + fieldMetadataId: fieldMetadata.id, + order, + }; + }; + + return { + ...indexMetadata, + indexFieldMetadatas: indexMetadata.columns.map((column, index) => + convertIndexFieldMetadataForSaving(column, index), + ), + }; + }; + + /** + * Create index metadata + */ + const createdIndexMetadataCollection = await indexMetadataRepository.save( + storage.indexMetadataCreateCollection.map(convertIndexMetadataForSaving), + ); + + /** + * Delete index metadata + */ + if (storage.indexMetadataDeleteCollection.length > 0) { + await indexMetadataRepository.delete( + storage.indexMetadataDeleteCollection.map( + (indexMetadata) => indexMetadata.id, + ), + ); + } + + return { + createdIndexMetadataCollection, + }; + } + /** * Update entities in the database * @param manager EntityManager diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts index 4fe25b628..e590900bc 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts @@ -48,7 +48,7 @@ export class WorkspaceSyncFieldMetadataService { relations: ['dataSource', 'fields'], }); - // Filter out custom objects + // Filter out non-custom objects const customObjectMetadataCollection = originalObjectMetadataCollection.filter( (objectMetadata) => objectMetadata.isCustom, @@ -61,7 +61,7 @@ export class WorkspaceSyncFieldMetadataService { workspaceFeatureFlagsMap, ); - // Loop over all standard objects and compare them with the objects in DB + // Loop over all custom objects from the DB and compare their fields with standard fields for (const customObjectMetadata of customObjectMetadataCollection) { // Also, maybe it's better to refactor a bit and move generation part into a separate module ? const standardObjectMetadata = computeStandardObject( diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts new file mode 100644 index 000000000..4f7a30ef9 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts @@ -0,0 +1,119 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { EntityManager } from 'typeorm'; + +import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface'; +import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; +import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; +import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface'; + +import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; +import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { mapObjectMetadataByUniqueIdentifier } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/sync-metadata.util'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects'; +import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator'; +import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service'; +import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory'; + +@Injectable() +export class WorkspaceSyncIndexMetadataService { + private readonly logger = new Logger(WorkspaceSyncIndexMetadataService.name); + + constructor( + private readonly standardIndexFactory: StandardIndexFactory, + private readonly workspaceIndexComparator: WorkspaceIndexComparator, + private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService, + private readonly workspaceMigrationIndexFactory: WorkspaceMigrationIndexFactory, + ) {} + + async synchronize( + context: WorkspaceSyncContext, + manager: EntityManager, + storage: WorkspaceSyncStorage, + workspaceFeatureFlagsMap: FeatureFlagMap, + ): Promise[]> { + this.logger.log('Syncing index metadata'); + + const objectMetadataRepository = + manager.getRepository(ObjectMetadataEntity); + + // Retrieve object metadata collection from DB + const originalObjectMetadataCollection = + await objectMetadataRepository.find({ + where: { + workspaceId: context.workspaceId, + // We're only interested in standard fields + fields: { isCustom: false }, + isCustom: false, + }, + relations: ['dataSource', 'fields', 'indexes'], + }); + + // Create map of object metadata & field metadata by unique identifier + const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier( + originalObjectMetadataCollection, + // Relation are based on the singular name + (objectMetadata) => objectMetadata.nameSingular, + ); + + const indexMetadataRepository = manager.getRepository(IndexMetadataEntity); + + const originalIndexMetadataCollection = await indexMetadataRepository.find({ + where: { + workspaceId: context.workspaceId, + }, + relations: ['indexFieldMetadatas.fieldMetadata'], + }); + + // Generate index metadata from models + const standardIndexMetadataCollection = this.standardIndexFactory.create( + standardObjectMetadataDefinitions, + context, + originalObjectMetadataMap, + workspaceFeatureFlagsMap, + ); + + const indexComparatorResults = this.workspaceIndexComparator.compare( + originalIndexMetadataCollection, + standardIndexMetadataCollection, + ); + + for (const indexComparatorResult of indexComparatorResults) { + if (indexComparatorResult.action === ComparatorAction.CREATE) { + storage.addCreateIndexMetadata(indexComparatorResult.object); + } else if (indexComparatorResult.action === ComparatorAction.DELETE) { + storage.addDeleteIndexMetadata(indexComparatorResult.object); + } + } + + const metadataIndexUpdaterResult = + await this.workspaceMetadataUpdaterService.updateIndexMetadata( + manager, + storage, + originalObjectMetadataCollection, + ); + + // Create migrations + const createIndexWorkspaceMigrations = + await this.workspaceMigrationIndexFactory.create( + originalObjectMetadataCollection, + metadataIndexUpdaterResult.createdIndexMetadataCollection, + WorkspaceMigrationBuilderAction.CREATE, + ); + + const deleteIndexWorkspaceMigrations = + await this.workspaceMigrationIndexFactory.create( + originalObjectMetadataCollection, + storage.indexMetadataDeleteCollection, + WorkspaceMigrationBuilderAction.DELETE, + ); + + return [ + ...createIndexWorkspaceMigrations, + ...deleteIndexWorkspaceMigrations, + ]; + } +} diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts index 147389b18..7d809b3f8 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts @@ -112,6 +112,10 @@ export class WorkspaceSyncObjectMetadataService { /** * COMPARE FIELD METADATA + * NOTE: This should be moved to WorkspaceSyncFieldMetadataService for more clarity since + * this code only adds field metadata to the storage but it's actually used in the other service. + * NOTE2: WorkspaceSyncFieldMetadataService has been added for custom fields sync, it should be refactored to handle + * both custom and non-custom fields. */ const fieldComparatorResults = this.workspaceFieldComparator.compare( originalObjectMetadata, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts index 6d0a7a9d3..5cf0fd60a 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage.ts @@ -4,6 +4,7 @@ import { ComputedPartialFieldMetadata } from 'src/engine/workspace-manager/works import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; export class WorkspaceSyncStorage { // Object metadata @@ -25,10 +26,17 @@ export class WorkspaceSyncStorage { // Relation metadata private readonly _relationMetadataCreateCollection: Partial[] = []; - private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] = - []; private readonly _relationMetadataUpdateCollection: Partial[] = []; + private readonly _relationMetadataDeleteCollection: RelationMetadataEntity[] = + []; + + // Index metadata + private readonly _indexMetadataCreateCollection: Partial[] = + []; + private readonly _indexMetadataUpdateCollection: Partial[] = + []; + private readonly _indexMetadataDeleteCollection: IndexMetadataEntity[] = []; constructor() {} @@ -68,6 +76,18 @@ export class WorkspaceSyncStorage { return this._relationMetadataDeleteCollection; } + get indexMetadataCreateCollection() { + return this._indexMetadataCreateCollection; + } + + get indexMetadataUpdateCollection() { + return this._indexMetadataUpdateCollection; + } + + get indexMetadataDeleteCollection() { + return this._indexMetadataDeleteCollection; + } + addCreateObjectMetadata(object: ComputedPartialWorkspaceEntity) { this._objectMetadataCreateCollection.push(object); } @@ -107,4 +127,16 @@ export class WorkspaceSyncStorage { addDeleteRelationMetadata(relation: RelationMetadataEntity) { this._relationMetadataDeleteCollection.push(relation); } + + addCreateIndexMetadata(index: Partial) { + this._indexMetadataCreateCollection.push(index); + } + + addUpdateIndexMetadata(index: Partial) { + this._indexMetadataUpdateCollection.push(index); + } + + addDeleteIndexMetadata(index: IndexMetadataEntity) { + this._indexMetadataDeleteCollection.push(index); + } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts index 7e8d046cb..951d9e74a 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts @@ -16,6 +16,7 @@ import { WorkspaceSyncRelationMetadataService } from 'src/engine/workspace-manag import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service'; import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module'; import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service'; @Module({ imports: [ @@ -41,6 +42,7 @@ import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspa WorkspaceSyncRelationMetadataService, WorkspaceSyncFieldMetadataService, WorkspaceSyncMetadataService, + WorkspaceSyncIndexMetadataService, ], exports: [...workspaceSyncMetadataFactories, WorkspaceSyncMetadataService], }) diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index 8b4db8d36..b233be4e3 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -13,6 +13,7 @@ import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/ import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceSyncIndexMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service'; interface SynchronizeOptions { applyChanges?: boolean; @@ -31,6 +32,7 @@ export class WorkspaceSyncMetadataService { private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService, private readonly workspaceSyncFieldMetadataService: WorkspaceSyncFieldMetadataService, private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceSyncIndexMetadataService: WorkspaceSyncIndexMetadataService, ) {} /** @@ -97,11 +99,21 @@ export class WorkspaceSyncMetadataService { workspaceFeatureFlagsMap, ); + // 4 - Sync standard indexes on standard objects + const workspaceIndexMigrations = + await this.workspaceSyncIndexMetadataService.synchronize( + context, + manager, + storage, + workspaceFeatureFlagsMap, + ); + // Save workspace migrations into the database workspaceMigrations = await workspaceMigrationRepository.save([ ...workspaceObjectMigrations, ...workspaceFieldMigrations, ...workspaceRelationMigrations, + ...workspaceIndexMigrations, ]); // If we're running a dry run, rollback the transaction and do not execute migrations