From 6af1bcd55c2e83ad76645d78fb591f2c0d4cffd2 Mon Sep 17 00:00:00 2001 From: Weiko Date: Tue, 9 Jul 2024 14:51:24 +0200 Subject: [PATCH] [Flexible Schema] Create indexes for join columns (#6165) ## Context We want to add an index on our foreign keys since PG does not do it for us. An index can sometimes be expensive and not always meaningful depending on different usages but in our case we decided to apply an index for every foreign keys. ```typescript @WorkspaceIndex() @WorkspaceJoinColumn('author') authorId: string; ``` This syntax is valid but since we want to apply it to every join column I've decided to update the code of WorkspaceJoinColumn so it properly registers a new index at the same time which is less error-prone. Note: We had a bug on index name generation since postgres index names are unique per schema and not table, the object metadata id (hashed) has been added to the formula that generates the name of the index ## Test Sync metadata. We have 45 join columns as of today per workspace, we should see 45 rows inside IndexMetadata table --- ...524654925-addDateColumnsToIndexMetadata.ts | 37 +++++++++++++++++++ .../index-field-metadata.entity.ts | 14 +++++-- .../index-metadata/index-metadata.entity.ts | 16 ++++++-- .../decorators/workspace-index.decorator.ts | 12 ++++-- .../workspace-join-column.decorator.ts | 4 ++ .../factories/standard-index.factory.ts | 6 +-- .../partial-index-metadata.interface.ts | 2 +- 7 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/metadata/migrations/1720524654925-addDateColumnsToIndexMetadata.ts diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1720524654925-addDateColumnsToIndexMetadata.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1720524654925-addDateColumnsToIndexMetadata.ts new file mode 100644 index 000000000..6753c5c01 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1720524654925-addDateColumnsToIndexMetadata.ts @@ -0,0 +1,37 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddDateColumnsToIndexMetadata1720524654925 + implements MigrationInterface +{ + name = 'AddDateColumnsToIndexMetadata1720524654925'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" ADD "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" ADD "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" ADD "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" ADD "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" DROP COLUMN "updatedAt"`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexFieldMetadata" DROP COLUMN "createdAt"`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" DROP COLUMN "updatedAt"`, + ); + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" DROP COLUMN "createdAt"`, + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts index 79ce331c7..036bee830 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity.ts @@ -1,14 +1,16 @@ import { - Entity, - PrimaryGeneratedColumn, Column, + CreateDateColumn, + Entity, JoinColumn, ManyToOne, + PrimaryGeneratedColumn, Relation, + UpdateDateColumn, } from 'typeorm'; -import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; @Entity('indexFieldMetadata') export class IndexFieldMetadataEntity { @@ -43,4 +45,10 @@ export class IndexFieldMetadataEntity { @Column({ nullable: false }) order: number; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt: Date; + + @UpdateDateColumn({ type: 'timestamptz' }) + updatedAt: Date; } diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts index d006a092e..74b15a10d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts @@ -1,15 +1,17 @@ import { - Entity, - PrimaryGeneratedColumn, Column, + CreateDateColumn, + Entity, JoinColumn, ManyToOne, - Relation, OneToMany, + PrimaryGeneratedColumn, + Relation, + UpdateDateColumn, } from 'typeorm'; -import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-field-metadata/index-field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; @Entity('indexMetadata') export class IndexMetadataEntity { @@ -40,4 +42,10 @@ export class IndexMetadataEntity { }, ) indexFieldMetadatas: Relation; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt: Date; + + @UpdateDateColumn({ type: 'timestamptz' }) + updatedAt: Date; } diff --git a/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts index e74394c2c..0ba01f8e0 100644 --- a/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts +++ b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-index.decorator.ts @@ -1,5 +1,6 @@ import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name'; import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage'; +import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util'; export interface WorkspaceIndexOptions { columns?: string[]; @@ -16,11 +17,13 @@ export function WorkspaceIndex( } // TODO: handle composite field metadata types - // TODO: handle relation field metadata types if (Array.isArray(columns) && columns.length > 0) { metadataArgsStorage.addIndexes({ - name: `IDX_${generateDeterministicIndexName(columns)}`, + name: `IDX_${generateDeterministicIndexName([ + convertClassNameToObjectMetadataName(target.name), + ...columns, + ])}`, columns, target: target, }); @@ -29,7 +32,10 @@ export function WorkspaceIndex( } metadataArgsStorage.addIndexes({ - name: `IDX_${generateDeterministicIndexName([propertyKey.toString()])}`, + name: `IDX_${generateDeterministicIndexName([ + convertClassNameToObjectMetadataName(target.constructor.name), + propertyKey.toString(), + ])}`, columns: [propertyKey.toString()], target: target.constructor, }); diff --git a/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-join-column.decorator.ts b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-join-column.decorator.ts index 4d1326f9b..98dc6eedb 100644 --- a/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-join-column.decorator.ts +++ b/packages/twenty-server/src/engine/twenty-orm/decorators/workspace-join-column.decorator.ts @@ -1,3 +1,4 @@ +import { WorkspaceIndex } from 'src/engine/twenty-orm/decorators/workspace-index.decorator'; import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage'; export function WorkspaceJoinColumn( @@ -9,5 +10,8 @@ export function WorkspaceJoinColumn( relationName: relationPropertyKey, joinColumn: propertyKey.toString(), }); + + // Register index for join column + WorkspaceIndex()(object, propertyKey); }; } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts index 8ad2c89d1..83e450cbe 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface'; -import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface'; +import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity'; import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity'; import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage'; @Injectable() @@ -31,7 +31,7 @@ export class StandardIndexFactory { target: typeof BaseWorkspaceEntity, context: WorkspaceSyncContext, originalObjectMetadataMap: Record, - workspaceFeatureFlagsMap: FeatureFlagMap, + _workspaceFeatureFlagsMap: FeatureFlagMap, ): Partial[] { const workspaceEntity = metadataArgsStorage.filterEntities(target); diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts index 7b174d6fb..26939eaa7 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface.ts @@ -2,7 +2,7 @@ import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/ export type PartialIndexMetadata = Omit< IndexMetadataEntity, - 'id' | 'objectMetadata' | 'indexFieldMetadatas' + 'id' | 'objectMetadata' | 'indexFieldMetadatas' | 'createdAt' | 'updatedAt' > & { columns: string[]; };