From 1de739176c47ae3f3b05c9ee80cbc87059d85975 Mon Sep 17 00:00:00 2001 From: Marie <51697796+ijreilly@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:34:05 +0200 Subject: [PATCH] Update searchVector at label identifier update for custom fields (#7588) By default, when custom fields are created, a searchVector field is created based on the "name" field, which is also the label identifier by default. When this label identifier is updated, we want to update the searchVector field to use this field as searchable field instead, if it is of "searchable type" (today it is only possible to select a text or number field as label identifier, while number fields are not searchable). --- ...bjectMetadataItemsToObjectMetadataItems.ts | 2 +- .../modules/workspace/types/FeatureFlagKey.ts | 2 - .../commands/database-command.module.ts | 2 + ...implify-search-vector-expression.module.ts | 21 +++ .../0-32-simplify-search-vector-expression.ts | 115 ++++++++++++ .../typeorm-seeds/core/feature-flags.ts | 10 -- ...8999374151-addConstraintOnIndexMetadata.ts | 17 ++ .../graphql-query-search-resolver.service.ts | 22 +-- .../enums/feature-flag-key.enum.ts | 2 - .../field-metadata/field-metadata.service.ts | 2 - .../index-metadata/index-metadata.entity.ts | 6 + .../index-metadata/index-metadata.service.ts | 43 +++-- .../object-metadata/object-metadata.module.ts | 4 +- .../object-metadata.service.ts | 127 ++++--------- .../metadata-modules/search/search.module.ts | 23 +++ .../metadata-modules/search/search.service.ts | 169 ++++++++++++++++++ .../ts-vector-column-action.factory.ts | 35 ++-- .../twenty-orm/custom.workspace-entity.ts | 20 ++- .../workspace-migration-runner.service.ts | 2 + .../constants/default-feature-flags.ts | 7 +- .../workspace-sync-field-metadata.service.ts | 31 +--- .../workspace-sync-index-metadata.service.ts | 18 +- ...ts-vectors-column-expression.utils.spec.ts | 13 +- .../get-ts-vector-column-expression.util.ts | 16 +- .../utils/is-searchable-field.util.ts | 17 ++ .../workspace-sync-metadata.service.ts | 8 - .../company.workspace-entity.ts | 17 +- .../opportunity.workspace-entity.ts | 15 +- .../person.workspace-entity.ts | 19 +- 29 files changed, 535 insertions(+), 250 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts create mode 100644 packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/search/search.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/search/search.service.ts create mode 100644 packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts diff --git a/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts b/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts index db3506c2d..0ab2a8229 100644 --- a/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts +++ b/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts @@ -11,7 +11,7 @@ export const mapPaginatedObjectMetadataItemsToObjectMetadataItems = ({ pagedObjectMetadataItems?.objects.edges.map((object) => ({ ...object.node, fields: object.node.fields.edges.map((field) => field.node), - indexMetadatas: object.node.indexMetadatas.edges.map((index) => ({ + indexMetadatas: object.node.indexMetadatas?.edges.map((index) => ({ ...index.node, indexFieldMetadatas: index.node.indexFieldMetadatas?.edges.map( (indexField) => indexField.node, diff --git a/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts b/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts index cde449770..5471c5d4d 100644 --- a/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts +++ b/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts @@ -10,9 +10,7 @@ export type FeatureFlagKey = | 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED' | 'IS_WORKFLOW_ENABLED' | 'IS_WORKSPACE_FAVORITE_ENABLED' - | 'IS_SEARCH_ENABLED' | 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED' | 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED' - | 'IS_WORKSPACE_MIGRATED_FOR_SEARCH' | 'IS_ANALYTICS_V2_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED'; diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index f8207c318..808785d9f 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -7,6 +7,7 @@ import { DataSeedDemoWorkspaceCommand } from 'src/database/commands/data-seed-de import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module'; import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command'; import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; +import { SimplifySearchVectorExpressionCommandModule } from 'src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module'; import { UpgradeTo0_32CommandModule } from 'src/database/commands/upgrade-version/0-32/0-32-upgrade-version.module'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity'; @@ -46,6 +47,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp DataSeedDemoWorkspaceModule, WorkspaceCacheStorageModule, WorkspaceMetadataVersionModule, + SimplifySearchVectorExpressionCommandModule, UpgradeTo0_32CommandModule, FeatureFlagModule, ], diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts new file mode 100644 index 000000000..9e6ea5e2b --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { SimplifySearchVectorExpressionCommand } from 'src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { SearchModule } from 'src/engine/metadata-modules/search/search.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; +import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([Workspace], 'core'), + TypeOrmModule.forFeature([FieldMetadataEntity], 'metadata'), + WorkspaceSyncMetadataCommandsModule, + SearchModule, + WorkspaceMigrationRunnerModule, + ], + providers: [SimplifySearchVectorExpressionCommand], +}) +export class SimplifySearchVectorExpressionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts new file mode 100644 index 000000000..27d75e213 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts @@ -0,0 +1,115 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import chalk from 'chalk'; +import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; + +import { + ActiveWorkspacesCommandOptions, + ActiveWorkspacesCommandRunner, +} from 'src/database/commands/active-workspaces.command'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; +import { SEARCH_FIELDS_FOR_CUSTOM_OBJECT } from 'src/engine/twenty-orm/custom.workspace-entity'; +import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; +import { + COMPANY_STANDARD_FIELD_IDS, + CUSTOM_OBJECT_STANDARD_FIELD_IDS, + OPPORTUNITY_STANDARD_FIELD_IDS, + PERSON_STANDARD_FIELD_IDS, +} from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; +import { FieldTypeAndNameMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { SEARCH_FIELDS_FOR_COMPANY } from 'src/modules/company/standard-objects/company.workspace-entity'; +import { SEARCH_FIELDS_FOR_OPPORTUNITY } from 'src/modules/opportunity/standard-objects/opportunity.workspace-entity'; +import { SEARCH_FIELDS_FOR_PERSON } from 'src/modules/person/standard-objects/person.workspace-entity'; + +@Command({ + name: 'fix-0.32:simplify-search-vector-expression', + description: 'Replace searchVector with simpler expression', +}) +export class SimplifySearchVectorExpressionCommand extends ActiveWorkspacesCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, + private readonly searchService: SearchService, + private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, + ) { + super(workspaceRepository); + } + + async executeActiveWorkspacesCommand( + _passedParam: string[], + _options: ActiveWorkspacesCommandOptions, + workspaceIds: string[], + ): Promise { + this.logger.log('Running command to fix migration'); + + for (const workspaceId of workspaceIds) { + this.logger.log(`Running command for workspace ${workspaceId}`); + + try { + const searchVectorFields = await this.fieldMetadataRepository.findBy({ + workspaceId: workspaceId, + type: FieldMetadataType.TS_VECTOR, + }); + + for (const searchVectorField of searchVectorFields) { + let fieldsUsedForSearch: FieldTypeAndNameMetadata[] = []; + + switch (searchVectorField.standardId) { + case CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_CUSTOM_OBJECT; + break; + } + case PERSON_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_PERSON; + break; + } + case COMPANY_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_COMPANY; + break; + } + case OPPORTUNITY_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_OPPORTUNITY; + break; + } + default: { + throw new Error( + `search vector has unexpected standardId: ${searchVectorField.standardId}`, + ); + } + } + + await this.searchService.updateSearchVector( + searchVectorField.objectMetadataId, + fieldsUsedForSearch, + workspaceId, + ); + + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + } + } catch (error) { + this.logger.log( + chalk.red( + `Running command on workspace ${workspaceId} failed with error: ${error}`, + ), + ); + continue; + } finally { + this.logger.log( + chalk.green(`Finished running command for workspace ${workspaceId}.`), + ); + } + + this.logger.log(chalk.green(`Command completed!`)); + } + } +} diff --git a/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts b/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts index c618677d3..97b34a884 100644 --- a/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts +++ b/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts @@ -55,16 +55,6 @@ export const seedFeatureFlags = async ( workspaceId: workspaceId, value: true, }, - { - key: FeatureFlagKey.IsSearchEnabled, - workspaceId: workspaceId, - value: true, - }, - { - key: FeatureFlagKey.IsWorkspaceMigratedForSearch, - workspaceId: workspaceId, - value: true, - }, { key: FeatureFlagKey.IsAnalyticsV2Enabled, workspaceId: workspaceId, diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts new file mode 100644 index 000000000..9659750f8 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddConstraintOnIndex1728999374151 implements MigrationInterface { + name = 'AddConstraintOnIndexMetadata1728999374151'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" ADD CONSTRAINT "IndexOnNameAndWorkspaceIdAndObjectMetadataUnique" UNIQUE ("name", "workspaceId", "objectMetadataId")`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" DROP CONSTRAINT "IndexOnNameAndWorkspaceIdAndObjectMetadataUnique"`, + ); + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts index a0fdbf0d3..cfd570281 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts @@ -10,10 +10,6 @@ import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-qu import { SearchResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant'; -import { - GraphqlQueryRunnerException, - GraphqlQueryRunnerExceptionCode, -} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; @@ -100,20 +96,6 @@ export class GraphqlQuerySearchResolverService async validate( _args: SearchResolverArgs, - options: WorkspaceQueryRunnerOptions, - ): Promise { - const featureFlagsForWorkspace = - await this.featureFlagService.getWorkspaceFeatureFlags( - options.authContext.workspace.id, - ); - - const isSearchEnabled = featureFlagsForWorkspace.IS_SEARCH_ENABLED; - - if (!isSearchEnabled) { - throw new GraphqlQueryRunnerException( - 'This endpoint is not available yet, please use findMany instead.', - GraphqlQueryRunnerExceptionCode.INVALID_QUERY_INPUT, - ); - } - } + _options: WorkspaceQueryRunnerOptions, + ): Promise {} } diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts index 8483f3089..58282f9de 100644 --- a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts @@ -10,8 +10,6 @@ export enum FeatureFlagKey { IsMessageThreadSubscriberEnabled = 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED', IsQueryRunnerTwentyORMEnabled = 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED', IsWorkspaceFavoriteEnabled = 'IS_WORKSPACE_FAVORITE_ENABLED', - IsSearchEnabled = 'IS_SEARCH_ENABLED', - IsWorkspaceMigratedForSearch = 'IS_WORKSPACE_MIGRATED_FOR_SEARCH', IsGmailSendEmailScopeEnabled = 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED', IsAnalyticsV2Enabled = 'IS_ANALYTICS_V2_ENABLED', IsUniqueIndexesEnabled = 'IS_UNIQUE_INDEXES_ENABLED', diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index f402f5b0b..4291fe44a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -29,7 +29,6 @@ import { import { generateNullable } from 'src/engine/metadata-modules/field-metadata/utils/generate-nullable'; import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util'; import { RelationMetadataEntity, @@ -76,7 +75,6 @@ export class FieldMetadataService extends TypeOrmQueryService, @InjectRepository(ObjectMetadataEntity, 'metadata') private readonly objectMetadataRepository: Repository, - private readonly objectMetadataService: ObjectMetadataService, private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, private readonly workspaceMigrationService: WorkspaceMigrationService, private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts index fc2f991ce..b748e2ea5 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts @@ -7,6 +7,7 @@ import { OneToMany, PrimaryGeneratedColumn, Relation, + Unique, UpdateDateColumn, } from 'typeorm'; @@ -18,6 +19,11 @@ export enum IndexType { GIN = 'GIN', } +@Unique('IndexOnNameAndWorkspaceIdAndObjectMetadataUnique', [ + 'name', + 'workspaceId', + 'objectMetadataId', +]) @Entity('indexMetadata') export class IndexMetadataEntity { @PrimaryGeneratedColumn('uuid') diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts index 362519ef7..1d75cbd7a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { isDefined } from 'class-validator'; -import { Repository } from 'typeorm'; +import { InsertResult, Repository } from 'typeorm'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { @@ -45,32 +45,37 @@ export class IndexMetadataService { const indexName = `IDX_${generateDeterministicIndexName([tableName, ...columnNames])}`; - let savedIndexMetadata: IndexMetadataEntity; + let result: InsertResult; try { - savedIndexMetadata = await this.indexMetadataRepository.save({ - name: indexName, - tableName, - indexFieldMetadatas: fieldMetadataToIndex.map( - (fieldMetadata, index) => { - return { - fieldMetadataId: fieldMetadata.id, - order: index, - }; - }, - ), - workspaceId, - objectMetadataId: objectMetadata.id, - ...(isDefined(indexType) ? { indexType: indexType } : {}), - isCustom: isCustom, - }); + result = await this.indexMetadataRepository.upsert( + { + name: indexName, + indexFieldMetadatas: fieldMetadataToIndex.map( + (fieldMetadata, index) => { + return { + fieldMetadataId: fieldMetadata.id, + order: index, + }; + }, + ), + workspaceId, + objectMetadataId: objectMetadata.id, + ...(isDefined(indexType) ? { indexType: indexType } : {}), + isCustom: isCustom, + }, + { + conflictPaths: ['workspaceId', 'name', 'objectMetadataId'], + skipUpdateIfNoValuesChanged: true, + }, + ); } catch (error) { throw new Error( `Failed to create index ${indexName} on object metadata ${objectMetadata.nameSingular}`, ); } - if (!savedIndexMetadata) { + if (!result.identifiers.length) { throw new Error( `Failed to return saved index ${indexName} on object metadata ${objectMetadata.nameSingular}`, ); diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts index 14d9d58c2..46992a6e1 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts @@ -14,12 +14,12 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature- import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module'; import { BeforeUpdateOneObject } from 'src/engine/metadata-modules/object-metadata/hooks/before-update-one-object.hook'; import { ObjectMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/object-metadata/interceptors/object-metadata-graphql-api-exception.interceptor'; import { ObjectMetadataResolver } from 'src/engine/metadata-modules/object-metadata/object-metadata.resolver'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { RemoteTableRelationsModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.module'; +import { SearchModule } from 'src/engine/metadata-modules/search/search.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @@ -46,8 +46,8 @@ import { UpdateObjectPayload } from './dtos/update-object.input'; WorkspaceMigrationRunnerModule, WorkspaceMetadataVersionModule, RemoteTableRelationsModule, - IndexMetadataModule, FeatureFlagModule, + SearchModule, ], services: [ObjectMetadataService], resolvers: [ diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index 83db047cd..8ec403f84 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -5,30 +5,20 @@ import console from 'console'; import { Query, QueryOptions } from '@ptc-org/nestjs-query-core'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; -import { isDefined } from 'class-validator'; import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; -import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; -import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { FieldMetadataEntity, FieldMetadataType, } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { - computeColumnName, - FieldTypeAndNameMetadata, -} from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; -import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; -import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service'; +import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; import { DeleteOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/delete-object.input'; import { UpdateOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/update-object.input'; -import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants'; import { ObjectMetadataException, ObjectMetadataExceptionCode, @@ -43,8 +33,8 @@ import { import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete'; import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service'; import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; -import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, @@ -72,7 +62,7 @@ import { createForeignKeyDeterministicUuid, createRelationDeterministicUuid, } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { isSearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; @@ -94,17 +84,14 @@ export class ObjectMetadataService extends TypeOrmQueryService - field.id === createdObjectMetadata.labelIdentifierFieldMetadataId, - ) - : createdObjectMetadata.fields.find( - (field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, - ); - - if (!isDefined(searchableFieldForCustomObject)) { - throw new Error('No searchable field found for custom object'); - } - - this.workspaceMigrationService.createCustomMigration( - generateMigrationName( - `update-${createdObjectMetadata.nameSingular}-add-searchVector`, - ), - createdObjectMetadata.workspaceId, - [ - { - name: computeTableName( - createdObjectMetadata.nameSingular, - createdObjectMetadata.isCustom, - ), - action: WorkspaceMigrationTableActionType.ALTER, - columns: this.tsVectorColumnActionFactory.handleCreateAction({ - ...searchVectorFieldMetadata, - defaultValue: undefined, - generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - searchableFieldForCustomObject as FieldTypeAndNameMetadata, - ]), - options: undefined, - } as FieldMetadataInterface), - }, - ], - ); - - await this.indexMetadataService.createIndex( - objectMetadataInput.workspaceId, - createdObjectMetadata, - [searchVectorFieldMetadata], - false, - false, - IndexType.GIN, - ); - } - private async createActivityTargetRelation( workspaceId: string, createdObjectMetadata: ObjectMetadataEntity, diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts new file mode 100644 index 000000000..2260d7218 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; + +import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; + +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; + +@Module({ + imports: [ + NestjsQueryTypeOrmModule.forFeature( + [ObjectMetadataEntity, FieldMetadataEntity], + 'metadata', + ), + IndexMetadataModule, + WorkspaceMigrationModule, + ], + providers: [SearchService], + exports: [SearchService], +}) +export class SearchModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts new file mode 100644 index 000000000..b24ad3816 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts @@ -0,0 +1,169 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; + +import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service'; +import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input'; +import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory'; +import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; +import { + WorkspaceMigrationColumnActionType, + WorkspaceMigrationTableActionType, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceMigrationFactory } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.factory'; +import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; +import { computeTableName } from 'src/engine/utils/compute-table-name.util'; +import { CUSTOM_OBJECT_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { SearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; +import { isDefined } from 'src/utils/is-defined'; + +@Injectable() +export class SearchService { + constructor( + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, + private readonly tsVectorColumnActionFactory: TsVectorColumnActionFactory, + private readonly indexMetadataService: IndexMetadataService, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, + private readonly workspaceMigrationService: WorkspaceMigrationService, + private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, + ) {} + + public async createSearchVectorFieldForObject( + objectMetadataInput: CreateObjectInput, + createdObjectMetadata: ObjectMetadataEntity, + ) { + const searchVectorFieldMetadata = await this.fieldMetadataRepository.save({ + standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector, + objectMetadataId: createdObjectMetadata.id, + workspaceId: objectMetadataInput.workspaceId, + isCustom: false, + isActive: false, + isSystem: true, + type: FieldMetadataType.TS_VECTOR, + name: SEARCH_VECTOR_FIELD.name, + label: SEARCH_VECTOR_FIELD.label, + description: SEARCH_VECTOR_FIELD.description, + isNullable: true, + }); + + const searchableFieldForCustomObject = + createdObjectMetadata.labelIdentifierFieldMetadataId + ? createdObjectMetadata.fields.find( + (field) => + field.id === createdObjectMetadata.labelIdentifierFieldMetadataId, + ) + : createdObjectMetadata.fields.find( + (field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, + ); + + if (!isDefined(searchableFieldForCustomObject)) { + throw new Error( + `No searchable field found for custom object (object name: ${createdObjectMetadata.nameSingular})`, + ); + } + + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`create-${createdObjectMetadata.nameSingular}`), + createdObjectMetadata.workspaceId, + [ + { + name: computeTableName( + createdObjectMetadata.nameSingular, + createdObjectMetadata.isCustom, + ), + action: WorkspaceMigrationTableActionType.ALTER, + columns: this.tsVectorColumnActionFactory.handleCreateAction({ + ...searchVectorFieldMetadata, + defaultValue: undefined, + generatedType: 'STORED', + asExpression: getTsVectorColumnExpressionFromFields([ + { + type: searchableFieldForCustomObject.type as SearchableFieldType, + name: searchableFieldForCustomObject.name, + }, + ]), + options: undefined, + } as FieldMetadataInterface), + }, + ], + ); + + await this.indexMetadataService.createIndex( + objectMetadataInput.workspaceId, + createdObjectMetadata, + [searchVectorFieldMetadata], + false, + false, + IndexType.GIN, + ); + } + + public async updateSearchVector( + objectMetadataId: string, + fieldMetadataNameAndTypeForSearch: FieldTypeAndNameMetadata[], + workspaceId: string, + ) { + const objectMetadata = await this.objectMetadataRepository.findOneByOrFail({ + id: objectMetadataId, + }); + + const existingSearchVectorFieldMetadata = + await this.fieldMetadataRepository.findOneByOrFail({ + name: SEARCH_VECTOR_FIELD.name, + objectMetadataId, + }); + + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`update-${objectMetadata.nameSingular}`), + workspaceId, + [ + { + name: computeTableName( + objectMetadata.nameSingular, + objectMetadata.isCustom, + ), + action: WorkspaceMigrationTableActionType.ALTER, + columns: this.workspaceMigrationFactory.createColumnActions( + WorkspaceMigrationColumnActionType.ALTER, + existingSearchVectorFieldMetadata, + { + ...existingSearchVectorFieldMetadata, + asExpression: getTsVectorColumnExpressionFromFields( + fieldMetadataNameAndTypeForSearch, + ), + generatedType: 'STORED', // Not stored on fieldMetadata + options: undefined, + }, + ), + }, + ], + ); + + // index needs to be recreated as typeorm deletes then recreates searchVector column at alter + await this.indexMetadataService.createIndex( + workspaceId, + objectMetadata, + [existingSearchVectorFieldMetadata], + false, + false, + IndexType.GIN, + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts index fa6aea201..71f1265b7 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts @@ -1,7 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; -import { WorkspaceColumnActionOptions } from 'src/engine/metadata-modules/workspace-migration/interfaces/workspace-column-action-options.interface'; import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; @@ -12,10 +11,6 @@ import { WorkspaceMigrationColumnAlter, WorkspaceMigrationColumnCreate, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; -import { - WorkspaceMigrationException, - WorkspaceMigrationExceptionCode, -} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception'; export type TsVectorFieldMetadataType = FieldMetadataType.TS_VECTOR; @@ -40,14 +35,28 @@ export class TsVectorColumnActionFactory extends ColumnActionAbstractFactory, - _alteredFieldMetadata: FieldMetadataInterface, - _options?: WorkspaceColumnActionOptions, + handleAlterAction( + currentFieldMetadata: FieldMetadataInterface, + alteredFieldMetadata: FieldMetadataInterface, ): WorkspaceMigrationColumnAlter[] { - throw new WorkspaceMigrationException( - `TsVectorColumnActionFactory.handleAlterAction has not been implemented yet.`, - WorkspaceMigrationExceptionCode.INVALID_FIELD_METADATA, - ); + return [ + { + action: WorkspaceMigrationColumnActionType.ALTER, + currentColumnDefinition: { + columnName: currentFieldMetadata.name, + columnType: fieldMetadataTypeToColumnType(currentFieldMetadata.type), + isNullable: currentFieldMetadata.isNullable ?? true, + defaultValue: undefined, + }, + alteredColumnDefinition: { + columnName: alteredFieldMetadata.name, + columnType: fieldMetadataTypeToColumnType(alteredFieldMetadata.type), + isNullable: alteredFieldMetadata.isNullable ?? true, + defaultValue: undefined, + asExpression: alteredFieldMetadata.asExpression, + generatedType: alteredFieldMetadata.generatedType, + }, + }, + ]; } } diff --git a/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts b/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts index c9dc92642..993441f4d 100644 --- a/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts +++ b/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts @@ -18,7 +18,10 @@ import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator'; import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { CUSTOM_OBJECT_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; @@ -26,6 +29,12 @@ import { NoteTargetWorkspaceEntity } from 'src/modules/note/standard-objects/not import { TaskTargetWorkspaceEntity } from 'src/modules/task/standard-objects/task-target.workspace-entity'; import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity'; +export const SEARCH_FIELDS_FOR_CUSTOM_OBJECT: FieldTypeAndNameMetadata[] = [ + { + name: DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, + type: FieldMetadataType.TEXT, + }, +]; @WorkspaceCustomEntity() export class CustomWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceField({ @@ -148,12 +157,9 @@ export class CustomWorkspaceEntity extends BaseWorkspaceEntity { label: SEARCH_VECTOR_FIELD.label, description: SEARCH_VECTOR_FIELD.description, generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { - name: DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, - type: FieldMetadataType.TEXT, - }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_CUSTOM_OBJECT, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem() diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index 878a25ed5..444477e71 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -469,6 +469,8 @@ export class WorkspaceMigrationRunnerService { ), isArray: migrationColumn.alteredColumnDefinition.isArray, isNullable: migrationColumn.alteredColumnDefinition.isNullable, + asExpression: migrationColumn.alteredColumnDefinition.asExpression, + generatedType: migrationColumn.alteredColumnDefinition.generatedType, isUnique: migrationColumn.alteredColumnDefinition.isUnique, }), ); diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts index 88ec505dd..4b179f5cd 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts @@ -1,6 +1 @@ -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; - -export const DEFAULT_FEATURE_FLAGS = [ - FeatureFlagKey.IsSearchEnabled, - FeatureFlagKey.IsWorkspaceMigratedForSearch, -]; +export const DEFAULT_FEATURE_FLAGS = []; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts index 4e10f7ea2..00115e2fb 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts @@ -10,7 +10,6 @@ import { } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { CustomWorkspaceEntity } from 'src/engine/twenty-orm/custom.workspace-entity'; @@ -145,26 +144,13 @@ export class WorkspaceSyncFieldMetadataService { const originalObjectMetadata = originalObjectMetadataMap[standardObjectId]; - let computedStandardFieldMetadataCollection = computeStandardFields( + const computedStandardFieldMetadataCollection = computeStandardFields( standardFieldMetadataCollection, originalObjectMetadata, // We need to provide this for generated relations with custom objects customObjectMetadataCollection, ); - let originalObjectMetadataFields = originalObjectMetadata.fields; - - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - computedStandardFieldMetadataCollection = - computedStandardFieldMetadataCollection.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - - originalObjectMetadataFields = originalObjectMetadataFields.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - } - const fieldComparatorResults = this.workspaceFieldComparator.compare( originalObjectMetadata.id, originalObjectMetadata.fields, @@ -192,24 +178,11 @@ export class WorkspaceSyncFieldMetadataService { // Loop over all custom objects from the DB and compare their fields with standard fields for (const customObjectMetadata of customObjectMetadataCollection) { // Also, maybe it's better to refactor a bit and move generation part into a separate module ? - let standardFieldMetadataCollection = computeStandardFields( + const standardFieldMetadataCollection = computeStandardFields( customObjectStandardFieldMetadataCollection, customObjectMetadata, ); - let customObjectMetadataFields = customObjectMetadata.fields; - - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - standardFieldMetadataCollection = - standardFieldMetadataCollection.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - - customObjectMetadataFields = customObjectMetadataFields.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - } - /** * COMPARE FIELD METADATA */ diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts index d9ee8908c..0173d171e 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts @@ -7,10 +7,7 @@ import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/wo import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { - IndexMetadataEntity, - IndexType, -} from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory'; @@ -73,7 +70,7 @@ export class WorkspaceSyncIndexMetadataService { const indexMetadataRepository = manager.getRepository(IndexMetadataEntity); - let originalIndexMetadataCollection = await indexMetadataRepository.find({ + const originalIndexMetadataCollection = await indexMetadataRepository.find({ where: { workspaceId: context.workspaceId, objectMetadataId: Any( @@ -87,7 +84,7 @@ export class WorkspaceSyncIndexMetadataService { }); // Generate index metadata from models - let standardIndexMetadataCollection = this.standardIndexFactory.create( + const standardIndexMetadataCollection = this.standardIndexFactory.create( standardObjectMetadataDefinitions, context, originalStandardObjectMetadataMap, @@ -95,15 +92,6 @@ export class WorkspaceSyncIndexMetadataService { workspaceFeatureFlagsMap, ); - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - originalIndexMetadataCollection = originalIndexMetadataCollection.filter( - (index) => index.indexType !== IndexType.GIN, - ); - - standardIndexMetadataCollection = standardIndexMetadataCollection.filter( - (index) => index.indexType !== IndexType.GIN, - ); - } const indexComparatorResults = this.workspaceIndexComparator.compare( originalIndexMetadataCollection, standardIndexMetadataCollection, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts index 8703879e5..a7a4f73bd 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts @@ -1,5 +1,8 @@ import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; const nameTextField = { name: 'name', type: FieldMetadataType.TEXT }; const nameFullNameField = { @@ -63,14 +66,18 @@ jest.mock( describe('getTsVectorColumnExpressionFromFields', () => { it('should generate correct expression for simple text field', () => { - const fields = [nameTextField]; + const fields = [nameTextField] as FieldTypeAndNameMetadata[]; const result = getTsVectorColumnExpressionFromFields(fields); expect(result).toContain("to_tsvector('simple', COALESCE(\"name\", ''))"); }); it('should handle multiple fields', () => { - const fields = [nameFullNameField, jobTitleTextField, emailsEmailsField]; + const fields = [ + nameFullNameField, + jobTitleTextField, + emailsEmailsField, + ] as FieldTypeAndNameMetadata[]; const result = getTsVectorColumnExpressionFromFields(fields); const expected = ` to_tsvector('simple', COALESCE("nameFirstName", '') || ' ' || COALESCE("nameLastName", '') || ' ' || COALESCE("jobTitle", '') || ' ' || diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts index 83cabf58b..f4f197ff5 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts @@ -9,15 +9,27 @@ import { WorkspaceMigrationException, WorkspaceMigrationExceptionCode, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception'; +import { + isSearchableFieldType, + SearchableFieldType, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; -type FieldTypeAndNameMetadata = { +export type FieldTypeAndNameMetadata = { name: string; - type: FieldMetadataType; + type: SearchableFieldType; }; export const getTsVectorColumnExpressionFromFields = ( fieldsUsedForSearch: FieldTypeAndNameMetadata[], ): string => { + const filteredFieldsUsedForSearch = fieldsUsedForSearch.filter((field) => + isSearchableFieldType(field.type), + ); + + if (filteredFieldsUsedForSearch.length < 1) { + throw new Error('No searchable fields found'); + } + const columnExpressions = fieldsUsedForSearch.flatMap( getColumnExpressionsFromField, ); diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts new file mode 100644 index 000000000..bb482ae4a --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts @@ -0,0 +1,17 @@ +import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; + +const SEARCHABLE_FIELD_TYPES = [ + FieldMetadataType.TEXT, + FieldMetadataType.FULL_NAME, + FieldMetadataType.EMAILS, + FieldMetadataType.ADDRESS, + FieldMetadataType.LINKS, +] as const; + +export type SearchableFieldType = (typeof SEARCHABLE_FIELD_TYPES)[number]; + +export const isSearchableFieldType = ( + type: FieldMetadataType, +): type is SearchableFieldType => { + return SEARCHABLE_FIELD_TYPES.includes(type as SearchableFieldType); +}; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index c090d413a..16f01b999 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -5,7 +5,6 @@ import { DataSource, QueryFailedError, Repository } from 'typeorm'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; @@ -153,13 +152,6 @@ export class WorkspaceSyncMetadataService { await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( context.workspaceId, ); - - if (workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - await this.featureFlagService.enableFeatureFlags( - [FeatureFlagKey.IsWorkspaceMigratedForSearch], - context.workspaceId, - ); - } } catch (error) { this.logger.error('Sync of standard objects failed with:', error); diff --git a/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts b/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts index 1abbf5dd3..97572f3d1 100644 --- a/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts +++ b/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts @@ -25,7 +25,10 @@ import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { COMPANY_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; @@ -39,6 +42,11 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta const NAME_FIELD_NAME = 'name'; const DOMAIN_NAME_FIELD_NAME = 'domainName'; +export const SEARCH_FIELDS_FOR_COMPANY: FieldTypeAndNameMetadata[] = [ + { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, + { name: DOMAIN_NAME_FIELD_NAME, type: FieldMetadataType.LINKS }, +]; + @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.company, namePlural: 'companies', @@ -292,10 +300,9 @@ export class CompanyWorkspaceEntity extends BaseWorkspaceEntity { description: SEARCH_VECTOR_FIELD.description, icon: 'IconUser', generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, - { name: DOMAIN_NAME_FIELD_NAME, type: FieldMetadataType.LINKS }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_COMPANY, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem() diff --git a/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts b/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts index 52f1a449f..22b3fae96 100644 --- a/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts +++ b/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts @@ -24,7 +24,10 @@ import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { OPPORTUNITY_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { CompanyWorkspaceEntity } from 'src/modules/company/standard-objects/company.workspace-entity'; @@ -36,6 +39,10 @@ import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-o const NAME_FIELD_NAME = 'name'; +export const SEARCH_FIELDS_FOR_OPPORTUNITY: FieldTypeAndNameMetadata[] = [ + { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, +]; + @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.opportunity, namePlural: 'opportunities', @@ -245,9 +252,9 @@ export class OpportunityWorkspaceEntity extends BaseWorkspaceEntity { description: SEARCH_VECTOR_FIELD.description, icon: 'IconUser', generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_OPPORTUNITY, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem() diff --git a/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts b/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts index 755308cee..589afaabc 100644 --- a/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts +++ b/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts @@ -26,7 +26,10 @@ import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { PERSON_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; @@ -42,6 +45,12 @@ const NAME_FIELD_NAME = 'name'; const EMAILS_FIELD_NAME = 'emails'; const JOB_TITLE_FIELD_NAME = 'jobTitle'; +export const SEARCH_FIELDS_FOR_PERSON: FieldTypeAndNameMetadata[] = [ + { name: NAME_FIELD_NAME, type: FieldMetadataType.FULL_NAME }, + { name: EMAILS_FIELD_NAME, type: FieldMetadataType.EMAILS }, + { name: JOB_TITLE_FIELD_NAME, type: FieldMetadataType.TEXT }, +]; + @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.person, namePlural: 'people', @@ -300,11 +309,9 @@ export class PersonWorkspaceEntity extends BaseWorkspaceEntity { description: SEARCH_VECTOR_FIELD.description, icon: 'IconUser', generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { name: NAME_FIELD_NAME, type: FieldMetadataType.FULL_NAME }, - { name: EMAILS_FIELD_NAME, type: FieldMetadataType.EMAILS }, - { name: JOB_TITLE_FIELD_NAME, type: FieldMetadataType.TEXT }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_PERSON, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem()