diff --git a/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts b/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts index db3506c2d..0ab2a8229 100644 --- a/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts +++ b/packages/twenty-front/src/modules/object-metadata/utils/mapPaginatedObjectMetadataItemsToObjectMetadataItems.ts @@ -11,7 +11,7 @@ export const mapPaginatedObjectMetadataItemsToObjectMetadataItems = ({ pagedObjectMetadataItems?.objects.edges.map((object) => ({ ...object.node, fields: object.node.fields.edges.map((field) => field.node), - indexMetadatas: object.node.indexMetadatas.edges.map((index) => ({ + indexMetadatas: object.node.indexMetadatas?.edges.map((index) => ({ ...index.node, indexFieldMetadatas: index.node.indexFieldMetadatas?.edges.map( (indexField) => indexField.node, diff --git a/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts b/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts index cde449770..5471c5d4d 100644 --- a/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts +++ b/packages/twenty-front/src/modules/workspace/types/FeatureFlagKey.ts @@ -10,9 +10,7 @@ export type FeatureFlagKey = | 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED' | 'IS_WORKFLOW_ENABLED' | 'IS_WORKSPACE_FAVORITE_ENABLED' - | 'IS_SEARCH_ENABLED' | 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED' | 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED' - | 'IS_WORKSPACE_MIGRATED_FOR_SEARCH' | 'IS_ANALYTICS_V2_ENABLED' | 'IS_UNIQUE_INDEXES_ENABLED'; diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index f8207c318..808785d9f 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -7,6 +7,7 @@ import { DataSeedDemoWorkspaceCommand } from 'src/database/commands/data-seed-de import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module'; import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command'; import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; +import { SimplifySearchVectorExpressionCommandModule } from 'src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module'; import { UpgradeTo0_32CommandModule } from 'src/database/commands/upgrade-version/0-32/0-32-upgrade-version.module'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity'; @@ -46,6 +47,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp DataSeedDemoWorkspaceModule, WorkspaceCacheStorageModule, WorkspaceMetadataVersionModule, + SimplifySearchVectorExpressionCommandModule, UpgradeTo0_32CommandModule, FeatureFlagModule, ], diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts new file mode 100644 index 000000000..9e6ea5e2b --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { SimplifySearchVectorExpressionCommand } from 'src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { SearchModule } from 'src/engine/metadata-modules/search/search.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; +import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([Workspace], 'core'), + TypeOrmModule.forFeature([FieldMetadataEntity], 'metadata'), + WorkspaceSyncMetadataCommandsModule, + SearchModule, + WorkspaceMigrationRunnerModule, + ], + providers: [SimplifySearchVectorExpressionCommand], +}) +export class SimplifySearchVectorExpressionCommandModule {} diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts new file mode 100644 index 000000000..27d75e213 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-31/0-32/0-32-simplify-search-vector-expression.ts @@ -0,0 +1,115 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import chalk from 'chalk'; +import { Command } from 'nest-commander'; +import { Repository } from 'typeorm'; + +import { + ActiveWorkspacesCommandOptions, + ActiveWorkspacesCommandRunner, +} from 'src/database/commands/active-workspaces.command'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; +import { SEARCH_FIELDS_FOR_CUSTOM_OBJECT } from 'src/engine/twenty-orm/custom.workspace-entity'; +import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; +import { + COMPANY_STANDARD_FIELD_IDS, + CUSTOM_OBJECT_STANDARD_FIELD_IDS, + OPPORTUNITY_STANDARD_FIELD_IDS, + PERSON_STANDARD_FIELD_IDS, +} from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; +import { FieldTypeAndNameMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { SEARCH_FIELDS_FOR_COMPANY } from 'src/modules/company/standard-objects/company.workspace-entity'; +import { SEARCH_FIELDS_FOR_OPPORTUNITY } from 'src/modules/opportunity/standard-objects/opportunity.workspace-entity'; +import { SEARCH_FIELDS_FOR_PERSON } from 'src/modules/person/standard-objects/person.workspace-entity'; + +@Command({ + name: 'fix-0.32:simplify-search-vector-expression', + description: 'Replace searchVector with simpler expression', +}) +export class SimplifySearchVectorExpressionCommand extends ActiveWorkspacesCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, + private readonly searchService: SearchService, + private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, + ) { + super(workspaceRepository); + } + + async executeActiveWorkspacesCommand( + _passedParam: string[], + _options: ActiveWorkspacesCommandOptions, + workspaceIds: string[], + ): Promise { + this.logger.log('Running command to fix migration'); + + for (const workspaceId of workspaceIds) { + this.logger.log(`Running command for workspace ${workspaceId}`); + + try { + const searchVectorFields = await this.fieldMetadataRepository.findBy({ + workspaceId: workspaceId, + type: FieldMetadataType.TS_VECTOR, + }); + + for (const searchVectorField of searchVectorFields) { + let fieldsUsedForSearch: FieldTypeAndNameMetadata[] = []; + + switch (searchVectorField.standardId) { + case CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_CUSTOM_OBJECT; + break; + } + case PERSON_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_PERSON; + break; + } + case COMPANY_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_COMPANY; + break; + } + case OPPORTUNITY_STANDARD_FIELD_IDS.searchVector: { + fieldsUsedForSearch = SEARCH_FIELDS_FOR_OPPORTUNITY; + break; + } + default: { + throw new Error( + `search vector has unexpected standardId: ${searchVectorField.standardId}`, + ); + } + } + + await this.searchService.updateSearchVector( + searchVectorField.objectMetadataId, + fieldsUsedForSearch, + workspaceId, + ); + + await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( + workspaceId, + ); + } + } catch (error) { + this.logger.log( + chalk.red( + `Running command on workspace ${workspaceId} failed with error: ${error}`, + ), + ); + continue; + } finally { + this.logger.log( + chalk.green(`Finished running command for workspace ${workspaceId}.`), + ); + } + + this.logger.log(chalk.green(`Command completed!`)); + } + } +} diff --git a/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts b/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts index c618677d3..97b34a884 100644 --- a/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts +++ b/packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts @@ -55,16 +55,6 @@ export const seedFeatureFlags = async ( workspaceId: workspaceId, value: true, }, - { - key: FeatureFlagKey.IsSearchEnabled, - workspaceId: workspaceId, - value: true, - }, - { - key: FeatureFlagKey.IsWorkspaceMigratedForSearch, - workspaceId: workspaceId, - value: true, - }, { key: FeatureFlagKey.IsAnalyticsV2Enabled, workspaceId: workspaceId, diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts new file mode 100644 index 000000000..9659750f8 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1728999374151-addConstraintOnIndexMetadata.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddConstraintOnIndex1728999374151 implements MigrationInterface { + name = 'AddConstraintOnIndexMetadata1728999374151'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" ADD CONSTRAINT "IndexOnNameAndWorkspaceIdAndObjectMetadataUnique" UNIQUE ("name", "workspaceId", "objectMetadataId")`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "metadata"."indexMetadata" DROP CONSTRAINT "IndexOnNameAndWorkspaceIdAndObjectMetadataUnique"`, + ); + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts index a0fdbf0d3..cfd570281 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-search-resolver.service.ts @@ -10,10 +10,6 @@ import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-qu import { SearchResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant'; -import { - GraphqlQueryRunnerException, - GraphqlQueryRunnerExceptionCode, -} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception'; import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; @@ -100,20 +96,6 @@ export class GraphqlQuerySearchResolverService async validate( _args: SearchResolverArgs, - options: WorkspaceQueryRunnerOptions, - ): Promise { - const featureFlagsForWorkspace = - await this.featureFlagService.getWorkspaceFeatureFlags( - options.authContext.workspace.id, - ); - - const isSearchEnabled = featureFlagsForWorkspace.IS_SEARCH_ENABLED; - - if (!isSearchEnabled) { - throw new GraphqlQueryRunnerException( - 'This endpoint is not available yet, please use findMany instead.', - GraphqlQueryRunnerExceptionCode.INVALID_QUERY_INPUT, - ); - } - } + _options: WorkspaceQueryRunnerOptions, + ): Promise {} } diff --git a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts index 8483f3089..58282f9de 100644 --- a/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/feature-flag/enums/feature-flag-key.enum.ts @@ -10,8 +10,6 @@ export enum FeatureFlagKey { IsMessageThreadSubscriberEnabled = 'IS_MESSAGE_THREAD_SUBSCRIBER_ENABLED', IsQueryRunnerTwentyORMEnabled = 'IS_QUERY_RUNNER_TWENTY_ORM_ENABLED', IsWorkspaceFavoriteEnabled = 'IS_WORKSPACE_FAVORITE_ENABLED', - IsSearchEnabled = 'IS_SEARCH_ENABLED', - IsWorkspaceMigratedForSearch = 'IS_WORKSPACE_MIGRATED_FOR_SEARCH', IsGmailSendEmailScopeEnabled = 'IS_GMAIL_SEND_EMAIL_SCOPE_ENABLED', IsAnalyticsV2Enabled = 'IS_ANALYTICS_V2_ENABLED', IsUniqueIndexesEnabled = 'IS_UNIQUE_INDEXES_ENABLED', diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index f402f5b0b..4291fe44a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -29,7 +29,6 @@ import { import { generateNullable } from 'src/engine/metadata-modules/field-metadata/utils/generate-nullable'; import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util'; import { RelationMetadataEntity, @@ -76,7 +75,6 @@ export class FieldMetadataService extends TypeOrmQueryService, @InjectRepository(ObjectMetadataEntity, 'metadata') private readonly objectMetadataRepository: Repository, - private readonly objectMetadataService: ObjectMetadataService, private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, private readonly workspaceMigrationService: WorkspaceMigrationService, private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts index fc2f991ce..b748e2ea5 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts @@ -7,6 +7,7 @@ import { OneToMany, PrimaryGeneratedColumn, Relation, + Unique, UpdateDateColumn, } from 'typeorm'; @@ -18,6 +19,11 @@ export enum IndexType { GIN = 'GIN', } +@Unique('IndexOnNameAndWorkspaceIdAndObjectMetadataUnique', [ + 'name', + 'workspaceId', + 'objectMetadataId', +]) @Entity('indexMetadata') export class IndexMetadataEntity { @PrimaryGeneratedColumn('uuid') diff --git a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts index 362519ef7..1d75cbd7a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { isDefined } from 'class-validator'; -import { Repository } from 'typeorm'; +import { InsertResult, Repository } from 'typeorm'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { @@ -45,32 +45,37 @@ export class IndexMetadataService { const indexName = `IDX_${generateDeterministicIndexName([tableName, ...columnNames])}`; - let savedIndexMetadata: IndexMetadataEntity; + let result: InsertResult; try { - savedIndexMetadata = await this.indexMetadataRepository.save({ - name: indexName, - tableName, - indexFieldMetadatas: fieldMetadataToIndex.map( - (fieldMetadata, index) => { - return { - fieldMetadataId: fieldMetadata.id, - order: index, - }; - }, - ), - workspaceId, - objectMetadataId: objectMetadata.id, - ...(isDefined(indexType) ? { indexType: indexType } : {}), - isCustom: isCustom, - }); + result = await this.indexMetadataRepository.upsert( + { + name: indexName, + indexFieldMetadatas: fieldMetadataToIndex.map( + (fieldMetadata, index) => { + return { + fieldMetadataId: fieldMetadata.id, + order: index, + }; + }, + ), + workspaceId, + objectMetadataId: objectMetadata.id, + ...(isDefined(indexType) ? { indexType: indexType } : {}), + isCustom: isCustom, + }, + { + conflictPaths: ['workspaceId', 'name', 'objectMetadataId'], + skipUpdateIfNoValuesChanged: true, + }, + ); } catch (error) { throw new Error( `Failed to create index ${indexName} on object metadata ${objectMetadata.nameSingular}`, ); } - if (!savedIndexMetadata) { + if (!result.identifiers.length) { throw new Error( `Failed to return saved index ${indexName} on object metadata ${objectMetadata.nameSingular}`, ); diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts index 14d9d58c2..46992a6e1 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts @@ -14,12 +14,12 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature- import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module'; import { BeforeUpdateOneObject } from 'src/engine/metadata-modules/object-metadata/hooks/before-update-one-object.hook'; import { ObjectMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/object-metadata/interceptors/object-metadata-graphql-api-exception.interceptor'; import { ObjectMetadataResolver } from 'src/engine/metadata-modules/object-metadata/object-metadata.resolver'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; import { RemoteTableRelationsModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.module'; +import { SearchModule } from 'src/engine/metadata-modules/search/search.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @@ -46,8 +46,8 @@ import { UpdateObjectPayload } from './dtos/update-object.input'; WorkspaceMigrationRunnerModule, WorkspaceMetadataVersionModule, RemoteTableRelationsModule, - IndexMetadataModule, FeatureFlagModule, + SearchModule, ], services: [ObjectMetadataService], resolvers: [ diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts index 83db047cd..8ec403f84 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts @@ -5,30 +5,20 @@ import console from 'console'; import { Query, QueryOptions } from '@ptc-org/nestjs-query-core'; import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm'; -import { isDefined } from 'class-validator'; import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm'; import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface'; -import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; -import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { FieldMetadataEntity, FieldMetadataType, } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { - computeColumnName, - FieldTypeAndNameMetadata, -} from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; -import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; -import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service'; +import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; import { DeleteOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/delete-object.input'; import { UpdateOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/update-object.input'; -import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants'; import { ObjectMetadataException, ObjectMetadataExceptionCode, @@ -43,8 +33,8 @@ import { import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete'; import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service'; import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; -import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, @@ -72,7 +62,7 @@ import { createForeignKeyDeterministicUuid, createRelationDeterministicUuid, } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { isSearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity'; @@ -94,17 +84,14 @@ export class ObjectMetadataService extends TypeOrmQueryService - field.id === createdObjectMetadata.labelIdentifierFieldMetadataId, - ) - : createdObjectMetadata.fields.find( - (field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, - ); - - if (!isDefined(searchableFieldForCustomObject)) { - throw new Error('No searchable field found for custom object'); - } - - this.workspaceMigrationService.createCustomMigration( - generateMigrationName( - `update-${createdObjectMetadata.nameSingular}-add-searchVector`, - ), - createdObjectMetadata.workspaceId, - [ - { - name: computeTableName( - createdObjectMetadata.nameSingular, - createdObjectMetadata.isCustom, - ), - action: WorkspaceMigrationTableActionType.ALTER, - columns: this.tsVectorColumnActionFactory.handleCreateAction({ - ...searchVectorFieldMetadata, - defaultValue: undefined, - generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - searchableFieldForCustomObject as FieldTypeAndNameMetadata, - ]), - options: undefined, - } as FieldMetadataInterface), - }, - ], - ); - - await this.indexMetadataService.createIndex( - objectMetadataInput.workspaceId, - createdObjectMetadata, - [searchVectorFieldMetadata], - false, - false, - IndexType.GIN, - ); - } - private async createActivityTargetRelation( workspaceId: string, createdObjectMetadata: ObjectMetadataEntity, diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts new file mode 100644 index 000000000..2260d7218 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; + +import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; + +import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { SearchService } from 'src/engine/metadata-modules/search/search.service'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; + +@Module({ + imports: [ + NestjsQueryTypeOrmModule.forFeature( + [ObjectMetadataEntity, FieldMetadataEntity], + 'metadata', + ), + IndexMetadataModule, + WorkspaceMigrationModule, + ], + providers: [SearchService], + exports: [SearchService], +}) +export class SearchModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts new file mode 100644 index 000000000..b24ad3816 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/search/search.service.ts @@ -0,0 +1,169 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; + +import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants'; +import { + FieldMetadataEntity, + FieldMetadataType, +} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service'; +import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input'; +import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants'; +import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; +import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory'; +import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; +import { + WorkspaceMigrationColumnActionType, + WorkspaceMigrationTableActionType, +} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; +import { WorkspaceMigrationFactory } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.factory'; +import { WorkspaceMigrationService } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.service'; +import { computeTableName } from 'src/engine/utils/compute-table-name.util'; +import { CUSTOM_OBJECT_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { SearchableFieldType } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; +import { isDefined } from 'src/utils/is-defined'; + +@Injectable() +export class SearchService { + constructor( + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, + private readonly tsVectorColumnActionFactory: TsVectorColumnActionFactory, + private readonly indexMetadataService: IndexMetadataService, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, + private readonly workspaceMigrationService: WorkspaceMigrationService, + private readonly workspaceMigrationFactory: WorkspaceMigrationFactory, + ) {} + + public async createSearchVectorFieldForObject( + objectMetadataInput: CreateObjectInput, + createdObjectMetadata: ObjectMetadataEntity, + ) { + const searchVectorFieldMetadata = await this.fieldMetadataRepository.save({ + standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector, + objectMetadataId: createdObjectMetadata.id, + workspaceId: objectMetadataInput.workspaceId, + isCustom: false, + isActive: false, + isSystem: true, + type: FieldMetadataType.TS_VECTOR, + name: SEARCH_VECTOR_FIELD.name, + label: SEARCH_VECTOR_FIELD.label, + description: SEARCH_VECTOR_FIELD.description, + isNullable: true, + }); + + const searchableFieldForCustomObject = + createdObjectMetadata.labelIdentifierFieldMetadataId + ? createdObjectMetadata.fields.find( + (field) => + field.id === createdObjectMetadata.labelIdentifierFieldMetadataId, + ) + : createdObjectMetadata.fields.find( + (field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, + ); + + if (!isDefined(searchableFieldForCustomObject)) { + throw new Error( + `No searchable field found for custom object (object name: ${createdObjectMetadata.nameSingular})`, + ); + } + + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`create-${createdObjectMetadata.nameSingular}`), + createdObjectMetadata.workspaceId, + [ + { + name: computeTableName( + createdObjectMetadata.nameSingular, + createdObjectMetadata.isCustom, + ), + action: WorkspaceMigrationTableActionType.ALTER, + columns: this.tsVectorColumnActionFactory.handleCreateAction({ + ...searchVectorFieldMetadata, + defaultValue: undefined, + generatedType: 'STORED', + asExpression: getTsVectorColumnExpressionFromFields([ + { + type: searchableFieldForCustomObject.type as SearchableFieldType, + name: searchableFieldForCustomObject.name, + }, + ]), + options: undefined, + } as FieldMetadataInterface), + }, + ], + ); + + await this.indexMetadataService.createIndex( + objectMetadataInput.workspaceId, + createdObjectMetadata, + [searchVectorFieldMetadata], + false, + false, + IndexType.GIN, + ); + } + + public async updateSearchVector( + objectMetadataId: string, + fieldMetadataNameAndTypeForSearch: FieldTypeAndNameMetadata[], + workspaceId: string, + ) { + const objectMetadata = await this.objectMetadataRepository.findOneByOrFail({ + id: objectMetadataId, + }); + + const existingSearchVectorFieldMetadata = + await this.fieldMetadataRepository.findOneByOrFail({ + name: SEARCH_VECTOR_FIELD.name, + objectMetadataId, + }); + + await this.workspaceMigrationService.createCustomMigration( + generateMigrationName(`update-${objectMetadata.nameSingular}`), + workspaceId, + [ + { + name: computeTableName( + objectMetadata.nameSingular, + objectMetadata.isCustom, + ), + action: WorkspaceMigrationTableActionType.ALTER, + columns: this.workspaceMigrationFactory.createColumnActions( + WorkspaceMigrationColumnActionType.ALTER, + existingSearchVectorFieldMetadata, + { + ...existingSearchVectorFieldMetadata, + asExpression: getTsVectorColumnExpressionFromFields( + fieldMetadataNameAndTypeForSearch, + ), + generatedType: 'STORED', // Not stored on fieldMetadata + options: undefined, + }, + ), + }, + ], + ); + + // index needs to be recreated as typeorm deletes then recreates searchVector column at alter + await this.indexMetadataService.createIndex( + workspaceId, + objectMetadata, + [existingSearchVectorFieldMetadata], + false, + false, + IndexType.GIN, + ); + } +} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts index fa6aea201..71f1265b7 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts @@ -1,7 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface'; -import { WorkspaceColumnActionOptions } from 'src/engine/metadata-modules/workspace-migration/interfaces/workspace-column-action-options.interface'; import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util'; @@ -12,10 +11,6 @@ import { WorkspaceMigrationColumnAlter, WorkspaceMigrationColumnCreate, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; -import { - WorkspaceMigrationException, - WorkspaceMigrationExceptionCode, -} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception'; export type TsVectorFieldMetadataType = FieldMetadataType.TS_VECTOR; @@ -40,14 +35,28 @@ export class TsVectorColumnActionFactory extends ColumnActionAbstractFactory, - _alteredFieldMetadata: FieldMetadataInterface, - _options?: WorkspaceColumnActionOptions, + handleAlterAction( + currentFieldMetadata: FieldMetadataInterface, + alteredFieldMetadata: FieldMetadataInterface, ): WorkspaceMigrationColumnAlter[] { - throw new WorkspaceMigrationException( - `TsVectorColumnActionFactory.handleAlterAction has not been implemented yet.`, - WorkspaceMigrationExceptionCode.INVALID_FIELD_METADATA, - ); + return [ + { + action: WorkspaceMigrationColumnActionType.ALTER, + currentColumnDefinition: { + columnName: currentFieldMetadata.name, + columnType: fieldMetadataTypeToColumnType(currentFieldMetadata.type), + isNullable: currentFieldMetadata.isNullable ?? true, + defaultValue: undefined, + }, + alteredColumnDefinition: { + columnName: alteredFieldMetadata.name, + columnType: fieldMetadataTypeToColumnType(alteredFieldMetadata.type), + isNullable: alteredFieldMetadata.isNullable ?? true, + defaultValue: undefined, + asExpression: alteredFieldMetadata.asExpression, + generatedType: alteredFieldMetadata.generatedType, + }, + }, + ]; } } diff --git a/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts b/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts index c9dc92642..993441f4d 100644 --- a/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts +++ b/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts @@ -18,7 +18,10 @@ import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator'; import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { CUSTOM_OBJECT_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; @@ -26,6 +29,12 @@ import { NoteTargetWorkspaceEntity } from 'src/modules/note/standard-objects/not import { TaskTargetWorkspaceEntity } from 'src/modules/task/standard-objects/task-target.workspace-entity'; import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-objects/timeline-activity.workspace-entity'; +export const SEARCH_FIELDS_FOR_CUSTOM_OBJECT: FieldTypeAndNameMetadata[] = [ + { + name: DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, + type: FieldMetadataType.TEXT, + }, +]; @WorkspaceCustomEntity() export class CustomWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceField({ @@ -148,12 +157,9 @@ export class CustomWorkspaceEntity extends BaseWorkspaceEntity { label: SEARCH_VECTOR_FIELD.label, description: SEARCH_VECTOR_FIELD.description, generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { - name: DEFAULT_LABEL_IDENTIFIER_FIELD_NAME, - type: FieldMetadataType.TEXT, - }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_CUSTOM_OBJECT, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem() diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts index 878a25ed5..444477e71 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service.ts @@ -469,6 +469,8 @@ export class WorkspaceMigrationRunnerService { ), isArray: migrationColumn.alteredColumnDefinition.isArray, isNullable: migrationColumn.alteredColumnDefinition.isNullable, + asExpression: migrationColumn.alteredColumnDefinition.asExpression, + generatedType: migrationColumn.alteredColumnDefinition.generatedType, isUnique: migrationColumn.alteredColumnDefinition.isUnique, }), ); diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts index 88ec505dd..4b179f5cd 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts @@ -1,6 +1 @@ -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; - -export const DEFAULT_FEATURE_FLAGS = [ - FeatureFlagKey.IsSearchEnabled, - FeatureFlagKey.IsWorkspaceMigratedForSearch, -]; +export const DEFAULT_FEATURE_FLAGS = []; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts index 4e10f7ea2..00115e2fb 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts @@ -10,7 +10,6 @@ import { } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { CustomWorkspaceEntity } from 'src/engine/twenty-orm/custom.workspace-entity'; @@ -145,26 +144,13 @@ export class WorkspaceSyncFieldMetadataService { const originalObjectMetadata = originalObjectMetadataMap[standardObjectId]; - let computedStandardFieldMetadataCollection = computeStandardFields( + const computedStandardFieldMetadataCollection = computeStandardFields( standardFieldMetadataCollection, originalObjectMetadata, // We need to provide this for generated relations with custom objects customObjectMetadataCollection, ); - let originalObjectMetadataFields = originalObjectMetadata.fields; - - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - computedStandardFieldMetadataCollection = - computedStandardFieldMetadataCollection.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - - originalObjectMetadataFields = originalObjectMetadataFields.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - } - const fieldComparatorResults = this.workspaceFieldComparator.compare( originalObjectMetadata.id, originalObjectMetadata.fields, @@ -192,24 +178,11 @@ export class WorkspaceSyncFieldMetadataService { // Loop over all custom objects from the DB and compare their fields with standard fields for (const customObjectMetadata of customObjectMetadataCollection) { // Also, maybe it's better to refactor a bit and move generation part into a separate module ? - let standardFieldMetadataCollection = computeStandardFields( + const standardFieldMetadataCollection = computeStandardFields( customObjectStandardFieldMetadataCollection, customObjectMetadata, ); - let customObjectMetadataFields = customObjectMetadata.fields; - - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - standardFieldMetadataCollection = - standardFieldMetadataCollection.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - - customObjectMetadataFields = customObjectMetadataFields.filter( - (field) => field.type !== FieldMetadataType.TS_VECTOR, - ); - } - /** * COMPARE FIELD METADATA */ diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts index d9ee8908c..0173d171e 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts @@ -7,10 +7,7 @@ import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/wo import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { - IndexMetadataEntity, - IndexType, -} from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; +import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory'; @@ -73,7 +70,7 @@ export class WorkspaceSyncIndexMetadataService { const indexMetadataRepository = manager.getRepository(IndexMetadataEntity); - let originalIndexMetadataCollection = await indexMetadataRepository.find({ + const originalIndexMetadataCollection = await indexMetadataRepository.find({ where: { workspaceId: context.workspaceId, objectMetadataId: Any( @@ -87,7 +84,7 @@ export class WorkspaceSyncIndexMetadataService { }); // Generate index metadata from models - let standardIndexMetadataCollection = this.standardIndexFactory.create( + const standardIndexMetadataCollection = this.standardIndexFactory.create( standardObjectMetadataDefinitions, context, originalStandardObjectMetadataMap, @@ -95,15 +92,6 @@ export class WorkspaceSyncIndexMetadataService { workspaceFeatureFlagsMap, ); - if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - originalIndexMetadataCollection = originalIndexMetadataCollection.filter( - (index) => index.indexType !== IndexType.GIN, - ); - - standardIndexMetadataCollection = standardIndexMetadataCollection.filter( - (index) => index.indexType !== IndexType.GIN, - ); - } const indexComparatorResults = this.workspaceIndexComparator.compare( originalIndexMetadataCollection, standardIndexMetadataCollection, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts index 8703879e5..a7a4f73bd 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts @@ -1,5 +1,8 @@ import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; const nameTextField = { name: 'name', type: FieldMetadataType.TEXT }; const nameFullNameField = { @@ -63,14 +66,18 @@ jest.mock( describe('getTsVectorColumnExpressionFromFields', () => { it('should generate correct expression for simple text field', () => { - const fields = [nameTextField]; + const fields = [nameTextField] as FieldTypeAndNameMetadata[]; const result = getTsVectorColumnExpressionFromFields(fields); expect(result).toContain("to_tsvector('simple', COALESCE(\"name\", ''))"); }); it('should handle multiple fields', () => { - const fields = [nameFullNameField, jobTitleTextField, emailsEmailsField]; + const fields = [ + nameFullNameField, + jobTitleTextField, + emailsEmailsField, + ] as FieldTypeAndNameMetadata[]; const result = getTsVectorColumnExpressionFromFields(fields); const expected = ` to_tsvector('simple', COALESCE("nameFirstName", '') || ' ' || COALESCE("nameLastName", '') || ' ' || COALESCE("jobTitle", '') || ' ' || diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts index 83cabf58b..f4f197ff5 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts @@ -9,15 +9,27 @@ import { WorkspaceMigrationException, WorkspaceMigrationExceptionCode, } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception'; +import { + isSearchableFieldType, + SearchableFieldType, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util'; -type FieldTypeAndNameMetadata = { +export type FieldTypeAndNameMetadata = { name: string; - type: FieldMetadataType; + type: SearchableFieldType; }; export const getTsVectorColumnExpressionFromFields = ( fieldsUsedForSearch: FieldTypeAndNameMetadata[], ): string => { + const filteredFieldsUsedForSearch = fieldsUsedForSearch.filter((field) => + isSearchableFieldType(field.type), + ); + + if (filteredFieldsUsedForSearch.length < 1) { + throw new Error('No searchable fields found'); + } + const columnExpressions = fieldsUsedForSearch.flatMap( getColumnExpressionsFromField, ); diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts new file mode 100644 index 000000000..bb482ae4a --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts @@ -0,0 +1,17 @@ +import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; + +const SEARCHABLE_FIELD_TYPES = [ + FieldMetadataType.TEXT, + FieldMetadataType.FULL_NAME, + FieldMetadataType.EMAILS, + FieldMetadataType.ADDRESS, + FieldMetadataType.LINKS, +] as const; + +export type SearchableFieldType = (typeof SEARCHABLE_FIELD_TYPES)[number]; + +export const isSearchableFieldType = ( + type: FieldMetadataType, +): type is SearchableFieldType => { + return SEARCHABLE_FIELD_TYPES.includes(type as SearchableFieldType); +}; diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index c090d413a..16f01b999 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -5,7 +5,6 @@ import { DataSource, QueryFailedError, Repository } from 'typeorm'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; @@ -153,13 +152,6 @@ export class WorkspaceSyncMetadataService { await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( context.workspaceId, ); - - if (workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) { - await this.featureFlagService.enableFeatureFlags( - [FeatureFlagKey.IsWorkspaceMigratedForSearch], - context.workspaceId, - ); - } } catch (error) { this.logger.error('Sync of standard objects failed with:', error); diff --git a/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts b/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts index 1abbf5dd3..97572f3d1 100644 --- a/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts +++ b/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts @@ -25,7 +25,10 @@ import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { COMPANY_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity'; @@ -39,6 +42,11 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta const NAME_FIELD_NAME = 'name'; const DOMAIN_NAME_FIELD_NAME = 'domainName'; +export const SEARCH_FIELDS_FOR_COMPANY: FieldTypeAndNameMetadata[] = [ + { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, + { name: DOMAIN_NAME_FIELD_NAME, type: FieldMetadataType.LINKS }, +]; + @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.company, namePlural: 'companies', @@ -292,10 +300,9 @@ export class CompanyWorkspaceEntity extends BaseWorkspaceEntity { description: SEARCH_VECTOR_FIELD.description, icon: 'IconUser', generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, - { name: DOMAIN_NAME_FIELD_NAME, type: FieldMetadataType.LINKS }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_COMPANY, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem() diff --git a/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts b/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts index 52f1a449f..22b3fae96 100644 --- a/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts +++ b/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts @@ -24,7 +24,10 @@ import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { OPPORTUNITY_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { CompanyWorkspaceEntity } from 'src/modules/company/standard-objects/company.workspace-entity'; @@ -36,6 +39,10 @@ import { TimelineActivityWorkspaceEntity } from 'src/modules/timeline/standard-o const NAME_FIELD_NAME = 'name'; +export const SEARCH_FIELDS_FOR_OPPORTUNITY: FieldTypeAndNameMetadata[] = [ + { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, +]; + @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.opportunity, namePlural: 'opportunities', @@ -245,9 +252,9 @@ export class OpportunityWorkspaceEntity extends BaseWorkspaceEntity { description: SEARCH_VECTOR_FIELD.description, icon: 'IconUser', generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { name: NAME_FIELD_NAME, type: FieldMetadataType.TEXT }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_OPPORTUNITY, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem() diff --git a/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts b/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts index 755308cee..589afaabc 100644 --- a/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts +++ b/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts @@ -26,7 +26,10 @@ import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace- import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator'; import { PERSON_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; -import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; +import { + FieldTypeAndNameMetadata, + getTsVectorColumnExpressionFromFields, +} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util'; import { ActivityTargetWorkspaceEntity } from 'src/modules/activity/standard-objects/activity-target.workspace-entity'; import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; import { CalendarEventParticipantWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-event-participant.workspace-entity'; @@ -42,6 +45,12 @@ const NAME_FIELD_NAME = 'name'; const EMAILS_FIELD_NAME = 'emails'; const JOB_TITLE_FIELD_NAME = 'jobTitle'; +export const SEARCH_FIELDS_FOR_PERSON: FieldTypeAndNameMetadata[] = [ + { name: NAME_FIELD_NAME, type: FieldMetadataType.FULL_NAME }, + { name: EMAILS_FIELD_NAME, type: FieldMetadataType.EMAILS }, + { name: JOB_TITLE_FIELD_NAME, type: FieldMetadataType.TEXT }, +]; + @WorkspaceEntity({ standardId: STANDARD_OBJECT_IDS.person, namePlural: 'people', @@ -300,11 +309,9 @@ export class PersonWorkspaceEntity extends BaseWorkspaceEntity { description: SEARCH_VECTOR_FIELD.description, icon: 'IconUser', generatedType: 'STORED', - asExpression: getTsVectorColumnExpressionFromFields([ - { name: NAME_FIELD_NAME, type: FieldMetadataType.FULL_NAME }, - { name: EMAILS_FIELD_NAME, type: FieldMetadataType.EMAILS }, - { name: JOB_TITLE_FIELD_NAME, type: FieldMetadataType.TEXT }, - ]), + asExpression: getTsVectorColumnExpressionFromFields( + SEARCH_FIELDS_FOR_PERSON, + ), }) @WorkspaceIsNullable() @WorkspaceIsSystem()