admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Search (#7237)

Browse Source 
Steps to test

1. Run metadata migrations
2. Run sync-metadata on your workspace
3. Enable the following feature flags: 
IS_SEARCH_ENABLED
IS_QUERY_RUNNER_TWENTY_ORM_ENABLED
IS_WORKSPACE_MIGRATED_FOR_SEARCH
4. Type Cmd + K and search anything
This commit is contained in:
Marie
2024-10-03 17:18:49 +02:00
committed by GitHub
parent 4c250dd811
commit 5f9435c718
71 changed files with 1517 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/twenty-server/src/engine/metadata-modules/constants/search-vector-field.constants.ts Normal file
View File

@ -0,0 +1,5 @@
export const SEARCH_VECTOR_FIELD = {
name: 'searchVector',
label: 'Search vector',
description: 'Field used for full-text search',
} as const;

1
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.entity.ts
View File

@ -47,6 +47,7 @@ export enum FieldMetadataType {
RICH_TEXT = 'RICH_TEXT',
ACTOR = 'ACTOR',
ARRAY = 'ARRAY',
TS_VECTOR = 'TS_VECTOR',
}
@Entity('fieldMetadata')

2
packages/twenty-server/src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface.ts
View File

@ -22,4 +22,6 @@ export interface FieldMetadataInterface<
fromRelationMetadata?: RelationMetadataEntity;
toRelationMetadata?: RelationMetadataEntity;
isCustom?: boolean;
generatedType?: 'STORED' | 'VIRTUAL';
asExpression?: string;
}

12
packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/compute-column-name.util.ts
View File

@ -11,6 +11,11 @@ import { pascalCase } from 'src/utils/pascal-case';
type ComputeColumnNameOptions = { isForeignKey?: boolean };
export type FieldTypeAndNameMetadata = {
name: string;
type: FieldMetadataType;
};
export function computeColumnName(
fieldName: string,
options?: ComputeColumnNameOptions,
@ -48,13 +53,16 @@ export function computeCompositeColumnName(
export function computeCompositeColumnName<
T extends FieldMetadataType | 'default',
>(
fieldMetadata: FieldMetadataInterface<T>,
fieldMetadata: FieldTypeAndNameMetadata | FieldMetadataInterface<T>,
compositeProperty: CompositeProperty,
): string;
export function computeCompositeColumnName<
T extends FieldMetadataType | 'default',
>(
fieldMetadataOrFieldName: FieldMetadataInterface<T> | string,
fieldMetadataOrFieldName:
| FieldTypeAndNameMetadata
| FieldMetadataInterface<T>
| string,
compositeProperty: CompositeProperty,
): string {
const generateName = (name: string) => {

16
packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.entity.ts
View File

@ -13,6 +13,11 @@ import {
import { IndexFieldMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
export enum IndexType {
BTREE = 'BTREE',
GIN = 'GIN',
}
@Entity('indexMetadata')
export class IndexMetadataEntity {
@PrimaryGeneratedColumn('uuid')
@ -48,4 +53,15 @@ export class IndexMetadataEntity {
@UpdateDateColumn({ type: 'timestamptz' })
updatedAt: Date;
@Column({ default: false })
isCustom: boolean;
@Column({
type: 'enum',
enum: IndexType,
nullable: true,
default: IndexType.BTREE,
})
indexType?: IndexType;
}

11
packages/twenty-server/src/engine/metadata-modules/index-metadata/index-metadata.service.ts
View File

@ -1,10 +1,14 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { isDefined } from 'class-validator';
import { Repository } from 'typeorm';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import {
IndexMetadataEntity,
IndexType,
} from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { generateDeterministicIndexName } from 'src/engine/metadata-modules/index-metadata/utils/generate-deterministic-index-name';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
@ -28,6 +32,8 @@ export class IndexMetadataService {
workspaceId: string,
objectMetadata: ObjectMetadataEntity,
fieldMetadataToIndex: Partial<FieldMetadataEntity>[],
isCustom: boolean,
indexType?: IndexType,
) {
const tableName = computeObjectTargetTable(objectMetadata);
@ -53,6 +59,8 @@ export class IndexMetadataService {
),
workspaceId,
objectMetadataId: objectMetadata.id,
...(isDefined(indexType) ? { indexType: indexType } : {}),
isCustom: isCustom,
});
} catch (error) {
throw new Error(
@ -74,6 +82,7 @@ export class IndexMetadataService {
action: WorkspaceMigrationIndexActionType.CREATE,
columns: columnNames,
name: indexName,
type: indexType,
},
],
} satisfies WorkspaceMigrationTableAction;

4
packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts
View File

@ -10,9 +10,11 @@ import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { IndexMetadataModule } from 'src/engine/metadata-modules/index-metadata/index-metadata.module';
import { BeforeUpdateOneObject } from 'src/engine/metadata-modules/object-metadata/hooks/before-update-one-object.hook';
import { ObjectMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/object-metadata/interceptors/object-metadata-graphql-api-exception.interceptor';
import { ObjectMetadataResolver } from 'src/engine/metadata-modules/object-metadata/object-metadata.resolver';
@ -44,6 +46,8 @@ import { UpdateObjectPayload } from './dtos/update-object.input';
WorkspaceMigrationRunnerModule,
WorkspaceMetadataVersionModule,
RemoteTableRelationsModule,
IndexMetadataModule,
FeatureFlagModule,
],
services: [ObjectMetadataService],
resolvers: [

96
packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts
View File

@ -5,19 +5,30 @@ import console from 'console';
import { Query, QueryOptions } from '@ptc-org/nestjs-query-core';
import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
import { isDefined } from 'class-validator';
import { FindManyOptions, FindOneOptions, In, Repository } from 'typeorm';
import { FieldMetadataSettings } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata-settings.interface';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { SEARCH_VECTOR_FIELD } from 'src/engine/metadata-modules/constants/search-vector-field.constants';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import {
FieldMetadataEntity,
FieldMetadataType,
} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import {
computeColumnName,
FieldTypeAndNameMetadata,
} from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { IndexMetadataService } from 'src/engine/metadata-modules/index-metadata/index-metadata.service';
import { DeleteOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/delete-object.input';
import { UpdateOneObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/update-object.input';
import { DEFAULT_LABEL_IDENTIFIER_FIELD_NAME } from 'src/engine/metadata-modules/object-metadata/object-metadata.constants';
import {
ObjectMetadataException,
ObjectMetadataExceptionCode,
@ -33,6 +44,7 @@ import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/
import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service';
import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,
@ -58,6 +70,7 @@ import {
createForeignKeyDeterministicUuid,
createRelationDeterministicUuid,
} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util';
import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util';
import { FavoriteWorkspaceEntity } from 'src/modules/favorite/standard-objects/favorite.workspace-entity';
import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity';
@ -79,9 +92,14 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
private readonly remoteTableRelationsService: RemoteTableRelationsService,
private readonly tsVectorColumnActionFactory: TsVectorColumnActionFactory,
private readonly dataSourceService: DataSourceService,
private readonly typeORMService: TypeORMService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly indexMetadataService: IndexMetadataService,
private readonly featureFlagService: FeatureFlagService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@ -350,6 +368,18 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
objectMetadataInput,
createdObjectMetadata,
);
const isSearchEnabled = await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IsSearchEnabled,
objectMetadataInput.workspaceId,
);
if (isSearchEnabled) {
await this.createSearchVectorField(
objectMetadataInput,
createdObjectMetadata,
);
}
} else {
await this.remoteTableRelationsService.createForeignKeysMetadataAndMigrations(
objectMetadataInput.workspaceId,
@ -548,6 +578,70 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
);
}
private async createSearchVectorField(
objectMetadataInput: CreateObjectInput,
createdObjectMetadata: ObjectMetadataEntity,
) {
const searchVectorFieldMetadata = await this.fieldMetadataRepository.save({
standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector,
objectMetadataId: createdObjectMetadata.id,
workspaceId: objectMetadataInput.workspaceId,
isCustom: false,
isActive: false,
isSystem: true,
type: FieldMetadataType.TS_VECTOR,
name: SEARCH_VECTOR_FIELD.name,
label: SEARCH_VECTOR_FIELD.label,
description: SEARCH_VECTOR_FIELD.description,
isNullable: true,
});
const searchableFieldForCustomObject =
createdObjectMetadata.labelIdentifierFieldMetadataId
? createdObjectMetadata.fields.find(
(field) =>
field.id === createdObjectMetadata.labelIdentifierFieldMetadataId,
)
: createdObjectMetadata.fields.find(
(field) => field.name === DEFAULT_LABEL_IDENTIFIER_FIELD_NAME,
);
if (!isDefined(searchableFieldForCustomObject)) {
throw new Error('No searchable field found for custom object');
}
this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdObjectMetadata.nameSingular}`),
createdObjectMetadata.workspaceId,
[
{
name: computeTableName(
createdObjectMetadata.nameSingular,
createdObjectMetadata.isCustom,
),
action: WorkspaceMigrationTableActionType.ALTER,
columns: this.tsVectorColumnActionFactory.handleCreateAction({
...searchVectorFieldMetadata,
defaultValue: undefined,
generatedType: 'STORED',
asExpression: getTsVectorColumnExpressionFromFields([
searchableFieldForCustomObject as FieldTypeAndNameMetadata,
]),
options: undefined,
} as FieldMetadataInterface<FieldMetadataType.TS_VECTOR>),
},
],
);
await this.indexMetadataService.createIndex(
objectMetadataInput.workspaceId,
createdObjectMetadata,
[searchVectorFieldMetadata],
false,
IndexType.GIN,
);
}
private async createActivityTargetRelation(
workspaceId: string,
createdObjectMetadata: ObjectMetadataEntity,

1
packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts
View File

@ -153,6 +153,7 @@ export class RelationMetadataService extends TypeOrmQueryService<RelationMetadat
relationMetadataInput.workspaceId,
toObjectMetadata,
[foreignKeyFieldMetadata, deletedFieldMetadata],
false,
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(

0
packages/twenty-server/src/engine/metadata-modules/utils/metadata.constants.ts → packages/twenty-server/src/engine/metadata-modules/utils/constants/identifier-max-char-length.constants.ts
View File

2
packages/twenty-server/src/engine/metadata-modules/utils/validate-database-identifier-length.utils.ts
View File

@ -1,4 +1,4 @@
import { IDENTIFIER_MAX_CHAR_LENGTH } from 'src/engine/metadata-modules/utils/metadata.constants';
import { IDENTIFIER_MAX_CHAR_LENGTH } from 'src/engine/metadata-modules/utils/constants/identifier-max-char-length.constants';
export const exceedsDatabaseIdentifierMaximumLength = (string: string) => {
return string.length > IDENTIFIER_MAX_CHAR_LENGTH;

2
packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/factories.ts
View File

@ -1,8 +1,10 @@
import { BasicColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/basic-column-action.factory';
import { CompositeColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { EnumColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/enum-column-action.factory';
import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory';
export const workspaceColumnActionFactories = [
TsVectorColumnActionFactory,
BasicColumnActionFactory,
EnumColumnActionFactory,
CompositeColumnActionFactory,

52
packages/twenty-server/src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory.ts Normal file
View File

@ -0,0 +1,52 @@
import { Injectable, Logger } from '@nestjs/common';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { WorkspaceColumnActionOptions } from 'src/engine/metadata-modules/workspace-migration/interfaces/workspace-column-action-options.interface';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { computeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { ColumnActionAbstractFactory } from 'src/engine/metadata-modules/workspace-migration/factories/column-action-abstract.factory';
import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
import {
WorkspaceMigrationColumnActionType,
WorkspaceMigrationColumnAlter,
WorkspaceMigrationColumnCreate,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import {
WorkspaceMigrationException,
WorkspaceMigrationExceptionCode,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception';
export type TsVectorFieldMetadataType = FieldMetadataType.TS_VECTOR;
@Injectable()
export class TsVectorColumnActionFactory extends ColumnActionAbstractFactory<TsVectorFieldMetadataType> {
protected readonly logger = new Logger(TsVectorColumnActionFactory.name);
handleCreateAction(
fieldMetadata: FieldMetadataInterface<TsVectorFieldMetadataType>,
): WorkspaceMigrationColumnCreate[] {
return [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: computeColumnName(fieldMetadata),
columnType: fieldMetadataTypeToColumnType(fieldMetadata.type),
isNullable: fieldMetadata.isNullable ?? true,
defaultValue: undefined,
generatedType: fieldMetadata.generatedType,
asExpression: fieldMetadata.asExpression,
},
];
}
protected handleAlterAction(
_currentFieldMetadata: FieldMetadataInterface<TsVectorFieldMetadataType>,
_alteredFieldMetadata: FieldMetadataInterface<TsVectorFieldMetadataType>,
_options?: WorkspaceColumnActionOptions,
): WorkspaceMigrationColumnAlter[] {
throw new WorkspaceMigrationException(
`TsVectorColumnActionFactory.handleAlterAction has not been implemented yet.`,
WorkspaceMigrationExceptionCode.INVALID_FIELD_METADATA,
);
}
}

2
packages/twenty-server/src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util.ts
View File

@ -38,6 +38,8 @@ export const fieldMetadataTypeToColumnType = <Type extends FieldMetadataType>(
return 'enum';
case FieldMetadataType.RAW_JSON:
return 'jsonb';
case FieldMetadataType.TS_VECTOR:
return 'tsvector';
default:
throw new WorkspaceMigrationException(
`Cannot convert ${fieldMetadataType} to column type.`,

4
packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.entity.ts
View File

@ -5,6 +5,7 @@ import {
PrimaryGeneratedColumn,
} from 'typeorm';
import { IndexType } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { RelationOnDeleteAction } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
export enum WorkspaceMigrationColumnActionType {
@ -30,12 +31,15 @@ export interface WorkspaceMigrationColumnDefinition {
isArray?: boolean;
isNullable: boolean;
defaultValue: any;
generatedType?: 'STORED' | 'VIRTUAL';
asExpression?: string;
}
export interface WorkspaceMigrationIndexAction {
action: WorkspaceMigrationIndexActionType;
name: string;
columns: string[];
type?: IndexType;
}
export interface WorkspaceMigrationColumnCreate

6
packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.factory.ts
View File

@ -8,6 +8,7 @@ import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/fi
import { BasicColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/basic-column-action.factory';
import { CompositeColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { EnumColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/enum-column-action.factory';
import { TsVectorColumnActionFactory } from 'src/engine/metadata-modules/workspace-migration/factories/ts-vector-column-action.factory';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
@ -30,6 +31,7 @@ export class WorkspaceMigrationFactory {
constructor(
private readonly basicColumnActionFactory: BasicColumnActionFactory,
private readonly tsVectorColumnActionFactory: TsVectorColumnActionFactory,
private readonly enumColumnActionFactory: EnumColumnActionFactory,
private readonly compositeColumnActionFactory: CompositeColumnActionFactory,
) {
@ -106,6 +108,10 @@ export class WorkspaceMigrationFactory {
FieldMetadataType.PHONES,
{ factory: this.compositeColumnActionFactory },
],
[
FieldMetadataType.TS_VECTOR,
{ factory: this.tsVectorColumnActionFactory },
],
]);
}

8
packages/twenty-server/src/engine/metadata-modules/workspace-migration/workspace-migration.module.ts
View File

@ -4,8 +4,8 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { workspaceColumnActionFactories } from 'src/engine/metadata-modules/workspace-migration/factories/factories';
import { WorkspaceMigrationFactory } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.factory';
import { WorkspaceMigrationService } from './workspace-migration.service';
import { WorkspaceMigrationEntity } from './workspace-migration.entity';
import { WorkspaceMigrationService } from './workspace-migration.service';
@Module({
imports: [TypeOrmModule.forFeature([WorkspaceMigrationEntity], 'metadata')],
@ -14,6 +14,10 @@ import { WorkspaceMigrationEntity } from './workspace-migration.entity';
WorkspaceMigrationFactory,
WorkspaceMigrationService,
],
exports: [WorkspaceMigrationFactory, WorkspaceMigrationService],
exports: [
...workspaceColumnActionFactories,
WorkspaceMigrationFactory,
WorkspaceMigrationService,
],
})
export class WorkspaceMigrationModule {}