From c7139cbc84c5074bbafff71a47bf4fb31fe686f8 Mon Sep 17 00:00:00 2001 From: Abdul Rahman <81605929+abdulrahmancodes@users.noreply.github.com> Date: Fri, 30 May 2025 19:24:50 +0530 Subject: [PATCH] feat: Add TS vector field filters support (#12376) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Implementation Details - Added support for 5 operators: `contains`, `containsAny`, `containsAll`, `matches`, and `fuzzy` - Works on any field of type `TS_VECTOR` - Added PostgreSQL `pg_trgm` extension for fuzzy search functionality. The extension provides the `similarity()` function needed for text similarity searches. - Not implemented in GraphQL ## Tradeoffs & Decisions 1. **Fuzzy Search Performance**: Using `pg_trgm` for fuzzy search is more accurate but slower than simple text matching. We might want to add a similarity threshold parameter in the future to control the tradeoff between accuracy and performance. 2. **Operator Naming**: Chose `contains`/`containsAny`/`containsAll` to be consistent with existing filter operators, though they might be less intuitive than `search`/`searchAny`/`searchAll`. ## Demo https://github.com/user-attachments/assets/790fc3ed-a188-4b49-864f-996a37481d99 --------- Co-authored-by: Félix Malfait Co-authored-by: Félix Malfait --- .../graphql-query-filter-field.parser.ts | 10 +++--- .../utils/compute-where-condition-parts.ts | 35 +++++++++++++++---- .../input/ts-vector-filter.input-type.ts | 8 +++++ .../graphql-types/scalars/index.ts | 5 +++ .../graphql-types/scalars/ts-vector.scalar.ts | 19 ++++++++++ .../services/type-mapper.service.ts | 6 ++-- .../twenty-orm/custom.workspace-entity.ts | 3 +- .../company.workspace-entity.ts | 3 +- .../standard-objects/note.workspace-entity.ts | 3 +- .../opportunity.workspace-entity.ts | 3 +- .../person.workspace-entity.ts | 3 +- .../standard-objects/task.workspace-entity.ts | 3 +- .../workspace-member.workspace-entity.ts | 3 +- 13 files changed, 76 insertions(+), 28 deletions(-) create mode 100644 packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/ts-vector.scalar.ts diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts index fd3a93c97..1c963a818 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts @@ -64,12 +64,12 @@ export class GraphqlQueryFilterFieldParser { ); } - const { sql, params } = computeWhereConditionParts( + const { sql, params } = computeWhereConditionParts({ operator, objectNameSingular, key, value, - ); + }); if (isFirst) { queryBuilder.where(sql, params); @@ -124,12 +124,12 @@ export class GraphqlQueryFilterFieldParser { ); } - const { sql, params } = computeWhereConditionParts( + const { sql, params } = computeWhereConditionParts({ operator, objectNameSingular, - fullFieldName, + key: fullFieldName, value, - ); + }); if (isFirst && index === 0) { queryBuilder.where(sql, params); diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts.ts index 729f6c0c2..159caca2f 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts.ts @@ -10,13 +10,18 @@ type WhereConditionParts = { params: ObjectLiteral; }; -export const computeWhereConditionParts = ( - operator: string, - objectNameSingular: string, - key: string, +export const computeWhereConditionParts = ({ + operator, + objectNameSingular, + key, + value, +}: { + operator: string; + objectNameSingular: string; + key: string; // eslint-disable-next-line @typescript-eslint/no-explicit-any - value: any, -): WhereConditionParts => { + value: any; +}): WhereConditionParts => { const uuid = Math.random().toString(36).slice(2, 7); switch (operator) { @@ -90,6 +95,23 @@ export const computeWhereConditionParts = ( sql: `"${objectNameSingular}"."${key}" @> ARRAY[:...${key}${uuid}]`, params: { [`${key}${uuid}`]: value }, }; + case 'search': { + const tsQuery = value + .split(/\s+/) + .map((term: string) => `${term}:*`) + .join(' & '); + + return { + sql: `( + "${objectNameSingular}"."${key}" @@ to_tsquery('simple', :${key}${uuid}Ts) OR + "${objectNameSingular}"."${key}"::text ILIKE :${key}${uuid}Like + )`, + params: { + [`${key}${uuid}Ts`]: tsQuery, + [`${key}${uuid}Like`]: `%${value}%`, + }, + }; + } case 'notContains': return { sql: `NOT ("${objectNameSingular}"."${key}"::text[] && ARRAY[:...${key}${uuid}]::text[])`, @@ -105,7 +127,6 @@ export const computeWhereConditionParts = ( sql: `EXISTS (SELECT 1 FROM unnest("${objectNameSingular}"."${key}") AS elem WHERE elem ILIKE :${key}${uuid})`, params: { [`${key}${uuid}`]: value }, }; - default: throw new GraphqlQueryRunnerException( `Operator "${operator}" is not supported`, diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type.ts new file mode 100644 index 000000000..719ddca4f --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type.ts @@ -0,0 +1,8 @@ +import { GraphQLInputObjectType, GraphQLString } from 'graphql'; + +export const TSVectorFilterType = new GraphQLInputObjectType({ + name: 'TSVectorFilter', + fields: { + search: { type: GraphQLString }, + }, +}); diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/index.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/index.ts index e5aec2019..203848eed 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/index.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/index.ts @@ -5,13 +5,17 @@ import { DateScalarType } from './date.scalar'; import { PositionScalarType } from './position.scalar'; import { RawJSONScalar } from './raw-json.scalar'; import { TimeScalarType } from './time.scalar'; +import { TSVectorScalarType } from './ts-vector.scalar'; import { UUIDScalarType } from './uuid.scalar'; export * from './big-float.scalar'; export * from './big-int.scalar'; export * from './cursor.scalar'; export * from './date.scalar'; +export * from './position.scalar'; +export * from './raw-json.scalar'; export * from './time.scalar'; +export * from './ts-vector.scalar'; export * from './uuid.scalar'; export const scalars = [ @@ -23,4 +27,5 @@ export const scalars = [ CursorScalarType, PositionScalarType, RawJSONScalar, + TSVectorScalarType, ]; diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/ts-vector.scalar.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/ts-vector.scalar.ts new file mode 100644 index 000000000..c1a65d8f9 --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/ts-vector.scalar.ts @@ -0,0 +1,19 @@ +import { GraphQLScalarType } from 'graphql'; + +export const TSVectorScalarType = new GraphQLScalarType({ + name: 'TSVector', + description: 'A custom scalar type for PostgreSQL tsvector fields', + serialize(value: string): string { + return value; + }, + parseValue(value: string): string { + return value; + }, + parseLiteral(ast): string | null { + if (ast.kind === 'StringValue') { + return ast.value; + } + + return null; + }, +}); diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/services/type-mapper.service.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/services/type-mapper.service.ts index 370f2d038..53207aa1b 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/services/type-mapper.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/services/type-mapper.service.ts @@ -30,9 +30,11 @@ import { import { MultiSelectFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/multi-select-filter.input-type'; import { RichTextV2FilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/rich-text.input-type'; import { SelectFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/select-filter.input-type'; +import { TSVectorFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type'; import { UUIDFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/uuid-filter.input-type'; import { BigFloatScalarType, + TSVectorScalarType, UUIDScalarType, } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars'; import { PositionScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/position.scalar'; @@ -83,7 +85,7 @@ export class TypeMapperService { StringArrayScalarType as unknown as GraphQLScalarType, ], [FieldMetadataType.RICH_TEXT, GraphQLString], - [FieldMetadataType.TS_VECTOR, GraphQLString], + [FieldMetadataType.TS_VECTOR, TSVectorScalarType], ]); return typeScalarMapping.get(fieldMetadataType); @@ -122,7 +124,7 @@ export class TypeMapperService { [FieldMetadataType.ARRAY, ArrayFilterType], [FieldMetadataType.MULTI_SELECT, MultiSelectFilterType], [FieldMetadataType.SELECT, SelectFilterType], - [FieldMetadataType.TS_VECTOR, StringFilterType], // TODO: Add TSVectorFilterType + [FieldMetadataType.TS_VECTOR, TSVectorFilterType], ]); return typeFilterMapping.get(fieldMetadataType); diff --git a/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts b/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts index 0752729b0..d67e50cfe 100644 --- a/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts +++ b/packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts @@ -158,6 +158,5 @@ export class CustomWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceIsNullable() @WorkspaceIsSystem() @WorkspaceFieldIndex({ indexType: IndexType.GIN }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchVector: any; + searchVector: string; } diff --git a/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts b/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts index d910a8a89..ead4cbc5e 100644 --- a/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts +++ b/packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts @@ -285,6 +285,5 @@ export class CompanyWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceIsNullable() @WorkspaceIsSystem() @WorkspaceFieldIndex({ indexType: IndexType.GIN }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchVector: any; + searchVector: string; } diff --git a/packages/twenty-server/src/modules/note/standard-objects/note.workspace-entity.ts b/packages/twenty-server/src/modules/note/standard-objects/note.workspace-entity.ts index 11c11e754..7c24eeeea 100644 --- a/packages/twenty-server/src/modules/note/standard-objects/note.workspace-entity.ts +++ b/packages/twenty-server/src/modules/note/standard-objects/note.workspace-entity.ts @@ -160,6 +160,5 @@ export class NoteWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceIsNullable() @WorkspaceIsSystem() @WorkspaceFieldIndex({ indexType: IndexType.GIN }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchVector: any; + searchVector: string; } diff --git a/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts b/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts index ff58f24b2..77b1350d8 100644 --- a/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts +++ b/packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts @@ -240,6 +240,5 @@ export class OpportunityWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceIsNullable() @WorkspaceIsSystem() @WorkspaceFieldIndex({ indexType: IndexType.GIN }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchVector: any; + searchVector: string; } diff --git a/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts b/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts index d34ebeec1..1a056d9bd 100644 --- a/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts +++ b/packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts @@ -304,6 +304,5 @@ export class PersonWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceIsNullable() @WorkspaceIsSystem() @WorkspaceFieldIndex({ indexType: IndexType.GIN }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchVector: any; + searchVector: string; } diff --git a/packages/twenty-server/src/modules/task/standard-objects/task.workspace-entity.ts b/packages/twenty-server/src/modules/task/standard-objects/task.workspace-entity.ts index 5e9b866ab..e79a44234 100644 --- a/packages/twenty-server/src/modules/task/standard-objects/task.workspace-entity.ts +++ b/packages/twenty-server/src/modules/task/standard-objects/task.workspace-entity.ts @@ -215,6 +215,5 @@ export class TaskWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceIsNullable() @WorkspaceIsSystem() @WorkspaceFieldIndex({ indexType: IndexType.GIN }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchVector: any; + searchVector: string; } diff --git a/packages/twenty-server/src/modules/workspace-member/standard-objects/workspace-member.workspace-entity.ts b/packages/twenty-server/src/modules/workspace-member/standard-objects/workspace-member.workspace-entity.ts index 54b18b90b..29075a405 100644 --- a/packages/twenty-server/src/modules/workspace-member/standard-objects/workspace-member.workspace-entity.ts +++ b/packages/twenty-server/src/modules/workspace-member/standard-objects/workspace-member.workspace-entity.ts @@ -358,6 +358,5 @@ export class WorkspaceMemberWorkspaceEntity extends BaseWorkspaceEntity { @WorkspaceIsNullable() @WorkspaceIsSystem() @WorkspaceFieldIndex({ indexType: IndexType.GIN }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - searchVector: any; + searchVector: string; }