mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: Add TS vector field filters support (#12376)

Browse Source 
# Implementation Details
- Added support for 5 operators: `contains`, `containsAny`,
`containsAll`, `matches`, and `fuzzy`
- Works on any field of type `TS_VECTOR`
- Added PostgreSQL `pg_trgm` extension for fuzzy search functionality.
The extension provides the `similarity()` function needed for text
similarity searches.
- Not implemented in GraphQL

## Tradeoffs & Decisions
1. **Fuzzy Search Performance**: Using `pg_trgm` for fuzzy search is
more accurate but slower than simple text matching. We might want to add
a similarity threshold parameter in the future to control the tradeoff
between accuracy and performance.

2. **Operator Naming**: Chose `contains`/`containsAny`/`containsAll` to
be consistent with existing filter operators, though they might be less
intuitive than `search`/`searchAny`/`searchAll`.

## Demo


https://github.com/user-attachments/assets/790fc3ed-a188-4b49-864f-996a37481d99

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
Co-authored-by: Félix Malfait <felix@twenty.com>
This commit is contained in:
Abdul Rahman
2025-05-30 19:24:50 +05:30
committed by GitHub
parent b7473371b3
commit c7139cbc84
13 changed files with 76 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-filter/graphql-query-filter-field.parser.ts
View File

@ -64,12 +64,12 @@ export class GraphqlQueryFilterFieldParser {
);
}
const { sql, params } = computeWhereConditionParts(
const { sql, params } = computeWhereConditionParts({
operator,
objectNameSingular,
key,
value,
);
});
if (isFirst) {
queryBuilder.where(sql, params);
@ -124,12 +124,12 @@ export class GraphqlQueryFilterFieldParser {
);
}
const { sql, params } = computeWhereConditionParts(
const { sql, params } = computeWhereConditionParts({
operator,
objectNameSingular,
fullFieldName,
key: fullFieldName,
value,
);
});
if (isFirst && index === 0) {
queryBuilder.where(sql, params);

35
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/compute-where-condition-parts.ts
View File

@ -10,13 +10,18 @@ type WhereConditionParts = {
params: ObjectLiteral;
};
export const computeWhereConditionParts = (
operator: string,
objectNameSingular: string,
key: string,
export const computeWhereConditionParts = ({
operator,
objectNameSingular,
key,
value,
}: {
operator: string;
objectNameSingular: string;
key: string;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
value: any,
): WhereConditionParts => {
value: any;
}): WhereConditionParts => {
const uuid = Math.random().toString(36).slice(2, 7);
switch (operator) {
@ -90,6 +95,23 @@ export const computeWhereConditionParts = (
sql: `"${objectNameSingular}"."${key}" @> ARRAY[:...${key}${uuid}]`,
params: { [`${key}${uuid}`]: value },
};
case 'search': {
const tsQuery = value
.split(/\s+/)
.map((term: string) => `${term}:*`)
.join(' & ');
return {
sql: `(
"${objectNameSingular}"."${key}" @@ to_tsquery('simple', :${key}${uuid}Ts) OR
"${objectNameSingular}"."${key}"::text ILIKE :${key}${uuid}Like
)`,
params: {
[`${key}${uuid}Ts`]: tsQuery,
[`${key}${uuid}Like`]: `%${value}%`,
},
};
}
case 'notContains':
return {
sql: `NOT ("${objectNameSingular}"."${key}"::text[] && ARRAY[:...${key}${uuid}]::text[])`,
@ -105,7 +127,6 @@ export const computeWhereConditionParts = (
sql: `EXISTS (SELECT 1 FROM unnest("${objectNameSingular}"."${key}") AS elem WHERE elem ILIKE :${key}${uuid})`,
params: { [`${key}${uuid}`]: value },
};
default:
throw new GraphqlQueryRunnerException(
`Operator "${operator}" is not supported`,

8
packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type.ts Normal file
View File

@ -0,0 +1,8 @@
import { GraphQLInputObjectType, GraphQLString } from 'graphql';
export const TSVectorFilterType = new GraphQLInputObjectType({
name: 'TSVectorFilter',
fields: {
search: { type: GraphQLString },
},
});

5
packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/index.ts
View File

@ -5,13 +5,17 @@ import { DateScalarType } from './date.scalar';
import { PositionScalarType } from './position.scalar';
import { RawJSONScalar } from './raw-json.scalar';
import { TimeScalarType } from './time.scalar';
import { TSVectorScalarType } from './ts-vector.scalar';
import { UUIDScalarType } from './uuid.scalar';
export * from './big-float.scalar';
export * from './big-int.scalar';
export * from './cursor.scalar';
export * from './date.scalar';
export * from './position.scalar';
export * from './raw-json.scalar';
export * from './time.scalar';
export * from './ts-vector.scalar';
export * from './uuid.scalar';
export const scalars = [
@ -23,4 +27,5 @@ export const scalars = [
CursorScalarType,
PositionScalarType,
RawJSONScalar,
TSVectorScalarType,
];

19
packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/ts-vector.scalar.ts Normal file
View File

@ -0,0 +1,19 @@
import { GraphQLScalarType } from 'graphql';
export const TSVectorScalarType = new GraphQLScalarType({
name: 'TSVector',
description: 'A custom scalar type for PostgreSQL tsvector fields',
serialize(value: string): string {
return value;
},
parseValue(value: string): string {
return value;
},
parseLiteral(ast): string | null {
if (ast.kind === 'StringValue') {
return ast.value;
}
return null;
},
});

6
packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/services/type-mapper.service.ts
View File

@ -30,9 +30,11 @@ import {
import { MultiSelectFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/multi-select-filter.input-type';
import { RichTextV2FilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/rich-text.input-type';
import { SelectFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/select-filter.input-type';
import { TSVectorFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/ts-vector-filter.input-type';
import { UUIDFilterType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/uuid-filter.input-type';
import {
BigFloatScalarType,
TSVectorScalarType,
UUIDScalarType,
} from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { PositionScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars/position.scalar';
@ -83,7 +85,7 @@ export class TypeMapperService {
StringArrayScalarType as unknown as GraphQLScalarType,
],
[FieldMetadataType.RICH_TEXT, GraphQLString],
[FieldMetadataType.TS_VECTOR, GraphQLString],
[FieldMetadataType.TS_VECTOR, TSVectorScalarType],
]);
return typeScalarMapping.get(fieldMetadataType);
@ -122,7 +124,7 @@ export class TypeMapperService {
[FieldMetadataType.ARRAY, ArrayFilterType],
[FieldMetadataType.MULTI_SELECT, MultiSelectFilterType],
[FieldMetadataType.SELECT, SelectFilterType],
[FieldMetadataType.TS_VECTOR, StringFilterType], // TODO: Add TSVectorFilterType
[FieldMetadataType.TS_VECTOR, TSVectorFilterType],
]);
return typeFilterMapping.get(fieldMetadataType);

3
packages/twenty-server/src/engine/twenty-orm/custom.workspace-entity.ts
View File

@ -158,6 +158,5 @@ export class CustomWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
@WorkspaceIsSystem()
@WorkspaceFieldIndex({ indexType: IndexType.GIN })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
searchVector: any;
searchVector: string;
}

3
packages/twenty-server/src/modules/company/standard-objects/company.workspace-entity.ts
View File

@ -285,6 +285,5 @@ export class CompanyWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
@WorkspaceIsSystem()
@WorkspaceFieldIndex({ indexType: IndexType.GIN })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
searchVector: any;
searchVector: string;
}

3
packages/twenty-server/src/modules/note/standard-objects/note.workspace-entity.ts
View File

@ -160,6 +160,5 @@ export class NoteWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
@WorkspaceIsSystem()
@WorkspaceFieldIndex({ indexType: IndexType.GIN })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
searchVector: any;
searchVector: string;
}

3
packages/twenty-server/src/modules/opportunity/standard-objects/opportunity.workspace-entity.ts
View File

@ -240,6 +240,5 @@ export class OpportunityWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
@WorkspaceIsSystem()
@WorkspaceFieldIndex({ indexType: IndexType.GIN })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
searchVector: any;
searchVector: string;
}

3
packages/twenty-server/src/modules/person/standard-objects/person.workspace-entity.ts
View File

@ -304,6 +304,5 @@ export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
@WorkspaceIsSystem()
@WorkspaceFieldIndex({ indexType: IndexType.GIN })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
searchVector: any;
searchVector: string;
}

3
packages/twenty-server/src/modules/task/standard-objects/task.workspace-entity.ts
View File

@ -215,6 +215,5 @@ export class TaskWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
@WorkspaceIsSystem()
@WorkspaceFieldIndex({ indexType: IndexType.GIN })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
searchVector: any;
searchVector: string;
}

3
packages/twenty-server/src/modules/workspace-member/standard-objects/workspace-member.workspace-entity.ts
View File

@ -358,6 +358,5 @@ export class WorkspaceMemberWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceIsNullable()
@WorkspaceIsSystem()
@WorkspaceFieldIndex({ indexType: IndexType.GIN })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
searchVector: any;
searchVector: string;
}