admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Update searchVector at label identifier update for custom fields (#7588)

Browse Source 
By default, when custom fields are created, a searchVector field is
created based on the "name" field, which is also the label identifier by
default.
When this label identifier is updated, we want to update the
searchVector field to use this field as searchable field instead, if it
is of "searchable type" (today it is only possible to select a text or
number field as label identifier, while number fields are not
searchable).
This commit is contained in:
Marie
2024-10-15 16:34:05 +02:00
committed by GitHub
parent b1cc7b7dbb
commit 1de739176c
29 changed files with 535 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags.ts
View File

@ -1,6 +1 @@
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
export const DEFAULT_FEATURE_FLAGS = [
FeatureFlagKey.IsSearchEnabled,
FeatureFlagKey.IsWorkspaceMigratedForSearch,
];
export const DEFAULT_FEATURE_FLAGS = [];

31
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts
View File

@ -10,7 +10,6 @@ import {
} from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { CustomWorkspaceEntity } from 'src/engine/twenty-orm/custom.workspace-entity';
@ -145,26 +144,13 @@ export class WorkspaceSyncFieldMetadataService {
const originalObjectMetadata =
originalObjectMetadataMap[standardObjectId];
let computedStandardFieldMetadataCollection = computeStandardFields(
const computedStandardFieldMetadataCollection = computeStandardFields(
standardFieldMetadataCollection,
originalObjectMetadata,
// We need to provide this for generated relations with custom objects
customObjectMetadataCollection,
);
let originalObjectMetadataFields = originalObjectMetadata.fields;
if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) {
computedStandardFieldMetadataCollection =
computedStandardFieldMetadataCollection.filter(
(field) => field.type !== FieldMetadataType.TS_VECTOR,
);
originalObjectMetadataFields = originalObjectMetadataFields.filter(
(field) => field.type !== FieldMetadataType.TS_VECTOR,
);
}
const fieldComparatorResults = this.workspaceFieldComparator.compare(
originalObjectMetadata.id,
originalObjectMetadata.fields,
@ -192,24 +178,11 @@ export class WorkspaceSyncFieldMetadataService {
// Loop over all custom objects from the DB and compare their fields with standard fields
for (const customObjectMetadata of customObjectMetadataCollection) {
// Also, maybe it's better to refactor a bit and move generation part into a separate module ?
let standardFieldMetadataCollection = computeStandardFields(
const standardFieldMetadataCollection = computeStandardFields(
customObjectStandardFieldMetadataCollection,
customObjectMetadata,
);
let customObjectMetadataFields = customObjectMetadata.fields;
if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) {
standardFieldMetadataCollection =
standardFieldMetadataCollection.filter(
(field) => field.type !== FieldMetadataType.TS_VECTOR,
);
customObjectMetadataFields = customObjectMetadataFields.filter(
(field) => field.type !== FieldMetadataType.TS_VECTOR,
);
}
/**
* COMPARE FIELD METADATA
*/

18
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts
View File

@ -7,10 +7,7 @@ import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/wo
import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import {
IndexMetadataEntity,
IndexType,
} from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory';
@ -73,7 +70,7 @@ export class WorkspaceSyncIndexMetadataService {
const indexMetadataRepository = manager.getRepository(IndexMetadataEntity);
let originalIndexMetadataCollection = await indexMetadataRepository.find({
const originalIndexMetadataCollection = await indexMetadataRepository.find({
where: {
workspaceId: context.workspaceId,
objectMetadataId: Any(
@ -87,7 +84,7 @@ export class WorkspaceSyncIndexMetadataService {
});
// Generate index metadata from models
let standardIndexMetadataCollection = this.standardIndexFactory.create(
const standardIndexMetadataCollection = this.standardIndexFactory.create(
standardObjectMetadataDefinitions,
context,
originalStandardObjectMetadataMap,
@ -95,15 +92,6 @@ export class WorkspaceSyncIndexMetadataService {
workspaceFeatureFlagsMap,
);
if (!workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) {
originalIndexMetadataCollection = originalIndexMetadataCollection.filter(
(index) => index.indexType !== IndexType.GIN,
);
standardIndexMetadataCollection = standardIndexMetadataCollection.filter(
(index) => index.indexType !== IndexType.GIN,
);
}
const indexComparatorResults = this.workspaceIndexComparator.compare(
originalIndexMetadataCollection,
standardIndexMetadataCollection,

13
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts
View File

@ -1,5 +1,8 @@
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util';
import {
FieldTypeAndNameMetadata,
getTsVectorColumnExpressionFromFields,
} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util';
const nameTextField = { name: 'name', type: FieldMetadataType.TEXT };
const nameFullNameField = {
@ -63,14 +66,18 @@ jest.mock(
describe('getTsVectorColumnExpressionFromFields', () => {
it('should generate correct expression for simple text field', () => {
const fields = [nameTextField];
const fields = [nameTextField] as FieldTypeAndNameMetadata[];
const result = getTsVectorColumnExpressionFromFields(fields);
expect(result).toContain("to_tsvector('simple', COALESCE(\"name\", ''))");
});
it('should handle multiple fields', () => {
const fields = [nameFullNameField, jobTitleTextField, emailsEmailsField];
const fields = [
nameFullNameField,
jobTitleTextField,
emailsEmailsField,
] as FieldTypeAndNameMetadata[];
const result = getTsVectorColumnExpressionFromFields(fields);
const expected = `
to_tsvector('simple', COALESCE("nameFirstName", '') || ' ' || COALESCE("nameLastName", '') || ' ' || COALESCE("jobTitle", '') || ' ' ||

16
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts
View File

@ -9,15 +9,27 @@ import {
WorkspaceMigrationException,
WorkspaceMigrationExceptionCode,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.exception';
import {
isSearchableFieldType,
SearchableFieldType,
} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util';
type FieldTypeAndNameMetadata = {
export type FieldTypeAndNameMetadata = {
name: string;
type: FieldMetadataType;
type: SearchableFieldType;
};
export const getTsVectorColumnExpressionFromFields = (
fieldsUsedForSearch: FieldTypeAndNameMetadata[],
): string => {
const filteredFieldsUsedForSearch = fieldsUsedForSearch.filter((field) =>
isSearchableFieldType(field.type),
);
if (filteredFieldsUsedForSearch.length < 1) {
throw new Error('No searchable fields found');
}
const columnExpressions = fieldsUsedForSearch.flatMap(
getColumnExpressionsFromField,
);

17
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/is-searchable-field.util.ts Normal file
View File

@ -0,0 +1,17 @@
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
const SEARCHABLE_FIELD_TYPES = [
FieldMetadataType.TEXT,
FieldMetadataType.FULL_NAME,
FieldMetadataType.EMAILS,
FieldMetadataType.ADDRESS,
FieldMetadataType.LINKS,
] as const;
export type SearchableFieldType = (typeof SEARCHABLE_FIELD_TYPES)[number];
export const isSearchableFieldType = (
type: FieldMetadataType,
): type is SearchableFieldType => {
return SEARCHABLE_FIELD_TYPES.includes(type as SearchableFieldType);
};

8
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts
View File

@ -5,7 +5,6 @@ import { DataSource, QueryFailedError, Repository } from 'typeorm';
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
@ -153,13 +152,6 @@ export class WorkspaceSyncMetadataService {
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
context.workspaceId,
);
if (workspaceFeatureFlagsMap.IS_SEARCH_ENABLED) {
await this.featureFlagService.enableFeatureFlags(
[FeatureFlagKey.IsWorkspaceMigratedForSearch],
context.workspaceId,
);
}
} catch (error) {
this.logger.error('Sync of standard objects failed with:', error);