admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FlexibleSchema] Add IndexMetadata decorator (#5981)

Browse Source 
## Context
Our Flexible Schema engine dynamically generates entities/tables/APIs
for us but was not flexible enough to build indexes in the DB. With more
and more features involving heavy queries such as Messaging, we are now
adding a new WorkspaceIndex() decorator for our standard objects (will
come later for custom objects). This decorator will give enough
information to the workspace sync metadata manager to generate the
proper migrations that will create or drop indexes on demand.
To be aligned with the rest of the engine, we are adding 2 new tables:
IndexMetadata and IndexFieldMetadata, that will store the info of our
indexes.

## Implementation

```typescript
@WorkspaceEntity({
  standardId: STANDARD_OBJECT_IDS.person,
  namePlural: 'people',
  labelSingular: 'Person',
  labelPlural: 'People',
  description: 'A person',
  icon: 'IconUser',
})
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
  @WorkspaceField({
    standardId: PERSON_STANDARD_FIELD_IDS.email,
    type: FieldMetadataType.EMAIL,
    label: 'Email',
    description: 'Contact’s Email',
    icon: 'IconMail',
  })
  @WorkspaceIndex()
  email: string;
```
By simply adding the WorkspaceIndex decorator, sync-metadata command
will create a new index for that column.
We can also add composite indexes, note that the order is important for
PSQL.
```typescript
@WorkspaceEntity({
  standardId: STANDARD_OBJECT_IDS.person,
  namePlural: 'people',
  labelSingular: 'Person',
  labelPlural: 'People',
  description: 'A person',
  icon: 'IconUser',
})
@WorkspaceIndex(['phone', 'email'])
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
```

Currently composite fields and relation fields are not handled by
@WorkspaceIndex() and you will need to use this notation instead
```typescript
@WorkspaceIndex(['companyId', 'nameFirstName'])
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
```
<img width="700" alt="Screenshot 2024-06-21 at 15 15 45"
src="https://github.com/twentyhq/twenty/assets/1834158/ac6da1d9-d315-40a4-9ba6-6ab9ae4709d4">

Next step: We might need to implement more complex index expressions,
this is why we have an expression column in IndexMetadata.
What I had in mind for the decorator, still open to discussion
```typescript
@WorkspaceIndex(['nameFirstName', 'nameLastName'], { expression: "$1 || ' ' || $2"})
export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
```

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Weiko
2024-06-22 12:39:57 +02:00
committed by GitHub
parent 0b4bfce324
commit e13dc7a1fc
29 changed files with 871 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service.ts
View File

@ -11,6 +11,7 @@ import { v4 as uuidV4 } from 'uuid';
import { DeepPartial } from 'typeorm/common/DeepPartial';
import { PartialFieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
import { PartialIndexMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/partial-index-metadata.interface';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import {
@ -22,6 +23,7 @@ import { FieldMetadataComplexOption } from 'src/engine/metadata-modules/field-me
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
import { FieldMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-field.factory';
import { ObjectMetadataUpdate } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-object.factory';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
@Injectable()
export class WorkspaceMetadataUpdaterService {
@ -230,6 +232,69 @@ export class WorkspaceMetadataUpdaterService {
};
}
async updateIndexMetadata(
manager: EntityManager,
storage: WorkspaceSyncStorage,
originalObjectMetadataCollection: ObjectMetadataEntity[],
): Promise<{
createdIndexMetadataCollection: IndexMetadataEntity[];
}> {
const indexMetadataRepository = manager.getRepository(IndexMetadataEntity);
const convertIndexMetadataForSaving = (
indexMetadata: PartialIndexMetadata,
) => {
const convertIndexFieldMetadataForSaving = (
column: string,
order: number,
) => {
const fieldMetadata = originalObjectMetadataCollection
.find((object) => object.id === indexMetadata.objectMetadataId)
?.fields.find((field) => column === field.name);
if (!fieldMetadata) {
throw new Error(`
Field metadata not found for column ${column} in object ${indexMetadata.objectMetadataId}
`);
}
return {
fieldMetadataId: fieldMetadata.id,
order,
};
};
return {
...indexMetadata,
indexFieldMetadatas: indexMetadata.columns.map((column, index) =>
convertIndexFieldMetadataForSaving(column, index),
),
};
};
/**
* Create index metadata
*/
const createdIndexMetadataCollection = await indexMetadataRepository.save(
storage.indexMetadataCreateCollection.map(convertIndexMetadataForSaving),
);
/**
* Delete index metadata
*/
if (storage.indexMetadataDeleteCollection.length > 0) {
await indexMetadataRepository.delete(
storage.indexMetadataDeleteCollection.map(
(indexMetadata) => indexMetadata.id,
),
);
}
return {
createdIndexMetadataCollection,
};
}
/**
* Update entities in the database
* @param manager EntityManager

4
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service.ts
View File

@ -48,7 +48,7 @@ export class WorkspaceSyncFieldMetadataService {
relations: ['dataSource', 'fields'],
});
// Filter out custom objects
// Filter out non-custom objects
const customObjectMetadataCollection =
originalObjectMetadataCollection.filter(
(objectMetadata) => objectMetadata.isCustom,
@ -61,7 +61,7 @@ export class WorkspaceSyncFieldMetadataService {
workspaceFeatureFlagsMap,
);
// Loop over all standard objects and compare them with the objects in DB
// Loop over all custom objects from the DB and compare their fields with standard fields
for (const customObjectMetadata of customObjectMetadataCollection) {
// Also, maybe it's better to refactor a bit and move generation part into a separate module ?
const standardObjectMetadata = computeStandardObject(

119
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-index-metadata.service.ts Normal file
View File

@ -0,0 +1,119 @@
import { Injectable, Logger } from '@nestjs/common';
import { EntityManager } from 'typeorm';
import { FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interfaces/feature-flag-map.interface';
import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
import { ComparatorAction } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/comparator.interface';
import { WorkspaceMigrationBuilderAction } from 'src/engine/workspace-manager/workspace-migration-builder/interfaces/workspace-migration-builder-action.interface';
import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceSyncStorage } from 'src/engine/workspace-manager/workspace-sync-metadata/storage/workspace-sync.storage';
import { StandardIndexFactory } from 'src/engine/workspace-manager/workspace-sync-metadata/factories/standard-index.factory';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { mapObjectMetadataByUniqueIdentifier } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/sync-metadata.util';
import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity';
import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects';
import { WorkspaceIndexComparator } from 'src/engine/workspace-manager/workspace-sync-metadata/comparators/workspace-index.comparator';
import { WorkspaceMetadataUpdaterService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-metadata-updater.service';
import { WorkspaceMigrationIndexFactory } from 'src/engine/workspace-manager/workspace-migration-builder/factories/workspace-migration-index.factory';
@Injectable()
export class WorkspaceSyncIndexMetadataService {
private readonly logger = new Logger(WorkspaceSyncIndexMetadataService.name);
constructor(
private readonly standardIndexFactory: StandardIndexFactory,
private readonly workspaceIndexComparator: WorkspaceIndexComparator,
private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService,
private readonly workspaceMigrationIndexFactory: WorkspaceMigrationIndexFactory,
) {}
async synchronize(
context: WorkspaceSyncContext,
manager: EntityManager,
storage: WorkspaceSyncStorage,
workspaceFeatureFlagsMap: FeatureFlagMap,
): Promise<Partial<WorkspaceMigrationEntity>[]> {
this.logger.log('Syncing index metadata');
const objectMetadataRepository =
manager.getRepository(ObjectMetadataEntity);
// Retrieve object metadata collection from DB
const originalObjectMetadataCollection =
await objectMetadataRepository.find({
where: {
workspaceId: context.workspaceId,
// We're only interested in standard fields
fields: { isCustom: false },
isCustom: false,
},
relations: ['dataSource', 'fields', 'indexes'],
});
// Create map of object metadata & field metadata by unique identifier
const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
originalObjectMetadataCollection,
// Relation are based on the singular name
(objectMetadata) => objectMetadata.nameSingular,
);
const indexMetadataRepository = manager.getRepository(IndexMetadataEntity);
const originalIndexMetadataCollection = await indexMetadataRepository.find({
where: {
workspaceId: context.workspaceId,
},
relations: ['indexFieldMetadatas.fieldMetadata'],
});
// Generate index metadata from models
const standardIndexMetadataCollection = this.standardIndexFactory.create(
standardObjectMetadataDefinitions,
context,
originalObjectMetadataMap,
workspaceFeatureFlagsMap,
);
const indexComparatorResults = this.workspaceIndexComparator.compare(
originalIndexMetadataCollection,
standardIndexMetadataCollection,
);
for (const indexComparatorResult of indexComparatorResults) {
if (indexComparatorResult.action === ComparatorAction.CREATE) {
storage.addCreateIndexMetadata(indexComparatorResult.object);
} else if (indexComparatorResult.action === ComparatorAction.DELETE) {
storage.addDeleteIndexMetadata(indexComparatorResult.object);
}
}
const metadataIndexUpdaterResult =
await this.workspaceMetadataUpdaterService.updateIndexMetadata(
manager,
storage,
originalObjectMetadataCollection,
);
// Create migrations
const createIndexWorkspaceMigrations =
await this.workspaceMigrationIndexFactory.create(
originalObjectMetadataCollection,
metadataIndexUpdaterResult.createdIndexMetadataCollection,
WorkspaceMigrationBuilderAction.CREATE,
);
const deleteIndexWorkspaceMigrations =
await this.workspaceMigrationIndexFactory.create(
originalObjectMetadataCollection,
storage.indexMetadataDeleteCollection,
WorkspaceMigrationBuilderAction.DELETE,
);
return [
...createIndexWorkspaceMigrations,
...deleteIndexWorkspaceMigrations,
];
}
}

4
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-object-metadata.service.ts
View File

@ -112,6 +112,10 @@ export class WorkspaceSyncObjectMetadataService {
/**
* COMPARE FIELD METADATA
* NOTE: This should be moved to WorkspaceSyncFieldMetadataService for more clarity since
* this code only adds field metadata to the storage but it's actually used in the other service.
* NOTE2: WorkspaceSyncFieldMetadataService has been added for custom fields sync, it should be refactored to handle
* both custom and non-custom fields.
*/
const fieldComparatorResults = this.workspaceFieldComparator.compare(
originalObjectMetadata,