feat: workspace sync (#3505)
* feat: wip workspace sync * feat: wip lot of debugging * feat: refactor and fix sync * fix: clean fix: clean * feat: add simple comparator tests * fix: remove debug * feat: wip drop table * fix: main merge * fix: some issues, and prepare storage system to handle complex deletion * feat: wip clean and fix * fix: reflect issue when using array instead of map and clean * fix: test & sync * fix: yarn files * fix: unecesary if-else * fix: if condition not needed * fix: remove debug * fix: replace EQUAL by SKIP * fix: sync metadata relation not applied properly * fix: lint issues * fix: merge issue
This commit is contained in:
Jérémy M
@ -0,0 +1,189 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager, In } from 'typeorm';
|
||||
import { v4 as uuidV4 } from 'uuid';
|
||||
import omit from 'lodash.omit';
|
||||
|
||||
import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
|
||||
import {
|
||||
FieldMetadataEntity,
|
||||
FieldMetadataType,
|
||||
} from 'src/metadata/field-metadata/field-metadata.entity';
|
||||
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
|
||||
import { FieldMetadataComplexOption } from 'src/metadata/field-metadata/dtos/options.input';
|
||||
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceMetadataUpdaterService {
|
||||
private readonly logger = new Logger(WorkspaceMetadataUpdaterService.name);
|
||||
|
||||
async updateObjectMetadata(
|
||||
manager: EntityManager,
|
||||
storage: WorkspaceSyncStorage,
|
||||
): Promise<{
|
||||
createdObjectMetadataCollection: ObjectMetadataEntity[];
|
||||
updatedObjectMetadataCollection: ObjectMetadataEntity[];
|
||||
}> {
|
||||
const objectMetadataRepository =
|
||||
manager.getRepository(ObjectMetadataEntity);
|
||||
|
||||
/**
|
||||
* Create object metadata
|
||||
*/
|
||||
const createdPartialObjectMetadataCollection =
|
||||
await objectMetadataRepository.save(
|
||||
storage.objectMetadataCreateCollection.map((objectMetadata) => ({
|
||||
...objectMetadata,
|
||||
isActive: true,
|
||||
fields: objectMetadata.fields.map((field) =>
|
||||
this.prepareFieldMetadataForCreation(field),
|
||||
),
|
||||
})) as DeepPartial<ObjectMetadataEntity>[],
|
||||
);
|
||||
const identifiers = createdPartialObjectMetadataCollection.map(
|
||||
(object) => object.id,
|
||||
);
|
||||
const createdObjectMetadataCollection = await manager.find(
|
||||
ObjectMetadataEntity,
|
||||
{
|
||||
where: { id: In(identifiers) },
|
||||
relations: ['dataSource', 'fields'],
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* Update object metadata
|
||||
*/
|
||||
const updatedObjectMetadataCollection = await objectMetadataRepository.save(
|
||||
storage.objectMetadataUpdateCollection.map((objectMetadata) =>
|
||||
omit(objectMetadata, ['fields']),
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* Delete object metadata
|
||||
*/
|
||||
if (storage.objectMetadataDeleteCollection.length > 0) {
|
||||
await objectMetadataRepository.delete(
|
||||
storage.objectMetadataDeleteCollection.map((object) => object.id),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
createdObjectMetadataCollection,
|
||||
updatedObjectMetadataCollection,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: Refactor this
|
||||
*/
|
||||
private prepareFieldMetadataForCreation(field: PartialFieldMetadata) {
|
||||
return {
|
||||
...field,
|
||||
...(field.type === FieldMetadataType.SELECT && field.options
|
||||
? {
|
||||
options: this.generateUUIDForNewSelectFieldOptions(
|
||||
field.options as FieldMetadataComplexOption[],
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
isActive: true,
|
||||
};
|
||||
}
|
||||
|
||||
private generateUUIDForNewSelectFieldOptions(
|
||||
options: FieldMetadataComplexOption[],
|
||||
): FieldMetadataComplexOption[] {
|
||||
return options.map((option) => ({
|
||||
...option,
|
||||
id: uuidV4(),
|
||||
}));
|
||||
}
|
||||
|
||||
async updateFieldMetadata(
|
||||
manager: EntityManager,
|
||||
storage: WorkspaceSyncStorage,
|
||||
): Promise<{
|
||||
createdFieldMetadataCollection: FieldMetadataEntity[];
|
||||
updatedFieldMetadataCollection: FieldMetadataEntity[];
|
||||
}> {
|
||||
const fieldMetadataRepository = manager.getRepository(FieldMetadataEntity);
|
||||
|
||||
/**
|
||||
* Create field metadata
|
||||
*/
|
||||
const createdFieldMetadataCollection = await fieldMetadataRepository.save(
|
||||
storage.fieldMetadataCreateCollection.map((field) =>
|
||||
this.prepareFieldMetadataForCreation(field),
|
||||
) as DeepPartial<FieldMetadataEntity>[],
|
||||
);
|
||||
|
||||
/**
|
||||
* Update field metadata
|
||||
*/
|
||||
const updatedFieldMetadataCollection = await fieldMetadataRepository.save(
|
||||
storage.fieldMetadataUpdateCollection as DeepPartial<FieldMetadataEntity>[],
|
||||
);
|
||||
|
||||
/**
|
||||
* Delete field metadata
|
||||
*/
|
||||
// TODO: handle relation fields deletion. We need to delete the relation metadata first due to the DB constraint.
|
||||
const fieldMetadataDeleteCollectionWithoutRelationType =
|
||||
storage.fieldMetadataDeleteCollection.filter(
|
||||
(field) => field.type !== FieldMetadataType.RELATION,
|
||||
);
|
||||
|
||||
if (fieldMetadataDeleteCollectionWithoutRelationType.length > 0) {
|
||||
await fieldMetadataRepository.delete(
|
||||
fieldMetadataDeleteCollectionWithoutRelationType.map(
|
||||
(field) => field.id,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
createdFieldMetadataCollection:
|
||||
createdFieldMetadataCollection as FieldMetadataEntity[],
|
||||
updatedFieldMetadataCollection:
|
||||
updatedFieldMetadataCollection as FieldMetadataEntity[],
|
||||
};
|
||||
}
|
||||
|
||||
async updateRelationMetadata(
|
||||
manager: EntityManager,
|
||||
storage: WorkspaceSyncStorage,
|
||||
): Promise<{
|
||||
createdRelationMetadataCollection: RelationMetadataEntity[];
|
||||
}> {
|
||||
const relationMetadataRepository = manager.getRepository(
|
||||
RelationMetadataEntity,
|
||||
);
|
||||
|
||||
/**
|
||||
* Create relation metadata
|
||||
*/
|
||||
const createdRelationMetadataCollection =
|
||||
await relationMetadataRepository.save(
|
||||
storage.relationMetadataCreateCollection,
|
||||
);
|
||||
|
||||
/**
|
||||
* Delete relation metadata
|
||||
*/
|
||||
if (storage.relationMetadataDeleteCollection.length > 0) {
|
||||
await relationMetadataRepository.delete(
|
||||
storage.relationMetadataDeleteCollection.map(
|
||||
(relationMetadata) => relationMetadata.id,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
createdRelationMetadataCollection,
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,154 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
|
||||
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
|
||||
import { mapObjectMetadataByUniqueIdentifier } from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util';
|
||||
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
|
||||
import { StandardObjectFactory } from 'src/workspace/workspace-sync-metadata/factories/standard-object.factory';
|
||||
import { WorkspaceObjectComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-object.comparator';
|
||||
import { WorkspaceFieldComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-field.comparator';
|
||||
import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service';
|
||||
import { WorkspaceSyncFactory } from 'src/workspace/workspace-sync-metadata/factories/workspace-sync.factory';
|
||||
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceSyncObjectMetadataService {
|
||||
private readonly logger = new Logger(WorkspaceSyncObjectMetadataService.name);
|
||||
|
||||
constructor(
|
||||
private readonly standardObjectFactory: StandardObjectFactory,
|
||||
private readonly workspaceObjectComparator: WorkspaceObjectComparator,
|
||||
private readonly workspaceFieldComparator: WorkspaceFieldComparator,
|
||||
private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService,
|
||||
private readonly workspaceSyncFactory: WorkspaceSyncFactory,
|
||||
) {}
|
||||
|
||||
async synchronize(
|
||||
context: WorkspaceSyncContext,
|
||||
manager: EntityManager,
|
||||
storage: WorkspaceSyncStorage,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): Promise<void> {
|
||||
const objectMetadataRepository =
|
||||
manager.getRepository(ObjectMetadataEntity);
|
||||
const workspaceMigrationRepository = manager.getRepository(
|
||||
WorkspaceMigrationEntity,
|
||||
);
|
||||
|
||||
// Retrieve object metadata collection from DB
|
||||
const originalObjectMetadataCollection =
|
||||
await objectMetadataRepository.find({
|
||||
where: { workspaceId: context.workspaceId, isCustom: false },
|
||||
relations: ['dataSource', 'fields'],
|
||||
});
|
||||
|
||||
// Create standard object metadata collection
|
||||
const standardObjectMetadataCollection = this.standardObjectFactory.create(
|
||||
context,
|
||||
workspaceFeatureFlagsMap,
|
||||
);
|
||||
|
||||
// Create map of original and standard object metadata by unique identifier
|
||||
const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
|
||||
originalObjectMetadataCollection,
|
||||
);
|
||||
const standardObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
|
||||
standardObjectMetadataCollection,
|
||||
);
|
||||
|
||||
this.logger.log('Comparing standard objects and fields metadata');
|
||||
|
||||
// Store object that need to be deleted
|
||||
for (const originalObjectMetadata of originalObjectMetadataCollection) {
|
||||
if (!standardObjectMetadataMap[originalObjectMetadata.nameSingular]) {
|
||||
storage.addDeleteObjectMetadata(originalObjectMetadata);
|
||||
}
|
||||
}
|
||||
|
||||
// Loop over all standard objects and compare them with the objects in DB
|
||||
for (const standardObjectName in standardObjectMetadataMap) {
|
||||
const originalObjectMetadata =
|
||||
originalObjectMetadataMap[standardObjectName];
|
||||
const standardObjectMetadata =
|
||||
standardObjectMetadataMap[standardObjectName];
|
||||
|
||||
/**
|
||||
* COMPARE OBJECT METADATA
|
||||
*/
|
||||
const objectComparatorResult = this.workspaceObjectComparator.compare(
|
||||
originalObjectMetadata,
|
||||
standardObjectMetadata,
|
||||
);
|
||||
|
||||
if (objectComparatorResult.action === ComparatorAction.CREATE) {
|
||||
storage.addCreateObjectMetadata(standardObjectMetadata);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (objectComparatorResult.action === ComparatorAction.UPDATE) {
|
||||
storage.addUpdateObjectMetadata(objectComparatorResult.object);
|
||||
}
|
||||
|
||||
/**
|
||||
* COMPARE FIELD METADATA
|
||||
*/
|
||||
const fieldComparatorResults = this.workspaceFieldComparator.compare(
|
||||
originalObjectMetadata,
|
||||
standardObjectMetadata,
|
||||
);
|
||||
|
||||
for (const fieldComparatorResult of fieldComparatorResults) {
|
||||
switch (fieldComparatorResult.action) {
|
||||
case ComparatorAction.CREATE: {
|
||||
storage.addCreateFieldMetadata(fieldComparatorResult.object);
|
||||
break;
|
||||
}
|
||||
case ComparatorAction.UPDATE: {
|
||||
storage.addUpdateFieldMetadata(fieldComparatorResult.object);
|
||||
break;
|
||||
}
|
||||
case ComparatorAction.DELETE: {
|
||||
storage.addDeleteFieldMetadata(fieldComparatorResult.object);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log('Updating workspace metadata');
|
||||
|
||||
// Apply changes to DB
|
||||
const metadataObjectUpdaterResult =
|
||||
await this.workspaceMetadataUpdaterService.updateObjectMetadata(
|
||||
manager,
|
||||
storage,
|
||||
);
|
||||
const metadataFieldUpdaterResult =
|
||||
await this.workspaceMetadataUpdaterService.updateFieldMetadata(
|
||||
manager,
|
||||
storage,
|
||||
);
|
||||
|
||||
this.logger.log('Generating migrations');
|
||||
|
||||
// Create migrations
|
||||
const workspaceObjectMigrations =
|
||||
await this.workspaceSyncFactory.createObjectMigration(
|
||||
originalObjectMetadataCollection,
|
||||
metadataObjectUpdaterResult.createdObjectMetadataCollection,
|
||||
storage.objectMetadataDeleteCollection,
|
||||
metadataFieldUpdaterResult.createdFieldMetadataCollection,
|
||||
storage.fieldMetadataDeleteCollection,
|
||||
);
|
||||
|
||||
this.logger.log('Saving migrations');
|
||||
|
||||
// Save migrations into DB
|
||||
await workspaceMigrationRepository.save(workspaceObjectMigrations);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,105 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { EntityManager } from 'typeorm';
|
||||
|
||||
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
|
||||
import { ComparatorAction } from 'src/workspace/workspace-sync-metadata/interfaces/comparator.interface';
|
||||
|
||||
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
|
||||
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
|
||||
import { mapObjectMetadataByUniqueIdentifier } from 'src/workspace/workspace-sync-metadata/utils/sync-metadata.util';
|
||||
import { StandardRelationFactory } from 'src/workspace/workspace-sync-metadata/factories/standard-relation.factory';
|
||||
import { WorkspaceRelationComparator } from 'src/workspace/workspace-sync-metadata/comparators/workspace-relation.comparator';
|
||||
import { WorkspaceMetadataUpdaterService } from 'src/workspace/workspace-sync-metadata/services/workspace-metadata-updater.service';
|
||||
import { WorkspaceSyncFactory } from 'src/workspace/workspace-sync-metadata/factories/workspace-sync.factory';
|
||||
import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity';
|
||||
import { WorkspaceSyncStorage } from 'src/workspace/workspace-sync-metadata/storage/workspace-sync.storage';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceSyncRelationMetadataService {
|
||||
private readonly logger = new Logger(
|
||||
WorkspaceSyncRelationMetadataService.name,
|
||||
);
|
||||
|
||||
constructor(
|
||||
private readonly standardRelationFactory: StandardRelationFactory,
|
||||
private readonly workspaceRelationComparator: WorkspaceRelationComparator,
|
||||
private readonly workspaceMetadataUpdaterService: WorkspaceMetadataUpdaterService,
|
||||
private readonly workspaceSyncFactory: WorkspaceSyncFactory,
|
||||
) {}
|
||||
|
||||
async synchronize(
|
||||
context: WorkspaceSyncContext,
|
||||
manager: EntityManager,
|
||||
storage: WorkspaceSyncStorage,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): Promise<void> {
|
||||
const objectMetadataRepository =
|
||||
manager.getRepository(ObjectMetadataEntity);
|
||||
const workspaceMigrationRepository = manager.getRepository(
|
||||
WorkspaceMigrationEntity,
|
||||
);
|
||||
|
||||
// Retrieve object metadata collection from DB
|
||||
const originalObjectMetadataCollection =
|
||||
await objectMetadataRepository.find({
|
||||
where: { workspaceId: context.workspaceId, isCustom: false },
|
||||
relations: ['dataSource', 'fields'],
|
||||
});
|
||||
|
||||
// Create map of object metadata & field metadata by unique identifier
|
||||
const originalObjectMetadataMap = mapObjectMetadataByUniqueIdentifier(
|
||||
originalObjectMetadataCollection,
|
||||
);
|
||||
|
||||
const relationMetadataRepository = manager.getRepository(
|
||||
RelationMetadataEntity,
|
||||
);
|
||||
|
||||
// Retrieve relation metadata collection from DB
|
||||
// TODO: filter out custom relations once isCustom has been added to relationMetadata table
|
||||
const originalRelationMetadataCollection =
|
||||
await relationMetadataRepository.find({
|
||||
where: { workspaceId: context.workspaceId },
|
||||
});
|
||||
|
||||
// Create standard relation metadata collection
|
||||
const standardRelationMetadataCollection =
|
||||
this.standardRelationFactory.create(
|
||||
context,
|
||||
originalObjectMetadataMap,
|
||||
workspaceFeatureFlagsMap,
|
||||
);
|
||||
|
||||
const relationComparatorResults = this.workspaceRelationComparator.compare(
|
||||
originalRelationMetadataCollection,
|
||||
standardRelationMetadataCollection,
|
||||
);
|
||||
|
||||
for (const relationComparatorResult of relationComparatorResults) {
|
||||
if (relationComparatorResult.action === ComparatorAction.CREATE) {
|
||||
storage.addCreateRelationMetadata(relationComparatorResult.object);
|
||||
} else if (relationComparatorResult.action === ComparatorAction.DELETE) {
|
||||
storage.addDeleteRelationMetadata(relationComparatorResult.object);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataRelationUpdaterResult =
|
||||
await this.workspaceMetadataUpdaterService.updateRelationMetadata(
|
||||
manager,
|
||||
storage,
|
||||
);
|
||||
|
||||
// Create migrations
|
||||
const workspaceRelationMigrations =
|
||||
await this.workspaceSyncFactory.createRelationMigration(
|
||||
originalObjectMetadataCollection,
|
||||
metadataRelationUpdaterResult.createdRelationMetadataCollection,
|
||||
storage.relationMetadataDeleteCollection,
|
||||
);
|
||||
|
||||
// Save migrations into DB
|
||||
await workspaceMigrationRepository.save(workspaceRelationMigrations);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user