diff --git a/server/package.json b/server/package.json index 410ff83aa..4814217f4 100644 --- a/server/package.json +++ b/server/package.json @@ -84,6 +84,7 @@ "lodash.merge": "^4.6.2", "lodash.snakecase": "^4.1.1", "lodash.upperfirst": "^4.3.1", + "microdiff": "^1.3.2", "nest-commander": "^3.12.0", "passport": "^0.6.0", "passport-google-oauth20": "^2.0.0", @@ -143,4 +144,4 @@ "resolutions": { "graphql": "16.8.0" } -} \ No newline at end of file +} diff --git a/server/src/database/typeorm-seeds/metadata/field-metadata/view-sort.ts b/server/src/database/typeorm-seeds/metadata/field-metadata/view-sort.ts index d90bd4842..e5fa45b16 100644 --- a/server/src/database/typeorm-seeds/metadata/field-metadata/view-sort.ts +++ b/server/src/database/typeorm-seeds/metadata/field-metadata/view-sort.ts @@ -122,13 +122,13 @@ export const seedViewSortFieldMetadata = async ( isCustom: false, workspaceId: SeedWorkspaceId, isActive: true, - type: FieldMetadataType.UUID, - name: 'viewId', - label: 'View Id', + type: FieldMetadataType.RELATION, + name: 'view', + label: 'View', targetColumnMap: {}, description: 'View Sort related view', icon: 'IconLayoutCollage', - isNullable: false, + isNullable: true, isSystem: false, defaultValue: undefined, }, diff --git a/server/src/database/typeorm-seeds/metadata/index.ts b/server/src/database/typeorm-seeds/metadata/index.ts index 37fe3aebd..e5a26dff3 100644 --- a/server/src/database/typeorm-seeds/metadata/index.ts +++ b/server/src/database/typeorm-seeds/metadata/index.ts @@ -23,6 +23,7 @@ import { seedPipelineStepRelationMetadata } from 'src/database/typeorm-seeds/met import { seedPersonRelationMetadata } from 'src/database/typeorm-seeds/metadata/relation-metadata/person'; import { seedWorkspaceMemberRelationMetadata } from 'src/database/typeorm-seeds/metadata/relation-metadata/workspace-member'; import { seedDataSource } from 'src/database/typeorm-seeds/metadata/data-source'; +import { seedWebhookFieldMetadata } from 'src/database/typeorm-seeds/metadata/field-metadata/webhook'; export const seedMetadataSchema = async (workspaceDataSource: DataSource) => { const schemaName = 'metadata'; @@ -33,6 +34,7 @@ export const seedMetadataSchema = async (workspaceDataSource: DataSource) => { await seedActivityFieldMetadata(workspaceDataSource, schemaName); await seedApiKeyFieldMetadata(workspaceDataSource, schemaName); await seedAttachmentFieldMetadata(workspaceDataSource, schemaName); + await seedWebhookFieldMetadata(workspaceDataSource, schemaName); await seedCommentFieldMetadata(workspaceDataSource, schemaName); await seedCompanyFieldMetadata(workspaceDataSource, schemaName); await seedFavoriteFieldMetadata(workspaceDataSource, schemaName); diff --git a/server/src/metadata/field-metadata/utils/generate-target-column-map.util.ts b/server/src/metadata/field-metadata/utils/generate-target-column-map.util.ts index 69ddb21d0..78abf2ef0 100644 --- a/server/src/metadata/field-metadata/utils/generate-target-column-map.util.ts +++ b/server/src/metadata/field-metadata/utils/generate-target-column-map.util.ts @@ -52,7 +52,8 @@ export function generateTargetColumnMap( firstName: `${columnName}FirstName`, lastName: `${columnName}LastName`, }; - + case FieldMetadataType.RELATION: + return {}; default: throw new BadRequestException(`Unknown type ${type}`); } diff --git a/server/src/workspace/workspace-manager/commands/sync-workspace-metadata.command.ts b/server/src/workspace/workspace-manager/commands/sync-workspace-metadata.command.ts index cfea77a47..f13cf4be1 100644 --- a/server/src/workspace/workspace-manager/commands/sync-workspace-metadata.command.ts +++ b/server/src/workspace/workspace-manager/commands/sync-workspace-metadata.command.ts @@ -31,7 +31,7 @@ export class SyncWorkspaceMetadataCommand extends CommandRunner { ); // TODO: This solution could be improved, using a diff for example, we should not have to delete all metadata and recreate them. - await this.workspaceManagerService.resetStandardObjectsAndFieldsMetadata( + await this.workspaceManagerService.syncStandardObjectsAndFieldsMetadata( dataSourceMetadata.id, options.workspaceId, ); diff --git a/server/src/workspace/workspace-manager/decorators/metadata.decorator.ts b/server/src/workspace/workspace-manager/decorators/metadata.decorator.ts new file mode 100644 index 000000000..b29ce3d9e --- /dev/null +++ b/server/src/workspace/workspace-manager/decorators/metadata.decorator.ts @@ -0,0 +1,103 @@ +import camelCase from 'lodash.camelcase'; +import 'reflect-metadata'; + +import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface'; + +import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; +import { generateTargetColumnMap } from 'src/metadata/field-metadata/utils/generate-target-column-map.util'; + +export type FieldMetadataDecorator = { + type: FieldMetadataType; + label: string; + description?: string | null; + icon?: string | null; + defaultValue?: FieldMetadataDefaultValue | null; +}; + +export type ObjectMetadataDecorator = { + namePlural: string; + labelSingular: string; + labelPlural: string; + description?: string | null; + icon?: string | null; +}; + +const classSuffix = 'ObjectMetadata'; + +export function ObjectMetadata( + metadata: ObjectMetadataDecorator, +): ClassDecorator { + return (target) => { + const isSystem = Reflect.getMetadata('isSystem', target) || false; + + let objectName = camelCase(target.name); + + if (objectName.endsWith(classSuffix)) { + objectName = objectName.slice(0, -classSuffix.length); + } + + Reflect.defineMetadata( + 'objectMetadata', + { + nameSingular: objectName, + ...metadata, + targetTableName: objectName, + isSystem, + isCustom: false, + isActive: true, + }, + target, + ); + }; +} + +export function IsNullable() { + return function (target: object, fieldKey: string) { + Reflect.defineMetadata('isNullable', true, target, fieldKey); + }; +} + +export function IsSystem() { + return function (target: object, fieldKey?: string) { + if (fieldKey) { + Reflect.defineMetadata('isSystem', true, target, fieldKey); + } else { + Reflect.defineMetadata('isSystem', true, target); + } + }; +} + +export function FieldMetadata( + metadata: FieldMetadataDecorator, +): PropertyDecorator { + return (target: object, fieldKey: string) => { + const existingFieldMetadata = + Reflect.getMetadata('fieldMetadata', target.constructor) || {}; + + const isNullable = + Reflect.getMetadata('isNullable', target, fieldKey) || false; + + const isSystem = Reflect.getMetadata('isSystem', target, fieldKey) || false; + + Reflect.defineMetadata( + 'fieldMetadata', + { + ...existingFieldMetadata, + [fieldKey]: { + name: fieldKey, + ...metadata, + targetColumnMap: generateTargetColumnMap( + metadata.type, + false, + fieldKey, + ), + isNullable, + isSystem, + isCustom: false, + isActive: true, + }, + }, + target.constructor, + ); + }; +} diff --git a/server/src/workspace/workspace-manager/standard-objects/api-key.object-metadata.ts b/server/src/workspace/workspace-manager/standard-objects/api-key.object-metadata.ts new file mode 100644 index 000000000..3b5eb45ad --- /dev/null +++ b/server/src/workspace/workspace-manager/standard-objects/api-key.object-metadata.ts @@ -0,0 +1,44 @@ +import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; +import { + FieldMetadata, + IsNullable, + IsSystem, + ObjectMetadata, +} from 'src/workspace/workspace-manager/decorators/metadata.decorator'; +import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata'; + +@ObjectMetadata({ + namePlural: 'apiKeys', + labelSingular: 'Api Key', + labelPlural: 'Api Keys', + description: 'A api key', + icon: 'IconRobot', +}) +@IsSystem() +export class ApiKeyObjectMetadata extends BaseObjectMetadata { + @FieldMetadata({ + type: FieldMetadataType.TEXT, + label: 'Name', + description: 'ApiKey name', + icon: 'IconLink', + defaultValue: { value: '' }, + }) + name: string; + + @FieldMetadata({ + type: FieldMetadataType.DATE_TIME, + label: 'Expiration date', + description: 'ApiKey expiration date', + icon: 'IconCalendar', + }) + expiresAt: Date; + + @FieldMetadata({ + type: FieldMetadataType.DATE_TIME, + label: 'Revocation date', + description: 'ApiKey revocation date', + icon: 'IconCalendar', + }) + @IsNullable() + revokedAt?: Date; +} diff --git a/server/src/workspace/workspace-manager/standard-objects/base.object-metadata.ts b/server/src/workspace/workspace-manager/standard-objects/base.object-metadata.ts new file mode 100644 index 000000000..9b23cbb76 --- /dev/null +++ b/server/src/workspace/workspace-manager/standard-objects/base.object-metadata.ts @@ -0,0 +1,37 @@ +import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; +import { + FieldMetadata, + IsSystem, +} from 'src/workspace/workspace-manager/decorators/metadata.decorator'; + +export abstract class BaseObjectMetadata { + @FieldMetadata({ + type: FieldMetadataType.UUID, + label: 'Id', + icon: null, + description: null, + defaultValue: { type: 'uuid' }, + }) + @IsSystem() + id: string; + + @FieldMetadata({ + type: FieldMetadataType.DATE_TIME, + label: 'Creation date', + description: null, + icon: 'IconCalendar', + defaultValue: { type: 'now' }, + }) + @IsSystem() + createdAt: Date; + + @FieldMetadata({ + type: FieldMetadataType.DATE_TIME, + label: 'Update date', + description: null, + icon: 'IconCalendar', + defaultValue: { type: 'now' }, + }) + @IsSystem() + updatedAt: Date; +} diff --git a/server/src/workspace/workspace-manager/standard-objects/standard-object-metadata.ts b/server/src/workspace/workspace-manager/standard-objects/standard-object-metadata.ts index e0064f189..f0596feab 100644 --- a/server/src/workspace/workspace-manager/standard-objects/standard-object-metadata.ts +++ b/server/src/workspace/workspace-manager/standard-objects/standard-object-metadata.ts @@ -1,23 +1,22 @@ -import activityTargetMetadata from 'src/workspace/workspace-manager/standard-objects/activity-target'; -import activityMetadata from 'src/workspace/workspace-manager/standard-objects/activity'; import apiKeyMetadata from 'src/workspace/workspace-manager/standard-objects/api-key'; -import attachmentMetadata from 'src/workspace/workspace-manager/standard-objects/attachment'; -import commentMetadata from 'src/workspace/workspace-manager/standard-objects/comment'; -import favoriteMetadata from 'src/workspace/workspace-manager/standard-objects/favorite'; -import opportunityMetadata from 'src/workspace/workspace-manager/standard-objects/opportunity'; -import personMetadata from 'src/workspace/workspace-manager/standard-objects/person'; -import viewMetadata from 'src/workspace/workspace-manager/standard-objects/view'; -import viewFieldMetadata from 'src/workspace/workspace-manager/standard-objects/view-field'; -import viewFilterMetadata from 'src/workspace/workspace-manager/standard-objects/view-filter'; -import viewSortMetadata from 'src/workspace/workspace-manager/standard-objects/view-sort'; -import webhookMetadata from 'src/workspace/workspace-manager/standard-objects/webhook'; -import pipelineStepMetadata from 'src/workspace/workspace-manager/standard-objects/pipeline-step'; -import companyMetadata from 'src/workspace/workspace-manager/standard-objects/company'; -import workspaceMemberMetadata from 'src/workspace/workspace-manager/standard-objects/workspace-member'; import { FieldMetadataEntity, FieldMetadataType, } from 'src/metadata/field-metadata/field-metadata.entity'; +import activityMetadata from 'src/workspace/workspace-manager/standard-objects/activity'; +import activityTargetMetadata from 'src/workspace/workspace-manager/standard-objects/activity-target'; +import attachmentMetadata from 'src/workspace/workspace-manager/standard-objects/attachment'; +import commentMetadata from 'src/workspace/workspace-manager/standard-objects/comment'; +import companyMetadata from 'src/workspace/workspace-manager/standard-objects/company'; +import favoriteMetadata from 'src/workspace/workspace-manager/standard-objects/favorite'; +import opportunityMetadata from 'src/workspace/workspace-manager/standard-objects/opportunity'; +import personMetadata from 'src/workspace/workspace-manager/standard-objects/person'; +import pipelineStepMetadata from 'src/workspace/workspace-manager/standard-objects/pipeline-step'; +import viewMetadata from 'src/workspace/workspace-manager/standard-objects/view'; +import viewFieldMetadata from 'src/workspace/workspace-manager/standard-objects/view-field'; +import viewFilterMetadata from 'src/workspace/workspace-manager/standard-objects/view-filter'; +import viewSortMetadata from 'src/workspace/workspace-manager/standard-objects/view-sort'; +import workspaceMemberMetadata from 'src/workspace/workspace-manager/standard-objects/workspace-member'; export const standardObjectsMetadata = { activityTarget: activityTargetMetadata, @@ -34,7 +33,6 @@ export const standardObjectsMetadata = { viewFilter: viewFilterMetadata, viewSort: viewSortMetadata, view: viewMetadata, - webhook: webhookMetadata, workspaceMember: workspaceMemberMetadata, }; diff --git a/server/src/workspace/workspace-manager/standard-objects/view-sort.object-metadata.ts b/server/src/workspace/workspace-manager/standard-objects/view-sort.object-metadata.ts new file mode 100644 index 000000000..53c790334 --- /dev/null +++ b/server/src/workspace/workspace-manager/standard-objects/view-sort.object-metadata.ts @@ -0,0 +1,36 @@ +import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; +import { + ObjectMetadata, + FieldMetadata, + IsNullable, + IsSystem, +} from 'src/workspace/workspace-manager/decorators/metadata.decorator'; +import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata'; + +@ObjectMetadata({ + namePlural: 'viewSorts', + labelSingular: 'View Sort', + labelPlural: 'View Sorts', + description: '(System) View Sorts', + icon: 'IconArrowsSort', +}) +@IsSystem() +export class ViewSortObjectMetadata extends BaseObjectMetadata { + @FieldMetadata({ + type: FieldMetadataType.UUID, + label: 'Field Metadata Id', + description: 'View Sort target field', + icon: null, + }) + fieldMetadataId: string; + + // TODO: We could create a relation decorator but let's keep it simple for now. + @FieldMetadata({ + type: FieldMetadataType.RELATION, + label: 'View', + description: 'View Sort related view', + icon: 'IconLayoutCollage', + }) + @IsNullable() + view?: object; +} diff --git a/server/src/workspace/workspace-manager/standard-objects/webook.object-metadata.ts b/server/src/workspace/workspace-manager/standard-objects/webook.object-metadata.ts new file mode 100644 index 000000000..3a2448e2e --- /dev/null +++ b/server/src/workspace/workspace-manager/standard-objects/webook.object-metadata.ts @@ -0,0 +1,35 @@ +import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; +import { + FieldMetadata, + IsSystem, + ObjectMetadata, +} from 'src/workspace/workspace-manager/decorators/metadata.decorator'; +import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata'; + +@ObjectMetadata({ + namePlural: 'webhooks', + labelSingular: 'Webhook', + labelPlural: 'Webhooks', + description: 'A webhook', + icon: 'IconRobot', +}) +@IsSystem() +export class WebhookObjectMetadata extends BaseObjectMetadata { + @FieldMetadata({ + type: FieldMetadataType.TEXT, + label: 'Target Url', + description: 'Webhook target url', + icon: 'IconLink', + defaultValue: { value: '' }, + }) + targetUrl: string; + + @FieldMetadata({ + type: FieldMetadataType.TEXT, + label: 'Operation', + description: 'Webhook operation', + icon: 'IconCheckbox', + defaultValue: { value: '' }, + }) + operation: string; +} diff --git a/server/src/workspace/workspace-manager/utils/metadata.parser.ts b/server/src/workspace/workspace-manager/utils/metadata.parser.ts new file mode 100644 index 000000000..73f26cb8c --- /dev/null +++ b/server/src/workspace/workspace-manager/utils/metadata.parser.ts @@ -0,0 +1,41 @@ +import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity'; +import { BaseObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/base.object-metadata'; + +export class MetadataParser { + static parseMetadata( + metadata: typeof BaseObjectMetadata, + workspaceId: string, + dataSourceId: string, + ) { + const objectMetadata = Reflect.getMetadata('objectMetadata', metadata); + const fieldMetadata = Reflect.getMetadata('fieldMetadata', metadata); + + if (objectMetadata) { + const fields = Object.values(fieldMetadata); + return { + ...objectMetadata, + workspaceId, + dataSourceId, + fields: fields.map((field: FieldMetadataEntity) => ({ + ...field, + workspaceId, + isSystem: objectMetadata.isSystem || field.isSystem, + defaultValue: field.defaultValue || null, // TODO: use default default value based on field type + options: field.options || null, + })), + }; + } + + return undefined; + } + + static parseAllMetadata( + metadata: (typeof BaseObjectMetadata)[], + workspaceId: string, + dataSourceId: string, + ) { + return metadata.map((_metadata) => + MetadataParser.parseMetadata(_metadata, workspaceId, dataSourceId), + ); + } +} diff --git a/server/src/workspace/workspace-manager/utils/sync-metadata.util.spec.ts b/server/src/workspace/workspace-manager/utils/sync-metadata.util.spec.ts new file mode 100644 index 000000000..3a4bdaac4 --- /dev/null +++ b/server/src/workspace/workspace-manager/utils/sync-metadata.util.spec.ts @@ -0,0 +1,101 @@ +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; +import { FieldMetadataType } from 'src/metadata/field-metadata/field-metadata.entity'; + +import { + filterIgnoredProperties, + mapObjectMetadataByUniqueIdentifier, +} from './sync-metadata.util'; + +describe('filterIgnoredProperties', () => { + it('should filter out properties based on the ignore list', () => { + const obj = { + name: 'John', + age: 30, + email: 'john@example.com', + address: '123 Main St', + }; + const propertiesToIgnore = ['age', 'address']; + + const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore); + + expect(filteredObj).toEqual({ + name: 'John', + email: 'john@example.com', + }); + }); + + it('should return the original object if ignore list is empty', () => { + const obj = { + name: 'John', + age: 30, + email: 'john@example.com', + address: '123 Main St', + }; + const propertiesToIgnore: string[] = []; + + const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore); + + expect(filteredObj).toEqual(obj); + }); + + it('should return an empty object if the original object is empty', () => { + const obj = {}; + const propertiesToIgnore = ['age', 'address']; + + const filteredObj = filterIgnoredProperties(obj, propertiesToIgnore); + + expect(filteredObj).toEqual({}); + }); +}); + +describe('mapObjectMetadataByUniqueIdentifier', () => { + it('should convert an array of ObjectMetadataEntity objects into a map', () => { + const arr: DeepPartial[] = [ + { + nameSingular: 'user', + fields: [ + { name: 'id', type: FieldMetadataType.UUID }, + { name: 'name', type: FieldMetadataType.TEXT }, + ], + }, + { + nameSingular: 'product', + fields: [ + { name: 'id', type: FieldMetadataType.UUID }, + { name: 'name', type: FieldMetadataType.TEXT }, + { name: 'price', type: FieldMetadataType.UUID }, + ], + }, + ]; + + const mappedObject = mapObjectMetadataByUniqueIdentifier( + arr as ObjectMetadataEntity[], + ); + + expect(mappedObject).toEqual({ + user: { + nameSingular: 'user', + fields: { + id: { name: 'id', type: FieldMetadataType.UUID }, + name: { name: 'name', type: FieldMetadataType.TEXT }, + }, + }, + product: { + nameSingular: 'product', + fields: { + id: { name: 'id', type: FieldMetadataType.UUID }, + name: { name: 'name', type: FieldMetadataType.TEXT }, + price: { name: 'price', type: FieldMetadataType.UUID }, + }, + }, + }); + }); + + it('should return an empty map if the input array is empty', () => { + const arr: ObjectMetadataEntity[] = []; + + const mappedObject = mapObjectMetadataByUniqueIdentifier(arr); + + expect(mappedObject).toEqual({}); + }); +}); diff --git a/server/src/workspace/workspace-manager/utils/sync-metadata.util.ts b/server/src/workspace/workspace-manager/utils/sync-metadata.util.ts new file mode 100644 index 000000000..18c96742f --- /dev/null +++ b/server/src/workspace/workspace-manager/utils/sync-metadata.util.ts @@ -0,0 +1,41 @@ +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; + +/** + * This utility function filters out properties from an object based on a list of properties to ignore. + * It returns a new object with only the properties that are not in the ignore list. + * + * @param obj - The object to filter. + * @param propertiesToIgnore - An array of property names to ignore. + * @returns A new object with filtered properties. + */ +export const filterIgnoredProperties = ( + obj: any, + propertiesToIgnore: string[], +) => { + return Object.fromEntries( + Object.entries(obj).filter(([key]) => !propertiesToIgnore.includes(key)), + ); +}; + +/** + * This utility function converts an array of ObjectMetadataEntity objects into a map, + * where the keys are the nameSingular properties of the objects. + * Each object in the map contains the original object metadata and its fields as a nested map. + * + * @param arr - The array of ObjectMetadataEntity objects to convert. + * @returns A map of object metadata, with nameSingular as the key and the object as the value. + */ +export const mapObjectMetadataByUniqueIdentifier = ( + arr: ObjectMetadataEntity[], +) => { + return arr.reduce((acc, curr) => { + acc[curr.nameSingular] = { + ...curr, + fields: curr.fields.reduce((acc, curr) => { + acc[curr.name] = curr; + return acc; + }, {}), + }; + return acc; + }, {}); +}; diff --git a/server/src/workspace/workspace-manager/workspace-manager.module.ts b/server/src/workspace/workspace-manager/workspace-manager.module.ts index 80c69ead8..00a420ce1 100644 --- a/server/src/workspace/workspace-manager/workspace-manager.module.ts +++ b/server/src/workspace/workspace-manager/workspace-manager.module.ts @@ -1,12 +1,14 @@ import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; import { DataSourceModule } from 'src/metadata/data-source/data-source.module'; -import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module'; import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module'; import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module'; import { WorkspaceMigrationRunnerModule } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.module'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; import { RelationMetadataModule } from 'src/metadata/relation-metadata/relation-metadata.module'; +import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity'; +import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; import { WorkspaceManagerService } from './workspace-manager.service'; @@ -16,9 +18,12 @@ import { WorkspaceManagerService } from './workspace-manager.service'; WorkspaceMigrationModule, WorkspaceMigrationRunnerModule, ObjectMetadataModule, - FieldMetadataModule, DataSourceModule, RelationMetadataModule, + TypeOrmModule.forFeature( + [FieldMetadataEntity, ObjectMetadataEntity], + 'metadata', + ), ], exports: [WorkspaceManagerService], providers: [WorkspaceManagerService], diff --git a/server/src/workspace/workspace-manager/workspace-manager.service.ts b/server/src/workspace/workspace-manager/workspace-manager.service.ts index 12589bf3a..c97dec498 100644 --- a/server/src/workspace/workspace-manager/workspace-manager.service.ts +++ b/server/src/workspace/workspace-manager/workspace-manager.service.ts @@ -1,7 +1,10 @@ import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; +import diff from 'microdiff'; import { DataSourceService } from 'src/metadata/data-source/data-source.service'; -import { FieldMetadataService } from 'src/metadata/field-metadata/field-metadata.service'; import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metadata.service'; import { WorkspaceMigrationRunnerService } from 'src/workspace/workspace-migration-runner/workspace-migration-runner.service'; import { WorkspaceMigrationService } from 'src/metadata/workspace-migration/workspace-migration.service'; @@ -16,6 +19,14 @@ import { FieldMetadataEntity, FieldMetadataType, } from 'src/metadata/field-metadata/field-metadata.entity'; +import { MetadataParser } from 'src/workspace/workspace-manager/utils/metadata.parser'; +import { WebhookObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/webook.object-metadata'; +import { ApiKeyObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/api-key.object-metadata'; +import { ViewSortObjectMetadata } from 'src/workspace/workspace-manager/standard-objects/view-sort.object-metadata'; +import { + filterIgnoredProperties, + mapObjectMetadataByUniqueIdentifier, +} from 'src/workspace/workspace-manager/utils/sync-metadata.util'; import { basicFieldsMetadata, @@ -29,9 +40,13 @@ export class WorkspaceManagerService { private readonly workspaceMigrationService: WorkspaceMigrationService, private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService, private readonly objectMetadataService: ObjectMetadataService, - private readonly fieldMetadataService: FieldMetadataService, private readonly dataSourceService: DataSourceService, private readonly relationMetadataService: RelationMetadataService, + + @InjectRepository(ObjectMetadataEntity, 'metadata') + private readonly objectMetadataRepository: Repository, + @InjectRepository(FieldMetadataEntity, 'metadata') + private readonly fieldMetadataRepository: Repository, ) {} /** @@ -239,23 +254,167 @@ export class WorkspaceManagerService { /** * - * Reset all standard objects and fields metadata for a given workspace + * Sync all standard objects and fields metadata for a given workspace and data source + * This will update the metadata if it has changed and generate migrations based on the diff. * * @param dataSourceId * @param workspaceId */ - public async resetStandardObjectsAndFieldsMetadata( + public async syncStandardObjectsAndFieldsMetadata( dataSourceId: string, workspaceId: string, ) { - await this.objectMetadataService.deleteMany({ - workspaceId: { eq: workspaceId }, + const standardObjects = MetadataParser.parseAllMetadata( + [WebhookObjectMetadata, ApiKeyObjectMetadata, ViewSortObjectMetadata], + workspaceId, + dataSourceId, + ); + const objectsInDB = await this.objectMetadataRepository.find({ + where: { workspaceId, dataSourceId, isCustom: false }, + relations: ['fields'], }); - await this.createStandardObjectsAndFieldsMetadata( - dataSourceId, - workspaceId, + const objectsInDBByName = mapObjectMetadataByUniqueIdentifier(objectsInDB); + const standardObjectsByName = + mapObjectMetadataByUniqueIdentifier(standardObjects); + + const objectsToCreate: ObjectMetadataEntity[] = []; + const objectsToDelete = objectsInDB.filter( + (objectInDB) => !standardObjectsByName[objectInDB.nameSingular], ); + const objectsToUpdate: Record = {}; + + const fieldsToCreate: FieldMetadataEntity[] = []; + const fieldsToDelete: FieldMetadataEntity[] = []; + const fieldsToUpdate: Record = {}; + + for (const standardObjectName in standardObjectsByName) { + const standardObject = standardObjectsByName[standardObjectName]; + const objectInDB = objectsInDBByName[standardObjectName]; + + if (!objectInDB) { + objectsToCreate.push(standardObject); + continue; + } + + // Deconstruct fields and compare objects and fields independently + const { fields: objectInDBFields, ...objectInDBWithoutFields } = + objectInDB; + const { fields: standardObjectFields, ...standardObjectWithoutFields } = + standardObject; + + const objectPropertiesToIgnore = [ + 'id', + 'createdAt', + 'updatedAt', + 'labelIdentifierFieldMetadataId', + 'imageIdentifierFieldMetadataId', + ]; + const objectDiffWithoutIgnoredProperties = filterIgnoredProperties( + objectInDBWithoutFields, + objectPropertiesToIgnore, + ); + + const fieldPropertiesToIgnore = [ + 'id', + 'createdAt', + 'updatedAt', + 'objectMetadataId', + ]; + const objectInDBFieldsWithoutDefaultFields = Object.fromEntries( + Object.entries(objectInDBFields).map(([key, value]) => { + if (value === null || typeof value !== 'object') { + return [key, value]; + } + return [key, filterIgnoredProperties(value, fieldPropertiesToIgnore)]; + }), + ); + + // Compare objects + const objectDiff = diff( + objectDiffWithoutIgnoredProperties, + standardObjectWithoutFields, + ); + + // Compare fields + const fieldsDiff = diff( + objectInDBFieldsWithoutDefaultFields, + standardObjectFields, + ); + + for (const diff of objectDiff) { + // We only handle CHANGE here as REMOVE and CREATE are handled earlier. + if (diff.type === 'CHANGE') { + const property = diff.path[0]; + objectsToUpdate[objectInDB.id] = { + ...objectsToUpdate[objectInDB.id], + [property]: diff.value, + }; + } + } + + for (const diff of fieldsDiff) { + if (diff.type === 'CREATE') { + const fieldName = diff.path[0]; + const fieldMetadata = standardObjectFields[fieldName]; + fieldsToCreate.push(fieldMetadata); + } + if (diff.type === 'CHANGE') { + const fieldName = diff.path[0]; + const property = diff.path[diff.path.length - 1]; + const fieldMetadata = objectInDBFields[fieldName]; + fieldsToUpdate[fieldMetadata.id] = { + ...fieldsToUpdate[fieldMetadata.id], + [property]: diff.value, + }; + } + if (diff.type === 'REMOVE') { + const fieldName = diff.path[0]; + const fieldMetadata = objectInDBFields[fieldName]; + fieldsToDelete.push(fieldMetadata); + } + } + // console.log(standardObjectName + ':objectDiff', objectDiff); + // console.log(standardObjectName + ':fieldsDiff', fieldsDiff); + } + + // TODO: Sync relationMetadata + // NOTE: Relations are handled like any field during the diff, so we ignore the relationMetadata table + // during the diff as it depends on the 2 fieldMetadata that we will compare here. + // However we need to make sure the relationMetadata table is in sync with the fieldMetadata table. + + // TODO: Use transactions + // CREATE OBJECTS + try { + await this.objectMetadataRepository.save(objectsToCreate); + // UPDATE OBJECTS, this is not optimal as we are running n queries here. + for (const [key, value] of Object.entries(objectsToUpdate)) { + await this.objectMetadataRepository.update(key, value); + } + // DELETE OBJECTS + if (objectsToDelete.length > 0) { + await this.objectMetadataRepository.delete( + objectsToDelete.map((object) => object.id), + ); + } + + // CREATE FIELDS + await this.fieldMetadataRepository.save(fieldsToCreate); + // UPDATE FIELDS + for (const [key, value] of Object.entries(fieldsToUpdate)) { + await this.fieldMetadataRepository.update(key, value); + } + // DELETE FIELDS + if (fieldsToDelete.length > 0) { + await this.fieldMetadataRepository.delete( + fieldsToDelete.map((field) => field.id), + ); + } + } catch (e) { + console.error('Sync of standard objects failed with:', e); + } + + // TODO: Create migrations based on diff from above. } /** diff --git a/server/yarn.lock b/server/yarn.lock index b0218fc62..3de8ba2ac 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -6951,6 +6951,11 @@ methods@^1.1.2, methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== +microdiff@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/microdiff/-/microdiff-1.3.2.tgz#b4fec53aca97371d5409a354913a65be2daec11d" + integrity sha512-pKy60S2febliZIbwdfEQKTtL5bLNxOyiRRmD400gueYl9XcHyNGxzHSlJWn9IMHwYXT0yohPYL08+bGozVk8cQ== + micromatch@^4.0.0, micromatch@^4.0.2, micromatch@^4.0.4: version "4.0.5" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6"